diff --git a/ibm/terraform/main.tf b/ibm/terraform/main.tf new file mode 100644 index 0000000..bfc9815 --- /dev/null +++ b/ibm/terraform/main.tf @@ -0,0 +1,23 @@ +module "resource-group" { + source = "terraform-ibm-modules/resource-group/ibm" + version = "1.1.5" + + resource_group_name = "qdrant-example-rg" +} + +module "cluster" { + source = "./modules/cluster" + +# source = "terraform-ibm-modules/base-ocp-vpc/ibm" +# version = "3.18.3" + + ibmcloud_api_key = var.ibmcloud_api_key + cluster_name = var.cluster_name + resource_group_id = module.resource-group.resource_group_id + region = var.region + force_delete_storage = true + vpc_id = module.vpc.vpc_id + vpc_subnets = module.vpc.subnet_detail_map + worker_pools = var.worker_pools +} + diff --git a/ibm/terraform/modules/cluster/kconfig/.gitignore b/ibm/terraform/modules/cluster/kconfig/.gitignore new file mode 100644 index 0000000..632a28f --- /dev/null +++ b/ibm/terraform/modules/cluster/kconfig/.gitignore @@ -0,0 +1,6 @@ +# Ignore everything +* + +# But not these files... +!.gitignore +!README.md diff --git a/ibm/terraform/modules/cluster/main.tf b/ibm/terraform/modules/cluster/main.tf new file mode 100644 index 0000000..65fe68f --- /dev/null +++ b/ibm/terraform/modules/cluster/main.tf @@ -0,0 +1,533 @@ +############################################################################## +# base-ocp-vpc-module +# Deploy Openshift cluster in IBM Cloud on VPC Gen 2 +############################################################################## + +# Segregate pools, as we need default pool for cluster creation +locals { + # ibm_container_vpc_cluster automatically names default pool "default" (See https://github.com/IBM-Cloud/terraform-provider-ibm/issues/2849) + default_pool = element([for pool in var.worker_pools : pool if pool.pool_name == "default"], 0) + other_pools = [for pool in var.worker_pools : pool if pool.pool_name != "default" && !var.ignore_worker_pool_size_changes] + other_autoscaling_pools = [for pool in var.worker_pools : pool if pool.pool_name != "default" && var.ignore_worker_pool_size_changes] + + default_ocp_version = "${data.ibm_container_cluster_versions.cluster_versions.default_openshift_version}_openshift" + ocp_version = var.ocp_version == null || var.ocp_version == "default" ? local.default_ocp_version : "${var.ocp_version}_openshift" + + cos_name = var.use_existing_cos == true || (var.use_existing_cos == false && var.cos_name != null) ? var.cos_name : "${var.cluster_name}_cos" + cos_location = "global" + cos_plan = "standard" + # if not enable_registry_storage then set cos to 'null', otherwise use existing or new CRN + cos_instance_crn = var.enable_registry_storage == true ? (var.use_existing_cos != false ? var.existing_cos_id : module.cos_instance[0].cos_instance_id) : null + + # Validation approach based on https://stackoverflow.com/a/66682419 + validate_condition = var.enable_registry_storage == true && var.use_existing_cos == true && var.existing_cos_id == null + validate_msg = "A value for 'existing_cos_id' variable must be passed when 'use_existing_cos = true'" + # tflint-ignore: terraform_unused_declarations + validate_check = regex("^${local.validate_msg}$", (!local.validate_condition ? local.validate_msg : "")) + + csi_driver_version = [ + for addon in data.ibm_container_addons.existing_addons.addons : + addon.version if addon.name == "vpc-block-csi-driver" + ] + addons_list = var.addons != null ? { for k, v in var.addons : k => v if v != null } : {} + addons = lookup(local.addons_list, "vpc-block-csi-driver", null) == null ? merge(local.addons_list, { vpc-block-csi-driver = local.csi_driver_version[0] }) : local.addons_list + + delete_timeout = "2h" + create_timeout = "3h" + update_timeout = "3h" + + cluster_id = var.ignore_worker_pool_size_changes ? ibm_container_vpc_cluster.autoscaling_cluster[0].id : ibm_container_vpc_cluster.cluster[0].id + + # security group attached to worker pool + # the terraform provider / iks api take a security group id hardcoded to "cluster", so this pseudo-value is injected into the array based on attach_default_cluster_security_group + # see https://cloud.ibm.com/docs/openshift?topic=openshift-vpc-security-group&interface=ui#vpc-sg-cluster + + # attach_ibm_managed_security_group is true and custom_security_group_ids is not set => default behavior, so set to null + # attach_ibm_managed_security_group is true and custom_security_group_ids is set => add "cluster" to the list of custom security group ids + + # attach_ibm_managed_security_group is false and custom_security_group_ids is not set => default behavior, so set to null + # attach_ibm_managed_security_group is false and custom_security_group_ids is set => only use the custom security group ids + cluster_security_groups = var.attach_ibm_managed_security_group == true ? (var.custom_security_group_ids == null ? null : concat(["cluster"], var.custom_security_group_ids)) : (var.custom_security_group_ids == null ? null : var.custom_security_group_ids) +} + +# Lookup the current default kube version +data "ibm_container_cluster_versions" "cluster_versions" { + resource_group_id = var.resource_group_id +} + +module "cos_instance" { + count = var.enable_registry_storage && !var.use_existing_cos ? 1 : 0 + + source = "terraform-ibm-modules/cos/ibm" + version = "7.1.5" + cos_instance_name = local.cos_name + resource_group_id = var.resource_group_id + cos_plan = local.cos_plan + cos_location = local.cos_location + kms_encryption_enabled = false + create_cos_bucket = false +} + +moved { + from = ibm_resource_instance.cos_instance[0] + to = module.cos_instance[0].ibm_resource_instance.cos_instance[0] +} + +resource "ibm_resource_tag" "cos_access_tag" { + count = var.enable_registry_storage && !var.use_existing_cos && length(var.access_tags) > 0 ? 1 : 0 + resource_id = module.cos_instance[0].cos_instance_id + tags = var.access_tags + tag_type = "access" +} + +############################################################################## +# Create a OCP Cluster +############################################################################## + +resource "ibm_container_vpc_cluster" "cluster" { + depends_on = [null_resource.reset_api_key] + count = var.ignore_worker_pool_size_changes ? 0 : 1 + name = var.cluster_name + vpc_id = var.vpc_id + tags = var.tags + kube_version = local.ocp_version + flavor = local.default_pool.machine_type +# entitlement = var.ocp_entitlement + cos_instance_crn = local.cos_instance_crn + worker_count = local.default_pool.workers_per_zone + resource_group_id = var.resource_group_id + wait_till = var.cluster_ready_when + force_delete_storage = var.force_delete_storage + disable_public_service_endpoint = var.disable_public_endpoint + worker_labels = local.default_pool.labels + crk = local.default_pool.boot_volume_encryption_kms_config == null ? null : local.default_pool.boot_volume_encryption_kms_config.crk + kms_instance_id = local.default_pool.boot_volume_encryption_kms_config == null ? null : local.default_pool.boot_volume_encryption_kms_config.kms_instance_id + kms_account_id = local.default_pool.boot_volume_encryption_kms_config == null ? null : local.default_pool.boot_volume_encryption_kms_config.kms_account_id + + security_groups = local.cluster_security_groups + + lifecycle { + ignore_changes = [kube_version] + } + + # default workers are mapped to the subnets that are "private" + dynamic "zones" { + for_each = local.default_pool.subnet_prefix != null ? var.vpc_subnets[local.default_pool.subnet_prefix] : local.default_pool.vpc_subnets + content { + subnet_id = zones.value.id + name = zones.value.zone + } + } + + # Apply taints to the default worker pools i.e private + + dynamic "taints" { + for_each = var.worker_pools_taints == null ? [] : concat(var.worker_pools_taints["all"], var.worker_pools_taints["default"]) + content { + effect = taints.value.effect + key = taints.value.key + value = taints.value.value + } + } + + dynamic "kms_config" { + for_each = var.kms_config != null ? [1] : [] + content { + crk_id = var.kms_config.crk_id + instance_id = var.kms_config.instance_id + private_endpoint = var.kms_config.private_endpoint == null ? true : var.kms_config.private_endpoint + account_id = var.kms_config.account_id + } + } + + timeouts { + # Extend create, update and delete timeout to static values. + delete = local.delete_timeout + create = local.create_timeout + update = local.update_timeout + } +} + +# copy of the cluster resource above which ignores changes to the worker pool for use in autoscaling scenarios +resource "ibm_container_vpc_cluster" "autoscaling_cluster" { + depends_on = [null_resource.reset_api_key] + count = var.ignore_worker_pool_size_changes ? 1 : 0 + name = var.cluster_name + vpc_id = var.vpc_id + tags = var.tags + kube_version = local.ocp_version + flavor = local.default_pool.machine_type + entitlement = var.ocp_entitlement + cos_instance_crn = local.cos_instance_crn + worker_count = local.default_pool.workers_per_zone + resource_group_id = var.resource_group_id + wait_till = var.cluster_ready_when + force_delete_storage = var.force_delete_storage + disable_public_service_endpoint = var.disable_public_endpoint + worker_labels = local.default_pool.labels + crk = local.default_pool.boot_volume_encryption_kms_config == null ? null : local.default_pool.boot_volume_encryption_kms_config.crk + kms_instance_id = local.default_pool.boot_volume_encryption_kms_config == null ? null : local.default_pool.boot_volume_encryption_kms_config.kms_instance_id + kms_account_id = local.default_pool.boot_volume_encryption_kms_config == null ? null : local.default_pool.boot_volume_encryption_kms_config.kms_account_id + + security_groups = local.cluster_security_groups + + lifecycle { + ignore_changes = [worker_count, kube_version] + } + + # default workers are mapped to the subnets that are "private" + dynamic "zones" { + for_each = local.default_pool.subnet_prefix != null ? var.vpc_subnets[local.default_pool.subnet_prefix] : local.default_pool.vpc_subnets + content { + subnet_id = zones.value.id + name = zones.value.zone + } + } + + # Apply taints to the default worker pools i.e private + + dynamic "taints" { + for_each = var.worker_pools_taints == null ? [] : concat(var.worker_pools_taints["all"], var.worker_pools_taints["default"]) + content { + effect = taints.value.effect + key = taints.value.key + value = taints.value.value + } + } + + dynamic "kms_config" { + for_each = var.kms_config != null ? [1] : [] + content { + crk_id = var.kms_config.crk_id + instance_id = var.kms_config.instance_id + private_endpoint = var.kms_config.private_endpoint + account_id = var.kms_config.account_id + } + } + + timeouts { + # Extend create, update and delete timeout to static values. + delete = local.delete_timeout + create = local.create_timeout + update = local.update_timeout + } +} + +############################################################################## +# Cluster Access Tag +############################################################################## + +resource "ibm_resource_tag" "cluster_access_tag" { + count = length(var.access_tags) == 0 ? 0 : 1 + resource_id = var.ignore_worker_pool_size_changes ? ibm_container_vpc_cluster.autoscaling_cluster[0].crn : ibm_container_vpc_cluster.cluster[0].crn + tags = var.access_tags + tag_type = "access" +} + +# Cluster provisioning will automatically create an IAM API key called "containers-kubernetes-key" if one does not exist +# for the given region and resource group. The API key is used to access several services, such as the IBM Cloud classic +# infrastructure portfolio, and is required to manage the cluster. Immediately after the IAM API key is created and +# added to the new resource group, it is replicated across IAM Cloudant instances. There is a small period of time from +# when the IAM API key is initially created and when it is fully replicated across Cloudant instances where the API key +# does not work because it is not fully replicated, so commands that require the API key may fail with 404. +# +# WORKAROUND: +# Run a script that checks if an IAM API key already exists for the given region and resource group, and if it does not, +# run the ibmcloud ks api-key reset command to create one. The script will then pause for some time to allow any IAM +# Cloudant replication to occur. By doing this, it means the cluster provisioning process will not attempt to create a +# new key, and simply use the key created by this script. So hence should not face 404s anymore. +# The IKS team are tracking internally https://github.ibm.com/alchemy-containers/armada-ironsides/issues/5023 + +resource "null_resource" "reset_api_key" { + provisioner "local-exec" { + command = "${path.module}/scripts/reset_iks_api_key.sh ${var.region} ${var.resource_group_id}" + interpreter = ["/bin/bash", "-c"] + environment = { + IBMCLOUD_API_KEY = var.ibmcloud_api_key + } + } +} + +############################################################################## +# Access cluster to kick off RBAC synchronisation +############################################################################## + +data "ibm_container_cluster_config" "cluster_config" { + count = var.verify_worker_network_readiness || lookup(local.addons_list, "cluster-autoscaler", null) != null ? 1 : 0 + cluster_name_id = local.cluster_id + config_dir = "${path.module}/kconfig" + admin = true # workaround for https://github.com/terraform-ibm-modules/terraform-ibm-base-ocp-vpc/issues/374 + resource_group_id = var.resource_group_id + endpoint_type = var.cluster_config_endpoint_type != "default" ? var.cluster_config_endpoint_type : null # null value represents default +} + +############################################################################## +# Worker Pools +############################################################################## + +resource "ibm_container_vpc_worker_pool" "pool" { + for_each = { for pool in local.other_pools : pool.pool_name => pool } + vpc_id = var.vpc_id + resource_group_id = var.resource_group_id + cluster = local.cluster_id + worker_pool_name = each.value.pool_name + flavor = each.value.machine_type + worker_count = each.value.workers_per_zone + labels = each.value.labels + crk = each.value.boot_volume_encryption_kms_config == null ? null : each.value.boot_volume_encryption_kms_config.crk + kms_instance_id = each.value.boot_volume_encryption_kms_config == null ? null : each.value.boot_volume_encryption_kms_config.kms_instance_id + kms_account_id = each.value.boot_volume_encryption_kms_config == null ? null : each.value.boot_volume_encryption_kms_config.kms_account_id + + security_groups = each.value.additional_security_group_ids + + dynamic "zones" { + for_each = each.value.subnet_prefix != null ? var.vpc_subnets[each.value.subnet_prefix] : each.value.vpc_subnets + content { + subnet_id = zones.value.id + name = zones.value.zone + } + } + + # Apply taints to worker pools i.e. other_pools + dynamic "taints" { + for_each = var.worker_pools_taints == null ? [] : concat(var.worker_pools_taints["all"], lookup(var.worker_pools_taints, each.value["pool_name"], [])) + content { + effect = taints.value.effect + key = taints.value.key + value = taints.value.value + } + } + + timeouts { + # Extend create and delete timeout to 2h + delete = "2h" + create = "2h" + } + +} + +# copy of the pool resource above which ignores changes to the worker pool for use in autoscaling scenarios +resource "ibm_container_vpc_worker_pool" "autoscaling_pool" { + for_each = { for pool in local.other_autoscaling_pools : pool.pool_name => pool } + vpc_id = var.vpc_id + resource_group_id = var.resource_group_id + cluster = local.cluster_id + worker_pool_name = each.value.pool_name + flavor = each.value.machine_type + worker_count = each.value.workers_per_zone + labels = each.value.labels + crk = each.value.boot_volume_encryption_kms_config == null ? null : each.value.boot_volume_encryption_kms_config.crk + kms_instance_id = each.value.boot_volume_encryption_kms_config == null ? null : each.value.boot_volume_encryption_kms_config.kms_instance_id + kms_account_id = each.value.boot_volume_encryption_kms_config == null ? null : each.value.boot_volume_encryption_kms_config.kms_account_id + + lifecycle { + ignore_changes = [worker_count] + } + + dynamic "zones" { + for_each = each.value.subnet_prefix != null ? var.vpc_subnets[each.value.subnet_prefix] : each.value.vpc_subnets + content { + subnet_id = zones.value.id + name = zones.value.zone + } + } + + # Apply taints to worker pools i.e. other_pools + + dynamic "taints" { + for_each = var.worker_pools_taints == null ? [] : concat(var.worker_pools_taints["all"], lookup(var.worker_pools_taints, each.value["pool_name"], [])) + content { + effect = taints.value.effect + key = taints.value.key + value = taints.value.value + } + } + +} + +############################################################################## +# Confirm network healthy by ensuring master can communicate with all workers. +# +# Please note: +# The network health check is applicable only if the cluster is accessible. +# +# To do this, we run a script to execute "kubectl logs" against each calico +# daemonset pod (as there will be one pod per node) and ensure it passes. +# +# Why? +# There can be a delay in getting the routes set up for the VPN that lets +# the master connect across accounts down to the workers, and that VPN +# connection is what is used by "kubectl logs". +# +# Why is there a delay? +# The network microservice has to trigger on new workers being created and +# push down an updated vpn config, and then the vpn server and client need +# to pick up this updated config. Depending on how busy the network +# microservice is handling requests, there might be a delay. + +############################################################################## + +resource "null_resource" "confirm_network_healthy" { + + count = var.verify_worker_network_readiness ? 1 : 0 + + # Worker pool creation can start before the 'ibm_container_vpc_cluster' completes since there is no explicit + # depends_on in 'ibm_container_vpc_worker_pool', just an implicit depends_on on the cluster ID. Cluster ID can exist before + # 'ibm_container_vpc_cluster' completes, so hence need to add explicit depends on against 'ibm_container_vpc_cluster' here. + depends_on = [ibm_container_vpc_cluster.cluster, ibm_container_vpc_cluster.autoscaling_cluster, ibm_container_vpc_worker_pool.pool, ibm_container_vpc_worker_pool.autoscaling_pool] + + provisioner "local-exec" { + command = "${path.module}/scripts/confirm_network_healthy.sh" + interpreter = ["/bin/bash", "-c"] + environment = { + KUBECONFIG = data.ibm_container_cluster_config.cluster_config[0].config_file_path + } + } +} + +# Lookup the current default csi-driver version +data "ibm_container_addons" "existing_addons" { + cluster = local.cluster_id +} + +resource "ibm_container_addons" "addons" { + + # Worker pool creation can start before the 'ibm_container_vpc_cluster' completes since there is no explicit + # depends_on in 'ibm_container_vpc_worker_pool', just an implicit depends_on on the cluster ID. Cluster ID can exist before + # 'ibm_container_vpc_cluster' completes, so hence need to add explicit depends on against 'ibm_container_vpc_cluster' here. + depends_on = [ibm_container_vpc_cluster.cluster, ibm_container_vpc_cluster.autoscaling_cluster, ibm_container_vpc_worker_pool.pool, ibm_container_vpc_worker_pool.autoscaling_pool, null_resource.confirm_network_healthy] + + cluster = local.cluster_id + resource_group_id = var.resource_group_id + + # setting to false means we do not want Terraform to manage addons that are managed elsewhere + manage_all_addons = var.manage_all_addons + + dynamic "addons" { + for_each = local.addons + content { + name = addons.key + version = addons.value + } + } + + timeouts { + create = "1h" + } +} + +locals { + worker_pool_config = [ + for worker in var.worker_pools : + { + name = worker.pool_name + minSize = worker.minSize + maxSize = worker.maxSize + enabled = worker.enableAutoscaling + } if worker.enableAutoscaling != null && worker.minSize != null && worker.maxSize != null + ] + +} + +resource "null_resource" "config_map_status" { + count = lookup(local.addons_list, "cluster-autoscaler", null) != null ? 1 : 0 + depends_on = [ibm_container_addons.addons] + + provisioner "local-exec" { + command = "${path.module}/scripts/get_config_map_status.sh" + interpreter = ["/bin/bash", "-c"] + environment = { + KUBECONFIG = data.ibm_container_cluster_config.cluster_config[0].config_file_path + } + } +} + +resource "kubernetes_config_map_v1_data" "set_autoscaling" { + count = lookup(local.addons_list, "cluster-autoscaler", null) != null ? 1 : 0 + depends_on = [null_resource.config_map_status] + + metadata { + name = "iks-ca-configmap" + namespace = "kube-system" + } + + data = { + "workerPoolsConfig.json" = jsonencode(local.worker_pool_config) + } + + force = true +} + + +############################################################################## +# Attach additional security groups to the load balancers managed by this +# cluster. Note that the module attaches security group to existing loadbalancer +# only. Re-run the module to attach security groups to new load balancers created +# after the initial run of this module. The module detects new load balancers. +# https://cloud.ibm.com/docs/openshift?topic=openshift-vpc-security-group&interface=ui#vpc-sg-vpe-alb +############################################################################## + +data "ibm_is_lbs" "all_lbs" { + depends_on = [ibm_container_vpc_cluster.cluster, ibm_container_vpc_worker_pool.pool, ibm_container_vpc_worker_pool.autoscaling_pool, null_resource.confirm_network_healthy] + count = length(var.additional_lb_security_group_ids) > 0 ? 1 : 0 +} + +locals { + lbs_associated_with_cluster = length(var.additional_lb_security_group_ids) > 0 ? [for lb in data.ibm_is_lbs.all_lbs[0].load_balancers : lb.id if strcontains(lb.name, local.cluster_id)] : [] +} + +module "attach_sg_to_lb" { + count = length(var.additional_lb_security_group_ids) + source = "terraform-ibm-modules/security-group/ibm" + version = "2.4.0" + existing_security_group_id = var.additional_lb_security_group_ids[count.index] + use_existing_security_group_id = true + target_ids = [for index in range(var.number_of_lbs) : local.lbs_associated_with_cluster[index]] # number_of_lbs is necessary to give a static number of elements to tf to accomplish the apply when the cluster does not initially exists +} + + +############################################################################## +# Attach additional security groups to the load balancers managed by this +# cluster. Note that the module attaches security group to existing loadbalancer +# only. Re-run the module to attach security groups to new load balancers created +# after the initial run of this module. The module detects new load balancers. +# https://cloud.ibm.com/docs/openshift?topic=openshift-vpc-security-group&interface=ui#vpc-sg-vpe-alb +############################################################################## + +data "ibm_is_virtual_endpoint_gateways" "all_vpes" { + depends_on = [ibm_container_vpc_cluster.cluster, ibm_container_vpc_worker_pool.pool, ibm_container_vpc_worker_pool.autoscaling_pool, null_resource.confirm_network_healthy] + count = var.additional_vpe_security_group_ids != {} ? 1 : 0 +} + +locals { + master_vpe_id = [for vpe in data.ibm_is_virtual_endpoint_gateways.all_vpes[0].virtual_endpoint_gateways : vpe.id if strcontains(vpe.name, "iks-${local.cluster_id}")][0] + api_vpe_id = length(var.additional_vpe_security_group_ids["api"]) > 0 ? [for vpe in data.ibm_is_virtual_endpoint_gateways.all_vpes[0].virtual_endpoint_gateways : vpe.id if strcontains(vpe.name, "iks-api-${var.vpc_id}")][0] : null + registry_vpe_id = length(var.additional_vpe_security_group_ids["registry"]) > 0 ? [for vpe in data.ibm_is_virtual_endpoint_gateways.all_vpes[0].virtual_endpoint_gateways : vpe.id if strcontains(vpe.name, "iks-registry-${var.vpc_id}")][0] : null +} + +module "attach_sg_to_master_vpe" { + count = length(var.additional_vpe_security_group_ids["master"]) + source = "terraform-ibm-modules/security-group/ibm" + version = "2.4.0" + existing_security_group_id = var.additional_vpe_security_group_ids["master"][count.index] + use_existing_security_group_id = true + target_ids = [local.master_vpe_id] +} + +module "attach_sg_to_api_vpe" { + count = length(var.additional_vpe_security_group_ids["api"]) + source = "terraform-ibm-modules/security-group/ibm" + version = "2.4.0" + existing_security_group_id = var.additional_vpe_security_group_ids["api"][count.index] + use_existing_security_group_id = true + target_ids = [local.api_vpe_id] +} + +module "attach_sg_to_registry_vpe" { + count = length(var.additional_vpe_security_group_ids["registry"]) + source = "terraform-ibm-modules/security-group/ibm" + version = "2.4.0" + existing_security_group_id = var.additional_vpe_security_group_ids["registry"][count.index] + use_existing_security_group_id = true + target_ids = [local.registry_vpe_id] +} diff --git a/ibm/terraform/modules/cluster/outputs.tf b/ibm/terraform/modules/cluster/outputs.tf new file mode 100644 index 0000000..a29a59a --- /dev/null +++ b/ibm/terraform/modules/cluster/outputs.tf @@ -0,0 +1,86 @@ +############################################################################## +# Outputs +############################################################################## + +output "cluster_id" { + description = "ID of cluster created" + value = var.ignore_worker_pool_size_changes ? ibm_container_vpc_cluster.autoscaling_cluster[0].id : ibm_container_vpc_cluster.cluster[0].id + depends_on = [null_resource.confirm_network_healthy] +} + +output "cluster_name" { + description = "Name of the created cluster" + value = var.ignore_worker_pool_size_changes ? ibm_container_vpc_cluster.autoscaling_cluster[0].name : ibm_container_vpc_cluster.cluster[0].name + depends_on = [null_resource.confirm_network_healthy] +} + +output "cluster_crn" { + description = "CRN for the created cluster" + value = var.ignore_worker_pool_size_changes ? ibm_container_vpc_cluster.autoscaling_cluster[0].crn : ibm_container_vpc_cluster.cluster[0].crn + depends_on = [null_resource.confirm_network_healthy] +} + +output "workerpools" { + description = "Worker pools created" + value = var.ignore_worker_pool_size_changes ? ibm_container_vpc_worker_pool.autoscaling_pool : ibm_container_vpc_worker_pool.pool +} + +output "ocp_version" { + description = "Openshift Version of the cluster" + value = var.ignore_worker_pool_size_changes ? ibm_container_vpc_cluster.autoscaling_cluster[0].kube_version : ibm_container_vpc_cluster.cluster[0].kube_version +} + +output "cos_crn" { + description = "CRN of the COS instance" + value = var.ignore_worker_pool_size_changes ? ibm_container_vpc_cluster.autoscaling_cluster[0].cos_instance_crn : ibm_container_vpc_cluster.cluster[0].cos_instance_crn +} + +output "vpc_id" { + description = "ID of the clusters VPC" + value = var.ignore_worker_pool_size_changes ? ibm_container_vpc_cluster.autoscaling_cluster[0].vpc_id : ibm_container_vpc_cluster.cluster[0].vpc_id +} + +output "region" { + description = "Region cluster is deployed in" + value = var.region +} + +output "resource_group_id" { + description = "Resource group ID the cluster is deployed in" + value = var.ignore_worker_pool_size_changes ? ibm_container_vpc_cluster.autoscaling_cluster[0].resource_group_id : ibm_container_vpc_cluster.cluster[0].resource_group_id +} + +output "ingress_hostname" { + description = "Ingress hostname" + value = var.ignore_worker_pool_size_changes ? ibm_container_vpc_cluster.autoscaling_cluster[0].ingress_hostname : ibm_container_vpc_cluster.cluster[0].ingress_hostname +} + +output "private_service_endpoint_url" { + description = "Private service endpoint URL" + value = var.ignore_worker_pool_size_changes ? ibm_container_vpc_cluster.autoscaling_cluster[0].private_service_endpoint_url : ibm_container_vpc_cluster.cluster[0].private_service_endpoint_url +} + +output "public_service_endpoint_url" { + description = "Public service endpoint URL" + value = var.ignore_worker_pool_size_changes ? ibm_container_vpc_cluster.autoscaling_cluster[0].public_service_endpoint_url : ibm_container_vpc_cluster.cluster[0].public_service_endpoint_url +} + +output "master_url" { + description = "The URL of the Kubernetes master." + value = var.ignore_worker_pool_size_changes ? ibm_container_vpc_cluster.autoscaling_cluster[0].master_url : ibm_container_vpc_cluster.cluster[0].master_url +} + +output "kms_config" { + description = "KMS configuration details" + value = var.kms_config +} + +output "operating_system" { + description = "The operating system of the workers in the default worker pool." + value = var.ignore_worker_pool_size_changes ? ibm_container_vpc_cluster.autoscaling_cluster[0].operating_system : ibm_container_vpc_cluster.cluster[0].operating_system +} + +output "master_status" { + description = "The status of the Kubernetes master." + value = var.ignore_worker_pool_size_changes ? ibm_container_vpc_cluster.autoscaling_cluster[0].master_status : ibm_container_vpc_cluster.cluster[0].master_status +} diff --git a/ibm/terraform/modules/cluster/scripts/confirm_network_healthy.sh b/ibm/terraform/modules/cluster/scripts/confirm_network_healthy.sh new file mode 100755 index 0000000..35c4bdf --- /dev/null +++ b/ibm/terraform/modules/cluster/scripts/confirm_network_healthy.sh @@ -0,0 +1,96 @@ +#!/bin/bash + +set -e + +function run_checks() { + + last_attempt=$1 + namespace=calico-system + + MAX_ATTEMPTS=10 + attempt=0 + PODS=() + while [ $attempt -lt $MAX_ATTEMPTS ]; do + # Get list of calico-node pods (There will be 1 pod per worker node) + if while IFS='' read -r line; do PODS+=("$line"); done < <(kubectl get pods -n "${namespace}" | grep calico-node | cut -f1 -d ' '); then + if [ ${#PODS[@]} -eq 0 ]; then + echo "No calico-node pods found. Retrying in 10s. (Attempt $((attempt+1)) / $MAX_ATTEMPTS)" + sleep 10 + ((attempt=attempt+1)) + else + # Pods found, break out of loop + break + fi + else + echo "Error getting calico-node pods. Retrying in 10s. (Attempt $((attempt+1)) / $MAX_ATTEMPTS)" + sleep 10 + ((attempt=attempt+1)) + fi + done + + if [ ${#PODS[@]} -eq 0 ]; then + echo "No calico-node pods found after $MAX_ATTEMPTS attempts. Exiting." + exit 1 + fi + + # Iterate through pods to check health + healthy=true + for pod in "${PODS[@]}"; do + command="kubectl logs ${pod} -n ${namespace} --tail=0" + # If it is the last attempt then print the output + if [ "${last_attempt}" == true ]; then + node=$(kubectl get pod "$pod" -n "${namespace}" -o=jsonpath='{.spec.nodeName}') + echo "Checking node: $node" + if ! ${command}; then + healthy=false + else + echo "OK" + fi + # Otherwise redirect output to /dev/null + else + if ! ${command} &> /dev/null; then + healthy=false + fi + fi + done + + if [ "$healthy" == "false" ]; then + return 1 + else + return 0 + fi + +} + +counter=0 +number_retries=40 +retry_wait_time=60 + +echo "Running script to ensure kube master can communicate with all worker nodes.." + +while [ ${counter} -le ${number_retries} ]; do + + # Determine if it is last attempt + last_attempt=false + if [ "${counter}" -eq ${number_retries} ]; then + last_attempt=true + fi + + ((counter=counter+1)) + if ! run_checks ${last_attempt}; then + if [ "${counter}" -gt ${number_retries} ]; then + echo "Maximum attempts reached, giving up." + echo + echo "Found kube master is unable to communicate with one or more of its workers." + echo "Please create a support issue with IBM Cloud and include the error message." + exit 1 + else + echo "Retrying in ${retry_wait_time}s. (Retry attempt ${counter} / ${number_retries})" + sleep ${retry_wait_time} + fi + else + break + fi +done + +echo "Success! Master can communicate with all worker nodes." diff --git a/ibm/terraform/modules/cluster/scripts/get_config_map_status.sh b/ibm/terraform/modules/cluster/scripts/get_config_map_status.sh new file mode 100755 index 0000000..4f9362b --- /dev/null +++ b/ibm/terraform/modules/cluster/scripts/get_config_map_status.sh @@ -0,0 +1,23 @@ +#!/bin/bash + +set -e + +CONFIGMAP_NAME="iks-ca-configmap" +NAMESPACE="kube-system" +COUNTER=0 +MAX_ATTEMPTS=40 + +while [[ $COUNTER -lt $MAX_ATTEMPTS ]] && ! kubectl get configmap $CONFIGMAP_NAME -n $NAMESPACE &>/dev/null; do + COUNTER=$((COUNTER + 1)) + echo "Attempt $COUNTER: ConfigMap '$CONFIGMAP_NAME' not found in namespace '$NAMESPACE', retrying..." + sleep 60 +done + +if [[ $COUNTER -eq $MAX_ATTEMPTS ]]; then + echo "ConfigMap '$CONFIGMAP_NAME' did not become available within $MAX_ATTEMPTS attempts." + # Output for debugging + kubectl get configmaps -n $NAMESPACE + exit 1 +else + echo "ConfigMap '$CONFIGMAP_NAME' is now available." >&2 +fi diff --git a/ibm/terraform/modules/cluster/scripts/reset_iks_api_key.sh b/ibm/terraform/modules/cluster/scripts/reset_iks_api_key.sh new file mode 100755 index 0000000..318a3cc --- /dev/null +++ b/ibm/terraform/modules/cluster/scripts/reset_iks_api_key.sh @@ -0,0 +1,69 @@ +#!/bin/bash + +set -euo pipefail + +REGION="$1" +RESOURCE_GROUP_ID="$2" +APIKEY_KEY_NAME="containers-kubernetes-key" + +# Expects the environment variable $IBMCLOUD_API_KEY to be set +if [[ -z "${IBMCLOUD_API_KEY}" ]]; then + echo "API key must be set with IBMCLOUD_API_KEY environment variable" >&2 + exit 1 +fi + +if [[ -z "${REGION}" ]]; then + echo "Region must be passed as first input script argument" >&2 + exit 1 +fi + +if [[ -z "${RESOURCE_GROUP_ID}" ]]; then + echo "Resource_group_id must be passed as second input script argument" >&2 + exit 1 +fi + +# Login to ibmcloud with cli +attempts=1 +until ibmcloud login -q -r "${REGION}" -g "${RESOURCE_GROUP_ID}" || [ $attempts -ge 3 ]; do + attempts=$((attempts+1)) + echo "Error logging in to IBM Cloud CLI..." >&2 + sleep 5 +done + +# run api-key reset command if apikey for given region + resource group does not already exist +reset=true +#key_descriptions=() +#while IFS='' read -r line; do key_descriptions+=("$line"); done < <(ibmcloud iam api-keys --all --output json | jq -r --arg name "${APIKEY_KEY_NAME}" '.[] | select(.name == $name) | .description') +#for i in "${key_descriptions[@]}"; do +# if [[ "$i" =~ ${REGION} ]] && [[ "$i" =~ ${RESOURCE_GROUP_ID} ]]; then +# echo "Found key named ${APIKEY_KEY_NAME} which covers clusters in ${REGION} and resource group ID ${RESOURCE_GROUP_ID}" +# reset=false +# break +# fi +#done +key_descriptions=() +ibmcloud iam api-keys --all --output json | jq -r --arg name "${APIKEY_KEY_NAME}" '.[] | select(.name == $name) | .description' | while IFS='' read -r line; do + echo "Line read: $line" # Debug output + key_descriptions+=("$line") +done + +# Check if the array is empty +if [ ${#key_descriptions[@]} -eq 0 ]; then + echo "No key descriptions found." +else + # Iterate over key descriptions + for i in "${key_descriptions[@]}"; do + if [[ "$i" =~ ${REGION} ]] && [[ "$i" =~ ${RESOURCE_GROUP_ID} ]]; then + echo "Found key named ${APIKEY_KEY_NAME} which covers clusters in ${REGION} and resource group ID ${RESOURCE_GROUP_ID}" + reset=false + break + fi + done +fi + +if [ "${reset}" == true ]; then + cmd="ibmcloud ks api-key reset --region ${REGION}" + yes | "${cmd}" || echo "Error executing command: ${cmd} && exit $?" + # sleep for 10 secs to allow the new key to be replicated across backend DB instances before attempting to create cluster + sleep 10 +fi diff --git a/ibm/terraform/modules/cluster/variables.tf b/ibm/terraform/modules/cluster/variables.tf new file mode 100644 index 0000000..dba7d0e --- /dev/null +++ b/ibm/terraform/modules/cluster/variables.tf @@ -0,0 +1,288 @@ +############################################################################## +# Input Variables +############################################################################## + +variable "ibmcloud_api_key" { + description = "APIkey that's associated with the account to use, set via environment variable TF_VAR_ibmcloud_api_key" + type = string + sensitive = true +} + +# Resource Group Variables +variable "resource_group_id" { + type = string + description = "The Id of an existing IBM Cloud resource group where the cluster will be grouped." +} + +variable "region" { + type = string + description = "The IBM Cloud region where the cluster will be provisioned." +} + +# Cluster Variables +variable "tags" { + type = list(string) + description = "Metadata labels describing this cluster deployment, i.e. test" + default = [] +} + +variable "cluster_name" { + type = string + description = "The name that will be assigned to the provisioned cluster" +} + +variable "vpc_subnets" { + type = map(list(object({ + id = string + zone = string + cidr_block = string + }))) + description = "Metadata that describes the VPC's subnets. Obtain this information from the VPC where this cluster will be created" +} + +variable "worker_pools" { + type = list(object({ + subnet_prefix = optional(string) + vpc_subnets = optional(list(object({ + id = string + zone = string + cidr_block = string + }))) + pool_name = string + machine_type = string + workers_per_zone = number + resource_group_id = optional(string) + labels = optional(map(string)) + minSize = optional(number) + maxSize = optional(number) + enableAutoscaling = optional(bool) + boot_volume_encryption_kms_config = optional(object({ + crk = string + kms_instance_id = string + kms_account_id = optional(string) + })) + additional_security_group_ids = optional(list(string)) + })) + description = "List of worker pools" + validation { + error_message = "Please provide value for minSize and maxSize while enableAutoscaling is set to true." + condition = length( + flatten( + [ + for worker in var.worker_pools : + worker if worker.enableAutoscaling == true && worker.minSize != null && worker.maxSize != null + ] + ) + ) == length( + flatten( + [ + for worker in var.worker_pools : + worker if worker.enableAutoscaling == true + ] + ) + ) + } + validation { + condition = length([for worker_pool in var.worker_pools : worker_pool if(worker_pool.subnet_prefix == null && worker_pool.vpc_subnets == null) || (worker_pool.subnet_prefix != null && worker_pool.vpc_subnets != null)]) == 0 + error_message = "Please provide exactly one of subnet_prefix or vpc_subnets. Passing neither or both is invalid." + } +} + +variable "worker_pools_taints" { + type = map(list(object({ key = string, value = string, effect = string }))) + description = "Optional, Map of lists containing node taints by node-pool name" + default = null +} + +variable "attach_ibm_managed_security_group" { + description = "Specify whether to attach the IBM-defined default security group (whose name is kube-) to all worker nodes. Only applicable if custom_security_group_ids is set." + type = bool + default = true +} + +variable "custom_security_group_ids" { + description = "Security groups to add to all worker nodes. This comes in addition to the IBM maintained security group if attach_ibm_managed_security_group is set to true. If this variable is set, the default VPC security group is NOT assigned to the worker nodes." + type = list(string) + default = null + validation { + condition = var.custom_security_group_ids == null ? true : length(var.custom_security_group_ids) <= 4 + error_message = "Please provide at most 4 additional security groups." + } +} + +variable "additional_lb_security_group_ids" { + description = "Additional security groups to add to the load balancers associated with the cluster. Ensure that the number_of_lbs is set to the number of LBs associated with the cluster. This comes in addition to the IBM maintained security group." + type = list(string) + default = [] + nullable = false + validation { + condition = var.additional_lb_security_group_ids == null ? true : length(var.additional_lb_security_group_ids) <= 4 + error_message = "Please provide at most 4 additional security groups." + } +} + +variable "number_of_lbs" { + description = "The number of LBs to associated the additional_lb_security_group_names security group with." + type = number + default = 1 + nullable = false + validation { + condition = var.number_of_lbs >= 1 + error_message = "Please set the number_of_lbs to a minumum of." + } +} + +variable "additional_vpe_security_group_ids" { + description = "Additional security groups to add to all existing load balancers. This comes in addition to the IBM maintained security group." + type = object({ + master = optional(list(string), []) + registry = optional(list(string), []) + api = optional(list(string), []) + }) + default = {} +} + +variable "ignore_worker_pool_size_changes" { + type = bool + description = "Enable if using worker autoscaling. Stops Terraform managing worker count" + default = false +} + +variable "ocp_version" { + type = string + description = "The version of the OpenShift cluster that should be provisioned (format 4.x). This is only used during initial cluster provisioning, but ignored for future updates. Supports passing the string 'default' (current IKS default recommended version). If no value is passed, it will default to 'default'." + default = null + + validation { + condition = anytrue([ + var.ocp_version == null, + var.ocp_version == "default", + var.ocp_version == "4.12", + var.ocp_version == "4.13", + var.ocp_version == "4.14", + ]) + error_message = "The specified ocp_version is not of the valid versions." + } +} + +variable "cluster_ready_when" { + type = string + description = "The cluster is ready when one of the following: MasterNodeReady (not recommended), OneWorkerNodeReady, Normal, IngressReady" + default = "IngressReady" + + validation { + condition = contains(["MasterNodeReady", "OneWorkerNodeReady", "Normal", "IngressReady"], var.cluster_ready_when) + error_message = "The input variable cluster_ready_when must one of: \"MasterNodeReady\", \"OneWorkerNodeReady\", \"Normal\" or \"IngressReady\"." + } +} +variable "disable_public_endpoint" { + type = bool + description = "Whether access to the public service endpoint is disabled when the cluster is created. Does not affect existing clusters. You can't disable a public endpoint on an existing cluster, so you can't convert a public cluster to a private cluster. To change a public endpoint to private, create another cluster with this input set to `true`." + default = false +} + +variable "ocp_entitlement" { + type = string + description = "Value that is applied to the entitlements for OCP cluster provisioning" + default = "cloud_pak" +} + +variable "force_delete_storage" { + type = bool + description = "Flag indicating whether or not to delete attached storage when destroying the cluster - Default: false" + default = false +} + +variable "cos_name" { + type = string + description = "Name of the COS instance to provision for OpenShift internal registry storage. New instance only provisioned if 'enable_registry_storage' is true and 'use_existing_cos' is false. Default: '_cos'" + default = null +} + +variable "use_existing_cos" { + type = bool + description = "Flag indicating whether or not to use an existing COS instance for OpenShift internal registry storage. Only applicable if 'enable_registry_storage' is true" + default = false +} + +variable "existing_cos_id" { + type = string + description = "The COS id of an already existing COS instance to use for OpenShift internal registry storage. Only required if 'enable_registry_storage' and 'use_existing_cos' are true" + default = null +} + +variable "enable_registry_storage" { + type = bool + description = "Set to `true` to enable IBM Cloud Object Storage for the Red Hat OpenShift internal image registry. Set to `false` only for new cluster deployments in an account that is allowlisted for this feature." + default = true +} + +variable "kms_config" { + type = object({ + crk_id = string + instance_id = string + private_endpoint = optional(bool, true) # defaults to true + account_id = optional(string) # To attach KMS instance from another account + }) + description = "Use to attach a KMS instance to the cluster. If account_id is not provided, defaults to the account in use." + default = null +} + +variable "access_tags" { + type = list(string) + description = "A list of access tags to apply to the resources created by the module, see https://cloud.ibm.com/docs/account?topic=account-access-tags-tutorial for more details" + default = [] + + validation { + condition = alltrue([ + for tag in var.access_tags : can(regex("[\\w\\-_\\.]+:[\\w\\-_\\.]+", tag)) && length(tag) <= 128 + ]) + error_message = "Tags must match the regular expression \"[\\w\\-_\\.]+:[\\w\\-_\\.]+\", see https://cloud.ibm.com/docs/account?topic=account-tag&interface=ui#limits for more details" + } +} + +# VPC Variables +variable "vpc_id" { + type = string + description = "Id of the VPC instance where this cluster will be provisioned" +} + +variable "verify_worker_network_readiness" { + type = bool + description = "By setting this to true, a script will run kubectl commands to verify that all worker nodes can communicate successfully with the master. If the runtime does not have access to the kube cluster to run kubectl commands, this should be set to false." + default = true +} + +variable "addons" { + type = object({ + debug-tool = optional(string) + image-key-synchronizer = optional(string) + openshift-data-foundation = optional(string) + vpc-file-csi-driver = optional(string) + static-route = optional(string) + cluster-autoscaler = optional(string) + vpc-block-csi-driver = optional(string) + }) + description = "Map of OCP cluster add-on versions to install (NOTE: The 'vpc-block-csi-driver' add-on is installed by default for VPC clusters, however you can explicitly specify it here if you wish to choose a later version than the default one). For full list of all supported add-ons and versions, see https://cloud.ibm.com/docs/containers?topic=containers-supported-cluster-addon-versions" + default = null +} + +variable "manage_all_addons" { + type = bool + default = false + nullable = false # null values are set to default value + description = "Instructs Terraform to manage all cluster addons, even if addons were installed outside of the module. If set to 'true' this module will destroy any addons that were installed by other sources." +} + +variable "cluster_config_endpoint_type" { + description = "Specify which type of endpoint to use for for cluster config access: 'default', 'private', 'vpe', 'link'. 'default' value will use the default endpoint of the cluster." + type = string + default = "default" + nullable = false # use default if null is passed in + validation { + error_message = "Invalid Endpoint Type! Valid values are 'default', 'private', 'vpe', or 'link'" + condition = contains(["default", "private", "vpe", "link"], var.cluster_config_endpoint_type) + } +} + +############################################################################## diff --git a/ibm/terraform/modules/cluster/version.tf b/ibm/terraform/modules/cluster/version.tf new file mode 100644 index 0000000..47aec50 --- /dev/null +++ b/ibm/terraform/modules/cluster/version.tf @@ -0,0 +1,18 @@ +terraform { + required_version = ">= 1.3.0, < 1.7.0" + required_providers { + # Use "greater than or equal to" range in modules + ibm = { + source = "ibm-cloud/ibm" + version = ">= 1.62.0, < 2.0.0" + } + null = { + source = "hashicorp/null" + version = ">= 3.2.1, < 4.0.0" + } + kubernetes = { + source = "hashicorp/kubernetes" + version = ">= 2.16.1, < 3.0.0" + } + } +} diff --git a/ibm/terraform/provider.tf b/ibm/terraform/provider.tf new file mode 100644 index 0000000..ec9cd3d --- /dev/null +++ b/ibm/terraform/provider.tf @@ -0,0 +1,3 @@ +provider "ibm" { + ibmcloud_api_key = var.ibmcloud_api_key +} \ No newline at end of file diff --git a/ibm/terraform/terraform.tfvars.example b/ibm/terraform/terraform.tfvars.example new file mode 100644 index 0000000..ca53a8a --- /dev/null +++ b/ibm/terraform/terraform.tfvars.example @@ -0,0 +1,44 @@ +ibmcloud_api_key = "" + +# Cluster name +cluster_name = "qdrant-hybrid-example" + +# Cluster name +prefix = "qdrant" + +# The location of the cluster +region = "us-south" + +# List of subnets for the vpc. For each item in each array, a subnet will be created. Items can be either CIDR blocks or total ipv4 addressess. Public gateways will be enabled only in zones where a gateway has been created +subnets = { + "zone-1": [ + { + "acl_name": "vpc-acl", + "cidr": "10.10.10.0/24", + "name": "subnet-a", + "no_addr_prefix": false, + "public_gateway": true + } + ], + "zone-2": [ + { + "acl_name": "vpc-acl", + "cidr": "10.20.10.0/24", + "name": "subnet-b", + "no_addr_prefix": false, + "public_gateway": true + } + ], + "zone-3": [ + { + "acl_name": "vpc-acl", + "cidr": "10.30.10.0/24", + "name": "subnet-c", + "no_addr_prefix": false, + "public_gateway": false + } + ] +} + +# Cluster name +vpc_name = "example" \ No newline at end of file diff --git a/ibm/terraform/variables.tf b/ibm/terraform/variables.tf new file mode 100644 index 0000000..31775fc --- /dev/null +++ b/ibm/terraform/variables.tf @@ -0,0 +1,154 @@ +variable "vpc_name" { + description = "Cluster name" + type = string + default = "example" +} + +variable "prefix" { + description = "Cluster name" + type = string + default = "qdrant" +} + + +variable "cluster_name" { + description = "Cluster name" + type = string + default = "qdrant-hybrid-example" +} + +variable "region" { + type = string + description = "The location of the cluster" + default = "us-south" +} + +variable "ibmcloud_api_key" { + type = string +} + + +variable "subnets" { + description = "List of subnets for the vpc. For each item in each array, a subnet will be created. Items can be either CIDR blocks or total ipv4 addressess. Public gateways will be enabled only in zones where a gateway has been created" + type = object({ + zone-1 = list(object({ + name = string + cidr = string + public_gateway = optional(bool) + acl_name = string + no_addr_prefix = optional(bool, false) # do not automatically add address prefix for subnet, overrides other conditions if set to true + })) + zone-2 = optional(list(object({ + name = string + cidr = string + public_gateway = optional(bool) + acl_name = string + no_addr_prefix = optional(bool, false) # do not automatically add address prefix for subnet, overrides other conditions if set to true + }))) + zone-3 = optional(list(object({ + name = string + cidr = string + public_gateway = optional(bool) + acl_name = string + no_addr_prefix = optional(bool, false) # do not automatically add address prefix for subnet, overrides other conditions if set to true + }))) + }) + + default = { + zone-1 = [ + { + name = "subnet-a" + cidr = "10.10.10.0/24" + public_gateway = true + acl_name = "vpc-acl" + no_addr_prefix = false + } + ], + zone-2 = [ + { + name = "subnet-b" + cidr = "10.20.10.0/24" + public_gateway = true + acl_name = "vpc-acl" + no_addr_prefix = false + } + ], + zone-3 = [ + { + name = "subnet-c" + cidr = "10.30.10.0/24" + public_gateway = false + acl_name = "vpc-acl" + no_addr_prefix = false + } + ] + } + + validation { + error_message = "Keys for `subnets` must be in the order `zone-1`, `zone-2`, `zone-3`. " + condition = ( + (length(var.subnets) == 1 && keys(var.subnets)[0] == "zone-1") || + (length(var.subnets) == 2 && keys(var.subnets)[0] == "zone-1" && keys(var.subnets)[1] == "zone-2") || + (length(var.subnets) == 3 && keys(var.subnets)[0] == "zone-1" && keys(var.subnets)[1] == "zone-2") && keys(var.subnets)[2] == "zone-3" + ) + } +} + +variable "worker_pools" { + type = list(object({ + subnet_prefix = optional(string) + vpc_subnets = optional(list(object({ + id = string + zone = string + cidr_block = string + }))) + pool_name = string + machine_type = string + workers_per_zone = number + resource_group_id = optional(string) + labels = optional(map(string)) + minSize = optional(number) + maxSize = optional(number) + enableAutoscaling = optional(bool) + boot_volume_encryption_kms_config = optional(object({ + crk = string + kms_instance_id = string + kms_account_id = optional(string) + })) + additional_security_group_ids = optional(list(string)) + })) + description = "List of worker pools" + validation { + error_message = "Please provide value for minSize and maxSize while enableAutoscaling is set to true." + condition = length( + flatten( + [ + for worker in var.worker_pools : + worker if worker.enableAutoscaling == true && worker.minSize != null && worker.maxSize != null + ] + ) + ) == length( + flatten( + [ + for worker in var.worker_pools : + worker if worker.enableAutoscaling == true + ] + ) + ) + } + default = [ + { + subnet_prefix = "zone-1" + pool_name = "default" # ibm_container_vpc_cluster automatically names default pool "default" (See https://github.com/IBM-Cloud/terraform-provider-ibm/issues/2849) + machine_type = "bx2.4x16" + workers_per_zone = 2 # minimum of 2 is allowed when using single zone + enableAutoscaling = true + minSize = 2 + maxSize = 10 + } + ] + validation { + condition = length([for worker_pool in var.worker_pools : worker_pool if(worker_pool.subnet_prefix == null && worker_pool.vpc_subnets == null) || (worker_pool.subnet_prefix != null && worker_pool.vpc_subnets != null)]) == 0 + error_message = "Please provide exactly one of subnet_prefix or vpc_subnets. Passing neither or both is invalid." + } +} \ No newline at end of file diff --git a/ibm/terraform/versions.tf b/ibm/terraform/versions.tf new file mode 100644 index 0000000..6793c3d --- /dev/null +++ b/ibm/terraform/versions.tf @@ -0,0 +1,10 @@ +terraform { + required_version = ">= 1.0" + + required_providers { + ibm = { + source = "IBM-Cloud/ibm" + version = ">= 1.12.0" + } + } +} diff --git a/ibm/terraform/vpc.tf b/ibm/terraform/vpc.tf new file mode 100644 index 0000000..ae2f16a --- /dev/null +++ b/ibm/terraform/vpc.tf @@ -0,0 +1,10 @@ +module vpc { + source = "terraform-ibm-modules/landing-zone-vpc/ibm" + version = "7.17.1" + resource_group_id = module.resource-group.resource_group_id + region = var.region + prefix = var.prefix + name = var.vpc_name + subnets = var.subnets + +} \ No newline at end of file