Merge pull request #2818 from oracle-devrel/oke-rm

oke-rm-1.3.4

Author: martatolosa

app-dev/devops-and-containers/oke/oke-rm/README.md

@@ -17,13 +17,13 @@ This stack is used to create the initial network infrastructure for OKE. When co
 * You can apply this stack even on an existing VCN, so that only the NSGs for OKE will be created
 * The default CNI is the VCN Native CNI, and it is the recommended one
 
-[![Deploy to Oracle Cloud](https://oci-resourcemanager-plugin.plugins.oci.oraclecloud.com/latest/deploy-to-oracle-cloud.svg)](https://cloud.oracle.com/resourcemanager/stacks/create?zipUrl=https://github.com/oracle-devrel/technology-engineering/releases/download/oke-rm-1.3.3/infra.zip)
+[![Deploy to Oracle Cloud](https://oci-resourcemanager-plugin.plugins.oci.oraclecloud.com/latest/deploy-to-oracle-cloud.svg)](https://cloud.oracle.com/resourcemanager/stacks/create?zipUrl=https://github.com/oracle-devrel/technology-engineering/releases/download/oke-rm-1.3.4/infra.zip)
 
 ## Step 2: Create the OKE control plane
 
 This stack is used to create the OKE control plane ONLY.
 
-[![Deploy to Oracle Cloud](https://oci-resourcemanager-plugin.plugins.oci.oraclecloud.com/latest/deploy-to-oracle-cloud.svg)](https://cloud.oracle.com/resourcemanager/stacks/create?zipUrl=https://github.com/oracle-devrel/technology-engineering/releases/download/oke-rm-1.3.3/oke.zip)
+[![Deploy to Oracle Cloud](https://oci-resourcemanager-plugin.plugins.oci.oraclecloud.com/latest/deploy-to-oracle-cloud.svg)](https://cloud.oracle.com/resourcemanager/stacks/create?zipUrl=https://github.com/oracle-devrel/technology-engineering/releases/download/oke-rm-1.3.4/oke.zip)
 
 Also note that if the network infrastructure is located in a different compartment than the OKE cluster AND you are planning to use the OCI_VCN_NATIVE CNI,
 you must add these policies:
@@ -90,4 +90,5 @@ If you are looking to provision an OKE cluster for RDMA and GPUs using this stac
 
 Provisioning an OKE cluster is just the first step, be sure to also check out these guides to learn how to configure it:
 * [OKE policies](../oke-policies/policies.md)
-* [GitOps with ArgoCD](../oke-gitops/README.md)
+* [Karpenter guide](oke-oci-karpenter-guide.md)
+* [OKE GitOps Solution](../oke-gitops/README.md)

app-dev/devops-and-containers/oke/oke-rm/infra/infra.zip

@@ -33,11 +33,13 @@ module "network" {
   worker_subnet_name      = var.worker_subnet_name
   allow_worker_nat_egress = var.allow_worker_nat_egress
   # POD SUBNET
-  create_pod_subnet    = var.create_pod_subnet
-  pod_subnet_cidr      = local.subnets.cidr.pod
-  pod_subnet_dns_label = local.subnets.dns.pod
-  pod_subnet_name      = var.pod_subnet_name
-  allow_pod_nat_egress = var.allow_pod_nat_egress
+  create_pod_subnet          = var.create_pod_subnet
+  pod_subnet_cidr            = local.subnets.cidr.pod
+  pod_subnet_dns_label       = local.subnets.dns.pod
+  pod_subnet_name            = var.pod_subnet_name
+  allow_pod_nat_egress       = var.allow_pod_nat_egress
+  create_additional_pod_cidr = var.create_additional_pod_cidr
+  additional_pod_cidr        = var.additional_pod_cidr
   # BASTION SUBNET
   create_bastion_subnet    = var.create_bastion_subnet
   bastion_subnet_cidr      = local.subnets.cidr.bastion

app-dev/devops-and-containers/oke/oke-rm/infra/main.tf

@@ -28,6 +28,16 @@ locals {
 
   create_msg_subnet = var.create_msg_subnet && var.create_vcn
 
+  karpenter_role_tag_key          = "karpenter-oci/role"
+  karpenter_worker_role_tag_value = "worker"
+  karpenter_pod_role_tag_value    = "pod"
+
+  karpenter_worker_role_freeform_tag = { (local.karpenter_role_tag_key) = local.karpenter_worker_role_tag_value }
+  karpenter_pod_role_freeform_tag    = { (local.karpenter_role_tag_key) = local.karpenter_pod_role_tag_value }
+
+  vcn_cidr_blocks     = var.create_additional_pod_cidr ? distinct(concat(var.vcn_cidr_blocks, var.additional_pod_cidr)) : var.vcn_cidr_blocks
+  pod_ipv4cidr_blocks = var.create_additional_pod_cidr && length(var.additional_pod_cidr) > 0 ? concat([var.pod_subnet_cidr], var.additional_pod_cidr) : [var.pod_subnet_cidr]
+
   tcp_protocol       = "6"
   icmp_protocol      = "1"
   udp_protocol       = "17"

app-dev/devops-and-containers/oke/oke-rm/infra/modules/network/local.tf

@@ -1,6 +1,7 @@
 locals {
   # Only used when NSG exists
-  peer_vcns_set = local.create_drg_attachment ? toset(var.peer_vcns) : []
+  peer_vcns_open_set = local.create_drg_attachment ? toset([for cidr in var.peer_vcns : cidr if cidr == "0.0.0.0/0"]) : []
+  peer_vcns_safe_set = local.create_drg_attachment ? toset([for cidr in var.peer_vcns : cidr if cidr != "0.0.0.0/0"]) : []
 }
 
 resource "oci_core_network_security_group" "peering" {
@@ -13,7 +14,7 @@ resource "oci_core_network_security_group" "peering" {
 }
 
 resource "oci_core_network_security_group_security_rule" "peering_egress" {
-  for_each                  = local.peer_vcns_set
+  for_each                  = local.peer_vcns_safe_set
   direction                 = "EGRESS"
   network_security_group_id = oci_core_network_security_group.peering.0.id
   protocol                  = "all"
@@ -24,12 +25,23 @@ resource "oci_core_network_security_group_security_rule" "peering_egress" {
 }
 
 resource "oci_core_network_security_group_security_rule" "peering_ingress" {
-  for_each                  = local.peer_vcns_set
+  for_each                  = local.peer_vcns_safe_set
   direction                 = "INGRESS"
   network_security_group_id = oci_core_network_security_group.peering.0.id
   protocol                  = "all"
   source_type               = "CIDR_BLOCK"
   source                    = each.value
   stateless                 = true
   description               = "Allow ingress traffic from peered VCN ${each.value}"
-}
+}
+
+resource "oci_core_network_security_group_security_rule" "peering_egress_open_stateful" {
+  for_each                  = local.peer_vcns_open_set
+  direction                 = "EGRESS"
+  network_security_group_id = oci_core_network_security_group.peering.0.id
+  protocol                  = "all"
+  destination_type          = "CIDR_BLOCK"
+  destination               = each.value
+  stateless                 = false
+  description               = "Allow stateful egress traffic for open peering CIDR ${each.value}"
+}

app-dev/devops-and-containers/oke/oke-rm/infra/modules/network/peering-nsg.tf

@@ -2,7 +2,7 @@ resource "oci_core_network_security_group" "pod_nsg" {
   compartment_id = var.network_compartment_id
   vcn_id         = local.vcn_id
   display_name   = "pod"
-  freeform_tags  = var.tag_value.freeformTags
+  freeform_tags  = merge(var.tag_value.freeformTags, local.karpenter_pod_role_freeform_tag)
   defined_tags   = var.tag_value.definedTags
   count          = local.is_npn ? 1 : 0
 }

app-dev/devops-and-containers/oke/oke-rm/infra/modules/network/pod-nsg.tf

@@ -1,3 +1,18 @@
+locals {
+  peer_vcns_default_set     = local.create_drg_attachment ? toset([for cidr in var.peer_vcns : cidr if cidr == "0.0.0.0/0"]) : toset([])
+  peer_vcns_non_default_set = local.create_drg_attachment ? toset([for cidr in var.peer_vcns : cidr if cidr != "0.0.0.0/0"]) : toset([])
+
+  cp_has_default_route     = (!var.cp_subnet_private) || local.cp_nat_mode
+  lb_int_has_default_route = false
+  worker_has_default_route = local.create_nat_gateway && var.allow_worker_nat_egress
+  pod_has_default_route    = local.create_nat_gateway && var.allow_pod_nat_egress
+
+  cp_drg_peer_vcns     = setunion(local.peer_vcns_non_default_set, local.cp_has_default_route ? toset([]) : local.peer_vcns_default_set)
+  lb_int_drg_peer_vcns = setunion(local.peer_vcns_non_default_set, local.lb_int_has_default_route ? toset([]) : local.peer_vcns_default_set)
+  worker_drg_peer_vcns = setunion(local.peer_vcns_non_default_set, local.worker_has_default_route ? toset([]) : local.peer_vcns_default_set)
+  pod_drg_peer_vcns    = setunion(local.peer_vcns_non_default_set, local.pod_has_default_route ? toset([]) : local.peer_vcns_default_set)
+}
+
 resource "oci_core_route_table" "bastion_route_table" {
   compartment_id = var.network_compartment_id
   vcn_id         = local.vcn_id
@@ -59,7 +74,7 @@ resource "oci_core_route_table" "cp_route_table" {
     }
   }
   dynamic "route_rules" {
-    for_each = local.create_drg_attachment ? var.peer_vcns : []
+    for_each = local.cp_drg_peer_vcns
     content {
       network_entity_id = local.drg_id
       destination_type  = "CIDR_BLOCK"
@@ -98,7 +113,7 @@ resource "oci_core_route_table" "lb_int_route_table" {
     description       = "Route for all internal OCI services in the region"
   }
   dynamic "route_rules" {
-    for_each = local.create_drg_attachment ? var.peer_vcns : []
+    for_each = local.lb_int_drg_peer_vcns
     content {
       network_entity_id = local.drg_id
       destination_type  = "CIDR_BLOCK"
@@ -131,7 +146,7 @@ resource "oci_core_route_table" "worker_route_table" {
     }
   }
   dynamic "route_rules" {
-    for_each = local.create_drg_attachment ? var.peer_vcns : []
+    for_each = local.worker_drg_peer_vcns
     content {
       network_entity_id = local.drg_id
       destination_type  = "CIDR_BLOCK"
@@ -164,7 +179,7 @@ resource "oci_core_route_table" "pod_route_table" {
     }
   }
   dynamic "route_rules" {
-    for_each = local.create_drg_attachment ? var.peer_vcns : []
+    for_each = local.pod_drg_peer_vcns
     content {
       network_entity_id = local.drg_id
       destination_type  = "CIDR_BLOCK"
@@ -218,4 +233,4 @@ resource "oci_core_route_table" "msg_route_table" {
     description       = "Route for all internal OCI services in the region"
   }
   count = local.create_msg_subnet ? 1 : 0
-}
+}

app-dev/devops-and-containers/oke/oke-rm/infra/modules/network/routing.tf

@@ -54,13 +54,13 @@ resource "oci_core_subnet" "worker_subnet" {
   route_table_id             = oci_core_route_table.worker_route_table.0.id
   security_list_ids          = [oci_core_security_list.worker_sl.0.id]
   dhcp_options_id            = oci_core_dhcp_options.worker_dhcp[0].id
-  freeform_tags              = var.tag_value.freeformTags
+  freeform_tags              = merge(var.tag_value.freeformTags, local.karpenter_worker_role_freeform_tag)
   defined_tags               = var.tag_value.definedTags
   count                      = local.create_worker_subnet ? 1 : 0
 }
 
 resource "oci_core_subnet" "pods_subnet" {
-  cidr_block                 = var.pod_subnet_cidr
+  ipv4cidr_blocks            = local.pod_ipv4cidr_blocks
   compartment_id             = var.network_compartment_id
   vcn_id                     = local.vcn_id
   dns_label                  = var.pod_subnet_dns_label
@@ -69,7 +69,7 @@ resource "oci_core_subnet" "pods_subnet" {
   route_table_id             = oci_core_route_table.pod_route_table.0.id
   security_list_ids          = [oci_core_security_list.pod_sl.0.id]
   dhcp_options_id            = oci_core_dhcp_options.pods_dhcp[0].id
-  freeform_tags              = var.tag_value.freeformTags
+  freeform_tags              = merge(var.tag_value.freeformTags, local.karpenter_pod_role_freeform_tag)
   defined_tags               = var.tag_value.definedTags
   count                      = local.create_pod_subnet ? 1 : 0
 }

app-dev/devops-and-containers/oke/oke-rm/infra/modules/network/subnet.tf

@@ -93,6 +93,14 @@ variable "allow_pod_nat_egress" {
   type = bool
 }
 
+variable "create_additional_pod_cidr" {
+  type = bool
+}
+
+variable "additional_pod_cidr" {
+  type = list(string)
+}
+
 # LB SUBNETS
 
 variable "create_external_lb_subnet" {

app-dev/devops-and-containers/oke/oke-rm/infra/modules/network/variable.tf

@@ -1,7 +1,7 @@
 resource "oci_core_vcn" "spoke_vcn" {
   compartment_id = var.network_compartment_id
   display_name   = var.vcn_name
-  cidr_blocks    = var.vcn_cidr_blocks
+  cidr_blocks    = local.vcn_cidr_blocks
   dns_label      = var.vcn_dns_label
   freeform_tags  = var.tag_value.freeformTags
   defined_tags   = var.tag_value.definedTags