Merge pull request #133 from oracle-devrel/develop

Automation Toolkit Release v2024.4.3

Author: xs2suruchi

Dockerfile

@@ -2,9 +2,22 @@ FROM oraclelinux:9-slim
 LABEL maintainer="Team at Oracle"
 LABEL description="OCI format to generate CD3 image"
 
-ARG USERNAME=cd3user
+
+########### Input Parameters for image creation ############
+# UID of user on underlying OS. eg 503 for Mac
 ARG USER_UID=1001
+# Whether to download Jenkins as part of image creation
+ARG USE_DEVOPS=YES
+#############################################################
+
+
+ARG USERNAME=cd3user
 ARG USER_GID=$USER_UID
+# Whether to download Provider as part of image creation
+ARG DOWNLOAD_PROVIDER=YES
+# TF Provider version
+ARG TF_OCI_PROVIDER=6.15.0
+ARG TF_NULL_PROVIDER=3.2.1
 
 RUN microdnf install -y sudo && \
     groupadd --gid $USER_GID $USERNAME && \
@@ -15,6 +28,8 @@ RUN microdnf install -y sudo && \
     chown -R $USERNAME:$USERNAME /cd3user/tenancies/ && \
     microdnf install -y vim && \
     microdnf install -y dnf && \
+    microdnf install -y wget && \
+    microdnf install -y unzip && \
     microdnf install -y graphviz && \
     echo 'alias vi="vim"' >> /etc/bashrc
 
@@ -29,12 +44,26 @@ RUN  sudo dnf install -y oraclelinux-release-el9 && \
 sudo chown -R $USERNAME:$USERNAME /cd3user/ && \
 sudo sed -i -e 's/\r$//' /cd3user/oci_tools/cd3_automation_toolkit/shell_script.sh && \
 bash /cd3user/oci_tools/cd3_automation_toolkit/shell_script.sh && \
-sudo chown -R cd3user:cd3user /cd3user/ && \
 sudo dnf clean all && \
 sudo rm -rf /var/cache/dnf && \
-sudo chmod -R 740 /cd3user/
+sudo chmod -R 740 /cd3user/ && \
+sudo chown -R cd3user:cd3user /cd3user/
 
 
+RUN if [ "$DOWNLOAD_PROVIDER" == "YES" ]; then \
+# oci provider
+sudo wget https://releases.hashicorp.com/terraform-provider-oci/${TF_OCI_PROVIDER}/terraform-provider-oci_${TF_OCI_PROVIDER}_linux_amd64.zip && \
+sudo mkdir -p /cd3user/.terraform.d/plugins/registry.terraform.io/oracle/oci/${TF_OCI_PROVIDER}/linux_amd64 && \
+sudo unzip terraform-provider-oci_${TF_OCI_PROVIDER}_linux_amd64.zip -d /cd3user/.terraform.d/plugins/registry.terraform.io/oracle/oci/${TF_OCI_PROVIDER}/linux_amd64 && \
+# null provider
+sudo wget https://releases.hashicorp.com/terraform-provider-null/${TF_NULL_PROVIDER}/terraform-provider-null_${TF_NULL_PROVIDER}_linux_amd64.zip && \
+sudo mkdir -p /cd3user/.terraform.d/plugins/registry.terraform.io/hashicorp/null/${TF_NULL_PROVIDER}/linux_amd64 && \
+sudo unzip terraform-provider-null_${TF_NULL_PROVIDER}_linux_amd64.zip -d /cd3user/.terraform.d/plugins/registry.terraform.io/hashicorp/null/${TF_NULL_PROVIDER}/linux_amd64 && \
+sudo cp -r /cd3user/.terraform.d/plugins/registry.terraform.io /cd3user/.terraform.d/plugins/registry.opentofu.org && \
+sudo chown -R cd3user:cd3user /cd3user/ && \
+sudo rm -rf terraform-provider-null_${TF_NULL_PROVIDER}_linux_amd64.zip terraform-provider-oci_${TF_OCI_PROVIDER}_linux_amd64.zip ;\
+fi
+
 ##################################### START INSTALLING JENKINS ###################################
 ARG JENKINS_VERSION=2.444
 ARG JENKINS_SHA=ab093a455fc35951c9b46361002e17cc3ed7c59b0943bbee3a57a363f3370d2e
@@ -45,15 +74,7 @@ ARG JENKINS_HOME=/cd3user/tenancies/jenkins_home
 ARG JENKINS_INSTALL=/usr/share/jenkins
 ARG REF=/usr/share/jenkins/ref
 
-RUN sudo microdnf install -y java-21-openjdk && \
-    sudo microdnf install -y java-21-openjdk-devel && \
-    sudo microdnf install git-2.39.3 -y && \
-    sudo mkdir -p ${REF}/init.groovy.d && \
-    sudo chown -R cd3user:cd3user ${JENKINS_INSTALL} && \
-    sudo curl -fsSL http://updates.jenkins-ci.org/download/war/${JENKINS_VERSION}/jenkins.war -o ${JENKINS_INSTALL}/jenkins.war && \
-    echo "${JENKINS_SHA}  ${JENKINS_INSTALL}/jenkins.war" | sha256sum -c - && \
-    sudo curl -fsSL ${PLUGIN_CLI_URL} -o ${JENKINS_INSTALL}/jenkins-plugin-manager.jar
-
+ENV USE_DEVOPS ${USE_DEVOPS}
 ENV JAVA_HOME /usr/lib/jvm/java-21-openjdk
 ENV JENKINS_HOME ${JENKINS_HOME}
 ENV JENKINS_INSTALL ${JENKINS_INSTALL}
@@ -64,11 +85,21 @@ ENV JENKINS_INCREMENTALS_REPO_MIRROR=https://repo.jenkins-ci.org/incrementals
 ENV JAVA_OPTS="-Djenkins.install.runSetupWizard=false"
 ENV COPY_REFERENCE_FILE_LOG ${JENKINS_HOME}/copy_reference_file.log
 ENV CASC_JENKINS_CONFIG ${JENKINS_HOME}/jcasc.yaml
-
 COPY --chown=cd3user:cd3user jenkins_install ${JENKINS_INSTALL}/
 COPY --chown=cd3user:cd3user jenkins_install/init/*.groovy ${REF}/init.groovy.d/
 COPY --chown=cd3user:cd3user jenkins_install/plugins.txt ${REF}/plugins.txt
 
-RUN sudo java -jar ${JENKINS_INSTALL}/jenkins-plugin-manager.jar --war ${JENKINS_INSTALL}/jenkins.war --verbose -f ${REF}/plugins.txt && \
+
+RUN if [ "$USE_DEVOPS" == "YES" ]; then \
+    sudo microdnf install -y java-21-openjdk && \
+    sudo microdnf install -y java-21-openjdk-devel && \
+    sudo microdnf install git-2.39.3 -y && \
+    sudo mkdir -p ${REF}/init.groovy.d && \
+    sudo chown -R cd3user:cd3user ${JENKINS_INSTALL} && \
+    sudo curl -fsSL http://updates.jenkins-ci.org/download/war/${JENKINS_VERSION}/jenkins.war -o ${JENKINS_INSTALL}/jenkins.war && \
+    echo "${JENKINS_SHA}  ${JENKINS_INSTALL}/jenkins.war" | sha256sum -c - && \
+    sudo curl -fsSL ${PLUGIN_CLI_URL} -o ${JENKINS_INSTALL}/jenkins-plugin-manager.jar && \
+    sudo java -jar ${JENKINS_INSTALL}/jenkins-plugin-manager.jar --war ${JENKINS_INSTALL}/jenkins.war --verbose -f ${REF}/plugins.txt && \
     sudo chown -R cd3user:cd3user ${JENKINS_INSTALL} && \
-    sudo chmod +x ${JENKINS_INSTALL}/jenkins.sh
+    sudo chmod +x ${JENKINS_INSTALL}/jenkins.sh ; \
+    fi

OCIWorkVMStack/modules/network/locals.tf

@@ -4,4 +4,5 @@ locals {
   create_inet_gw  = (var.vcn_strategy == "Create New VCN" && var.subnet_type == "Public") ? 1 : 0
   create_nat_gw   = (var.vcn_strategy == "Create New VCN" && var.subnet_type == "Private") ? 1 : 0
   create_nsg_rule = (var.vcn_strategy == "Create New VCN" && length(var.source_cidr) != 0) ? 1 : 0
+  route_rule_drg  = var.drg_attachment == true ? ( length(var.source_cidr) > 0 ? var.source_cidr : [] ) : []
 }

OCIWorkVMStack/modules/network/network.tf

@@ -49,12 +49,23 @@ resource "oci_core_route_table" "rt" {
   vcn_id         = local.vcn_id
   display_name   = "${var.subnet_name}-rt"
 
+ # Route rules to NGW or IGW
   route_rules {
     destination       = "0.0.0.0/0"
     destination_type  = "CIDR_BLOCK"
     network_entity_id = local.create_inet_gw == 1 ? oci_core_internet_gateway.internet_gw[0].id : oci_core_nat_gateway.nat_gw[0].id
   }
 
+ # Route rules to DRG
+ dynamic route_rules {
+  for_each = local.route_rule_drg
+  content {
+     destination       = route_rules.value
+     destination_type  = "CIDR_BLOCK"
+     network_entity_id = var.existing_drg_id
+   }
+ }
+
 }
 resource "oci_core_security_list" "security_list" {
   count          = local.create_vcn

OCIWorkVMStack/schema.yaml

@@ -406,7 +406,7 @@ variables:
          - drg_attachment
          - true
     type: string
-    pattern: '^ocid1\.([a-z0-9_-]{1,32})\.([a-z0-9_-]{1,15})\.([a-z0-9]{0,24})\.([a-z0-9]{60})$'
+    pattern: '^ocid1\.([a-z0-9_-]{1,32})\.([a-z0-9_-]{1,15})\.([a-z0-9_-]{0,24})\.([a-z0-9]{60})$'
     required: true
     title: Enter Existing DRG OCID
     description: Enter existing DRG OCID

README.md

@@ -7,7 +7,7 @@
 
 <br>
 
-  [What's New](https://github.com/oracle-devrel/cd3-automation-toolkit/releases/tag/v2024.4.2) &nbsp;•&nbsp;[Excel Templates](https://oracle-devrel.github.io/cd3-automation-toolkit/latest/excel-templates/) &nbsp;•&nbsp;[CD3 Docs](https://oracle-devrel.github.io/cd3-automation-toolkit/)&nbsp;•&nbsp; [Watch & Learn](https://www.youtube.com/playlist?list=PLPIzp-E1msrbJ3WawXVhzimQnLw5iafcp) &nbsp;•&nbsp;[Blogs & Tutorials](https://oracle-devrel.github.io/cd3-automation-toolkit/latest/tutorials/) &nbsp;•&nbsp;[Livelabs](https://apexapps.oracle.com/pls/apex/f?p=133:180:112501098061930::::wid:3724) &nbsp;•&nbsp;[Slack Channel](https://oracle-devrel.github.io/cd3-automation-toolkit/latest/queries) 
+  [What's New](https://github.com/oracle-devrel/cd3-automation-toolkit/releases/tag/v2024.4.3) &nbsp;•&nbsp;[Excel Templates](https://oracle-devrel.github.io/cd3-automation-toolkit/latest/excel-templates/) &nbsp;•&nbsp;[CD3 Docs](https://oracle-devrel.github.io/cd3-automation-toolkit/)&nbsp;•&nbsp; [Watch & Learn](https://www.youtube.com/playlist?list=PLPIzp-E1msrbJ3WawXVhzimQnLw5iafcp) &nbsp;•&nbsp;[Blogs & Tutorials](https://oracle-devrel.github.io/cd3-automation-toolkit/latest/tutorials/) &nbsp;•&nbsp;[Livelabs](https://apexapps.oracle.com/pls/apex/f?p=133:180:112501098061930::::wid:3724) &nbsp;•&nbsp;[Slack Channel](https://oracle-devrel.github.io/cd3-automation-toolkit/latest/queries) 
 
 <br>
 
@@ -81,7 +81,7 @@ Additionally, the toolkit also supports seamless resource management using OCI D
 
 📝 Creating Terraform Code for each module/resource can be cumbersome and requires Terraform expertise.
 
-🔁 Manually created infrastrucutre is hard to rebuild for different environments or regions.
+🔁 Manually created infrastructure is hard to rebuild for different environments or regions.
 
 <br>
 

cd3_automation_toolkit/Compute/create_terraform_instances.py

@@ -226,12 +226,12 @@ def create_terraform_instances(inputfile, outdir, service_dir, prefix, ct):
 
             if columnname == "Source Details":
                 if columnvalue.strip() != '' and columnvalue.strip().lower() != 'nan':
-                    if "ocid1.image.oc1" in columnvalue.strip():
+                    if "ocid1.image.oc" in columnvalue.strip():
                         ocid = columnvalue.strip()
                         type = "image"
                         source_details.append(type)
                         source_details.append(ocid)
-                    elif "ocid1.bootvolume.oc1" in columnvalue.strip():
+                    elif "ocid1.bootvolume.oc" in columnvalue.strip():
                         ocid = columnvalue.strip()
                         type = "bootVolume"
                         source_details.append(type)

cd3_automation_toolkit/DeveloperServices/OKE/export_oke_nonGreenField.py

@@ -531,7 +531,7 @@ def export_oke(inputfile, outdir,service_dir, config, signer, ct, export_compart
 
 
                         #Virtual NodePool
-                        if ("ocid1.virtualnodepool.oc1" in nodepool_info.id):
+                        if ("ocid1.virtualnodepool.oc" in nodepool_info.id):
                             nodepool_display_name = nodepool_info.display_name
                             np_tf_name = commonTools.check_tf_variable(nodepool_display_name)
                             tf_resource = f'module.virtual-nodepools[\\"{cluster_tf_name}_{np_tf_name}\\"].oci_containerengine_virtual_node_pool.virtual_nodepool'
@@ -540,7 +540,7 @@ def export_oke(inputfile, outdir,service_dir, config, signer, ct, export_compart
                             nodepool_type = "virtual"
 
                         # Managed NodePool
-                        if ("ocid1.nodepool.oc1" in nodepool_info.id):
+                        if ("ocid1.nodepool.oc" in nodepool_info.id):
                             nodepool_display_name = nodepool_info.name
                             np_tf_name = commonTools.check_tf_variable(nodepool_display_name)
                             nodepool_type = "managed"

cd3_automation_toolkit/DeveloperServices/ResourceManager/create_resource_manager_stack.py

@@ -119,12 +119,14 @@ def create_resource_manager(outdir,var_file, outdir_struct,prefix,auth_mechanism
 
         rm_dir = region_dir + '/RM/'
 
-        # 1. Copy all the TF files for specified regions to RM directory
+        # 1. Copy all the TF files for specified regions to RM directory. Also copy modules directory
         try:
             shutil.copytree(region_dir, rm_dir, ignore=shutil.ignore_patterns('*.terraform.lock.hcl','*.terraform','provider.tf','*.zip*','*.safe*','*.log*','*cis_report','*.csv*','*cd3validator', 'variables_*.tf*'))
+            shutil.copytree(outdir+"/modules", rm_dir)
         except FileExistsError as fe:
             shutil.rmtree(rm_dir)
             shutil.copytree(region_dir, rm_dir, ignore=shutil.ignore_patterns('*.terraform.lock.hcl','*.terraform','provider.tf','*.zip*','*.safe*','*.log*','*cis_report','*.csv*','*cd3validator', 'variables_*.tf*'))
+            shutil.copytree(outdir+"/modules", rm_dir+"/modules")
 
         #2. Change the provider.tf and variables_<region>.tf to include just the region variable in all stacks for specified regions
         tfStr[region]=''
@@ -286,6 +288,15 @@ def create_resource_manager(outdir,var_file, outdir_struct,prefix,auth_mechanism
             zip_name = rm_name + ".zip"
             # Fix for make_archive huge zip file issue - Ulag
             file_paths = []
+            for file in os.listdir(rm_dir):
+                if ".tf" in file and "variables" not in file and "provider" not in file and 'backend' not in file:
+                    with open(file, 'r') as tf_file:
+                        module_data = tf_file.read().rstrip()
+                        module_data = module_data.replace("\"../modules", "\"./modules")
+                    f = open(file, "w+")
+                    f.write(module_data)
+                    f.close()
+
             for root, directories, files in os.walk(rm_dir):
                 for filename in files:
                     rel_dir = os.path.relpath(root, rm_dir)
@@ -368,7 +379,7 @@ def create_resource_manager(outdir,var_file, outdir_struct,prefix,auth_mechanism
                     if os.path.exists(service_dir+"/"+ svc + ".tf"):
                         with open(service_dir+"/"+ svc + ".tf", 'r') as tf_file:
                             module_data = tf_file.read().rstrip()
-                            module_data = module_data.replace("\"../modules", "\"./modules")
+                            module_data = module_data.replace("\"../../modules", "\"./modules")
                         if svc == 'rpc':
                             f = open(service_dir+"/"+ svc + "-temp.tf", "w+")
                         else:

cd3_automation_toolkit/Governance/Tagging/create_terraform_tags.py

@@ -231,7 +231,7 @@ def create_terraform_tags(inputfile, outdir, service_dir, prefix, ct):
                                     else:
                                         if str(df.loc[i, 'Validator']).strip() == '' or  str(df.loc[i, 'Validator']).strip().lower() == 'nan':
                                             is_required_updated = 'true' #Uncomment this if needed
-                                            default_value = '-'
+                                            default_value = '[CANNOT_BE_EMPTY]'
                                             columnvalue = key_tf_name+"="+default_compartment+"="+default_value+"="+is_required_updated #Uncomment this if needed
                                             if columnvalue not in default_tags:
                                                 default_tags.append(columnvalue)

cd3_automation_toolkit/Identity/Users/export_users_nonGreenField.py

@@ -112,9 +112,14 @@ def export_users(inputfile, outdir, service_dir, config, signer, ct,export_domai
             domain_name = domain_key.split("@")[1]
             domain_client = oci.identity_domains.IdentityDomainsClient(config=config, signer=signer,
                                                                        service_endpoint=idcs_endpoint)
-            users = domain_client.list_users(limit=100000) # change this to pagination once api supports
+            list_users_response = domain_client.list_users() # change this to pagination once api supports
+            users = list_users_response.data.resources
+            while list_users_response.has_next_page:
+                list_users_response = domain_client.list_users(page=list_users_response.next_page)
+                users.extend(list_users_response.data.resources)
+
             index = 0
-            for user in users.data.resources:
+            for user in users:
                 defined_tags_info = user.urn_ietf_params_scim_schemas_oracle_idcs_extension_oci_tags
                 user_defined_tags = []