diff --git a/Dockerfile b/Dockerfile
index 7698a8f58..ab65c7a7f 100755
--- a/Dockerfile
+++ b/Dockerfile
@@ -45,6 +45,7 @@ RUN sudo dnf install -y oraclelinux-release-el9 && \
sudo chown -R $USERNAME:$USERNAME /cd3user/ && \
sudo sed -i -e 's/\r$//' /cd3user/oci_tools/cd3_automation_toolkit/shell_script.sh && \
bash /cd3user/oci_tools/cd3_automation_toolkit/shell_script.sh && \
+sudo microdnf install git-2.39.3 -y && \
sudo dnf clean all && \
sudo rm -rf /var/cache/dnf && \
sudo chmod -R 740 /cd3user/ && \
@@ -67,8 +68,8 @@ sudo rm -rf terraform-provider-null_${TF_NULL_PROVIDER}_linux_amd64.zip terrafor
fi
##################################### START INSTALLING JENKINS ###################################
-ARG JENKINS_VERSION=2.504.1
-ARG JENKINS_SHA=81026db18b0c4aad6b62cf408e4c42e5797661b41c517b37df606238e89b9df1
+ARG JENKINS_VERSION=2.504.3
+ARG JENKINS_SHA=ea8883431b8b5ef6b68fe0e5817c93dc0a11def380054e7de3136486796efeb0
ARG JENKINS_PLUGIN_MANAGER_VERSION=2.12.13
ARG PLUGIN_CLI_URL=https://github.com/jenkinsci/plugin-installation-manager-tool/releases/download/${JENKINS_PLUGIN_MANAGER_VERSION}/jenkins-plugin-manager-${JENKINS_PLUGIN_MANAGER_VERSION}.jar
@@ -95,7 +96,6 @@ COPY --chown=cd3user:cd3user jenkins_install/plugins.txt ${REF}/plugins.txt
RUN if [ "$USE_DEVOPS" == "YES" ]; then \
sudo microdnf install -y java-21-openjdk && \
sudo microdnf install -y java-21-openjdk-devel && \
- sudo microdnf install git-2.39.3 -y && \
sudo mkdir -p ${REF}/init.groovy.d && \
sudo chown -R cd3user:cd3user ${JENKINS_INSTALL} && \
sudo curl -fsSL http://updates.jenkins-ci.org/download/war/${JENKINS_VERSION}/jenkins.war -o ${JENKINS_INSTALL}/jenkins.war && \
diff --git a/OCIWorkVMStack/oci_images.tf b/OCIWorkVMStack/oci_images.tf
index 221b7d7ca..ba782a10c 100644
--- a/OCIWorkVMStack/oci_images.tf
+++ b/OCIWorkVMStack/oci_images.tf
@@ -1,5 +1,5 @@
variable "marketplace_source_images" {
- type = map(object({
+ /*type = map(object({
ocid = string
is_pricing_associated = bool
compatible_shapes = set(string)
@@ -10,5 +10,6 @@ variable "marketplace_source_images" {
is_pricing_associated = false
compatible_shapes = []
}
- }
+ }*/
+ default = ""
}
diff --git a/README.md b/README.md
index 20ef95eab..f4798bcc8 100644
--- a/README.md
+++ b/README.md
@@ -7,7 +7,7 @@
- [What's New](https://github.com/oracle-devrel/cd3-automation-toolkit/releases/tag/v2026.1.2) • [Excel Templates](https://oracle-devrel.github.io/cd3-automation-toolkit/latest/excel-templates/) • [CD3 Docs](https://oracle-devrel.github.io/cd3-automation-toolkit/latest/cd3-overview/) • [Watch & Learn](https://www.youtube.com/playlist?list=PLPIzp-E1msrbJ3WawXVhzimQnLw5iafcp) • [Blogs & Tutorials](https://oracle-devrel.github.io/cd3-automation-toolkit/latest/tutorials/) • [Livelabs](https://apexapps.oracle.com/pls/apex/f?p=133:180:112501098061930::::wid:3724) • [Slack Channel](https://oracle-devrel.github.io/cd3-automation-toolkit/latest/queries)
+ [What's New](https://github.com/oracle-devrel/cd3-automation-toolkit/releases/tag/v2026.2.0) • [Excel Templates](https://oracle-devrel.github.io/cd3-automation-toolkit/latest/excel-templates/) • [CD3 Docs](https://oracle-devrel.github.io/cd3-automation-toolkit/latest/cd3-overview/) • [Watch & Learn](https://www.youtube.com/playlist?list=PLPIzp-E1msrbJ3WawXVhzimQnLw5iafcp) • [Blogs & Tutorials](https://oracle-devrel.github.io/cd3-automation-toolkit/latest/tutorials/) • [Livelabs](https://apexapps.oracle.com/pls/f?p=133:180:112501098061930::::wid:3724) • [Slack Channel](https://oracle-devrel.github.io/cd3-automation-toolkit/latest/queries)
@@ -74,7 +74,7 @@ Evaluate Terraform plans to identify potential security risks, misconfigurations
-📌 The Toolkit also extends its support to ADB@Azure
+📌 The Toolkit now support to ADB@Azure, Exa@Azure(create only), ADB@GCP, Exa@GCP(create only).
diff --git a/cd3_automation_toolkit/Release-Notes b/cd3_automation_toolkit/Release-Notes
index 7c7fd8a4e..bfb359e5a 100644
--- a/cd3_automation_toolkit/Release-Notes
+++ b/cd3_automation_toolkit/Release-Notes
@@ -1,3 +1,15 @@
+-------------------------------------
+CD3 Automation Toolkit Tag v2026.2.0
+Jul 31st, 2026
+-------------------------------------
+1. Added multi-cloud Database Service support for Azure and GCP, including creation of ADB and Exadata resources, ADB export, and offline tfvars generation. Exadata export is not included in this release.
+2. Added support for multiple secondary VNICs for OCI instances.
+3. Added support for preemptible node pool configuration for OKE.
+4. Updated the CIS compliance report to version 3.2.1.
+5. Added identity output separation and fixed RPC deduplication and user export behavior.
+6. Included bug fixes for ADB, NSG rules, buckets, the OCI Resource Manager stack, and other minor issues.
+7. Upgraded Jenkins core from 2.504.1 to 2.504.3 for compatibility with the current Jenkins plugin update-center catalog.
+
-------------------------------------
CD3 Automation Toolkit Tag v2026.1.2
Jul 10th, 2026
@@ -9,7 +21,7 @@ CD3 Automation Toolkit Tag v2026.1.1
May 8th, 2026
-------------------------------------
1. Support for export of resources from all child compartments of a parent without specifying them separately in command line.
-2. Deprecate python script createTenancyConfig.py. Instead use connectCloud.py
+2. Deprecate python scripts createTenancyConfig.py and setUpOCI.py. Instead use connectCloud.py and setUpCloud.py
3. Fixed ordering issue for views for export of DNS resolvers.
4. Minor internal bug fixes.
diff --git a/cd3_automation_toolkit/azurecloud/python/database/export_adb_azure.py b/cd3_automation_toolkit/azurecloud/python/database/export_adb_azure.py
index edc265289..bf7c7f2fe 100644
--- a/cd3_automation_toolkit/azurecloud/python/database/export_adb_azure.py
+++ b/cd3_automation_toolkit/azurecloud/python/database/export_adb_azure.py
@@ -42,18 +42,18 @@ def normalize_enum_token(value):
return s.split(".", 1)[1] if "." in s else s
-def _format_rg_vnet_subnet_from_id(net_id: str) -> str:
- """Return 'resourceGroup@vnet::subnet' from a subnet ARM ID."""
+def _parse_subnet_arm_id(net_id: str):
+ """Return the resource group, virtual network, and subnet from a subnet ARM ID."""
if not net_id:
- return ""
+ return "", "", ""
parts = net_id.strip("/").split("/")
try:
rg = parts[parts.index("resourceGroups") + 1]
vnet = parts[parts.index("virtualNetworks") + 1]
subnet = parts[parts.index("subnets") + 1]
- return f"{rg}@{vnet}::{subnet}"
+ return rg, vnet, subnet
except (ValueError, IndexError):
- return ""
+ return "", "", ""
def _get_rg_from_id(resource_id: str) -> str:
@@ -106,21 +106,13 @@ def print_adbs_azure(adb, values_for_column: Dict[str, List], state: Dict, tf_or
# Tags (dict) — kept as metadata only if the sheet has 'Common Tags' column(not used yet)
common_tags = azrCommonTools._flatten_tags(getattr(adb, "tags", None))
- # Prefer subnetId to derive rg@vnet::subnet; private endpoint IDs don't contain vnet/subnet names
+ # Prefer subnetId because private endpoint IDs don't contain vnet/subnet names
subnet_id = pick_first_not_none(
getattr(props, "subnet_id", None),
getattr(props, "subnetId", None),
)
- formatted_net = _format_rg_vnet_subnet_from_id(subnet_id)
-
- # Optional: keep raw IDs if needed elsewhere
- network_details_raw = pick_first_not_none(
- getattr(props, "subnet_id", None),
- getattr(props, "subnetId", None),
- getattr(props, "private_endpoint_id", None),
- getattr(props, "privateEndpointId", None),
- "",
- )
+ network_rg, vnet_name, subnet_name = _parse_subnet_arm_id(subnet_id)
+ odb_network_details = f"{network_rg}::{vnet_name}" if network_rg and vnet_name else ""
# Whitelisted IPs (array of strings)
ips_list = pick_first_not_none(
@@ -204,9 +196,11 @@ def print_adbs_azure(adb, values_for_column: Dict[str, List], state: Dict, tf_or
values_for_column[col_header].append(adb_location)
elif col_header == "ADB Display Name":
values_for_column[col_header].append(adb_name)
- elif col_header == "Network Details":
- # Write the formatted rg@vnet::subnet if available; else leave blank or fall back to raw
- values_for_column[col_header].append(formatted_net or "")
+ # Keep network and subnet separate because the ADB-Azure generator reads them from two workbook columns.
+ elif col_header == "ODB Network Details":
+ values_for_column[col_header].append(odb_network_details)
+ elif col_header == "ODB Network Subnet Details":
+ values_for_column[col_header].append(subnet_name)
elif col_header == "Whitelisted IP Addresses":
values_for_column[col_header].append(whitelisted_ips)
elif col_header == "DB Name":
@@ -341,4 +335,4 @@ def export_adb_azure(inputfile: str, outdir: str,credentials,
if importCommands.get("global"):
importCommands["global"] += f'\n{tf_or_tofu} plan\n'
with open(script_file, 'a', encoding='utf-8') as importCommandsfile:
- importCommandsfile.write(init_commands + importCommands["global"])
\ No newline at end of file
+ importCommandsfile.write(init_commands + importCommands["global"])
diff --git a/cd3_automation_toolkit/common/opa/Network/oci_enforce_container_config.rego b/cd3_automation_toolkit/common/opa/Network/oci_enforce_container_config.rego
index 97925f517..57d701f0f 100755
--- a/cd3_automation_toolkit/common/opa/Network/oci_enforce_container_config.rego
+++ b/cd3_automation_toolkit/common/opa/Network/oci_enforce_container_config.rego
@@ -15,7 +15,6 @@ enforce_container_engine_config {
container.is_private_cluster
container.is_encryption_enabled
- container.is_pod_security_policy_enabled
}
#To enforce secure configuration for container instances
diff --git a/cd3_automation_toolkit/common/python/commonTools.py b/cd3_automation_toolkit/common/python/commonTools.py
index 47d49c467..1367f7868 100644
--- a/cd3_automation_toolkit/common/python/commonTools.py
+++ b/cd3_automation_toolkit/common/python/commonTools.py
@@ -120,7 +120,11 @@ def write_to_cd3(values_for_column, cd3file, sheet_name,append=False):
#rows_len=len(rows)
- rows_len = len(values_for_column["Region"])
+ try:
+ rows_len = len(values_for_column["Region"])
+ except KeyError as e:
+ rows_len = len(values_for_column["Location"])
+
sheet_max_rows = sheet.max_row
#If no rows exported from OCI, remove the sample data as well
if(rows_len == 0) :
diff --git a/cd3_automation_toolkit/connectGCP.properties b/cd3_automation_toolkit/connectGCP.properties
index d648b4727..f3b28e987 100644
--- a/cd3_automation_toolkit/connectGCP.properties
+++ b/cd3_automation_toolkit/connectGCP.properties
@@ -8,7 +8,7 @@
prefix=
# Auth Mechanism for GCP APIs - api_key
-# Please make sure to add IAM policies for service account before executing connectContainerToGCP.py
+# Please make sure to add IAM policies for service account before executing connectCloud.py
auth_mechanism=api_key
# Path to JSON file having private key information for the service account;
# Defaults to /cd3user/gcp/keys/gcp_api_private.json when left empty
diff --git a/cd3_automation_toolkit/example/CD3-Azure-template.xlsx b/cd3_automation_toolkit/example/CD3-Azure-template.xlsx
index 1d67462f1..506b0f9ba 100644
Binary files a/cd3_automation_toolkit/example/CD3-Azure-template.xlsx and b/cd3_automation_toolkit/example/CD3-Azure-template.xlsx differ
diff --git a/cd3_automation_toolkit/example/CD3-Blank-template.xlsx b/cd3_automation_toolkit/example/CD3-Blank-template.xlsx
index 13fbed379..eb57c8b29 100644
Binary files a/cd3_automation_toolkit/example/CD3-Blank-template.xlsx and b/cd3_automation_toolkit/example/CD3-Blank-template.xlsx differ
diff --git a/cd3_automation_toolkit/example/CD3-CIS-ManagementServices-template.xlsx b/cd3_automation_toolkit/example/CD3-CIS-ManagementServices-template.xlsx
index f526717d6..9dcf50d62 100644
Binary files a/cd3_automation_toolkit/example/CD3-CIS-ManagementServices-template.xlsx and b/cd3_automation_toolkit/example/CD3-CIS-ManagementServices-template.xlsx differ
diff --git a/cd3_automation_toolkit/example/CD3-CIS-template.xlsx b/cd3_automation_toolkit/example/CD3-CIS-template.xlsx
index dca5f12c1..8d1ec6e19 100644
Binary files a/cd3_automation_toolkit/example/CD3-CIS-template.xlsx and b/cd3_automation_toolkit/example/CD3-CIS-template.xlsx differ
diff --git a/cd3_automation_toolkit/example/CD3-Firewall-template.xlsx b/cd3_automation_toolkit/example/CD3-Firewall-template.xlsx
index 5d03186c5..510de86dc 100644
Binary files a/cd3_automation_toolkit/example/CD3-Firewall-template.xlsx and b/cd3_automation_toolkit/example/CD3-Firewall-template.xlsx differ
diff --git a/cd3_automation_toolkit/example/CD3-GCP-template.xlsx b/cd3_automation_toolkit/example/CD3-GCP-template.xlsx
index 671987e2c..40a417553 100644
Binary files a/cd3_automation_toolkit/example/CD3-GCP-template.xlsx and b/cd3_automation_toolkit/example/CD3-GCP-template.xlsx differ
diff --git a/cd3_automation_toolkit/example/CD3-HubSpoke-template.xlsx b/cd3_automation_toolkit/example/CD3-HubSpoke-template.xlsx
index 75ddc8fb6..a623fa2a0 100644
Binary files a/cd3_automation_toolkit/example/CD3-HubSpoke-template.xlsx and b/cd3_automation_toolkit/example/CD3-HubSpoke-template.xlsx differ
diff --git a/cd3_automation_toolkit/example/CD3-SingleVCN-template.xlsx b/cd3_automation_toolkit/example/CD3-SingleVCN-template.xlsx
index deff2f35b..ed6b29457 100644
Binary files a/cd3_automation_toolkit/example/CD3-SingleVCN-template.xlsx and b/cd3_automation_toolkit/example/CD3-SingleVCN-template.xlsx differ
diff --git a/cd3_automation_toolkit/gcpcloud/python/database/__init__.py b/cd3_automation_toolkit/gcpcloud/python/database/__init__.py
index 3a575345a..b917f0404 100644
--- a/cd3_automation_toolkit/gcpcloud/python/database/__init__.py
+++ b/cd3_automation_toolkit/gcpcloud/python/database/__init__.py
@@ -2,4 +2,7 @@
from .create_terraform_exa_infra_gcp import create_terraform_exa_infra_gcp
from .create_terraform_exa_vmclusters_gcp import create_terraform_exa_vmclusters_gcp
+from .create_terraform_adb_gcp import create_terraform_adb_gcp
+from .export_adb_gcp import export_adb_gcp
+from .export_exa_infra_gcp import export_exa_infra_gcp
diff --git a/cd3_automation_toolkit/gcpcloud/python/database/create_terraform_adb_gcp.py b/cd3_automation_toolkit/gcpcloud/python/database/create_terraform_adb_gcp.py
new file mode 100644
index 000000000..fb3742679
--- /dev/null
+++ b/cd3_automation_toolkit/gcpcloud/python/database/create_terraform_adb_gcp.py
@@ -0,0 +1,178 @@
+#!/usr/bin/python3
+# Copyright (c) 2016, 2019, Oracle and/or its affiliates. All rights reserved.
+#
+# This script will produce a Terraform file that will be used to set up OCI Database
+# Autonomous Database @GCP
+#
+# Author: Suruchi
+# Oracle Consulting
+#
+import os
+import sys
+import re
+from jinja2 import Environment, FileSystemLoader
+from pathlib import Path
+sys.path.append(os.getcwd()+"/..")
+from common.python.commonTools import *
+import gcpcloud.python.gcpCommonTools as gcpCommonTools
+
+
+######
+# Required Inputs- CD3 excel file, prefix AND outdir
+######
+# Execution of the code begins here
+def create_terraform_adb_gcp(inputfile, outdir, prefix):
+
+ filename = inputfile
+ sheetName = "ADB-GCP"
+ resource=sheetName.lower()
+ auto_tfvars_filename = prefix + '_' + sheetName.lower() + '.auto.tfvars'
+
+ # Load the template file
+ file_loader = FileSystemLoader(f'{Path(__file__).parent}/templates')
+ env = Environment(loader=file_loader, keep_trailing_newline=True, trim_blocks=True, lstrip_blocks=True)
+ template = env.get_template('adb-gcp-template')
+
+ tfStr=''
+
+
+ # Read cd3 using pandas dataframe
+ df, col_headers = commonTools.read_cd3(filename, sheetName)
+ #Remove empty rows
+ df = df.dropna(how='all')
+ df = df.reset_index(drop=True)
+
+ # List of the column headers
+ dfcolumns = df.columns.values.tolist()
+
+ # Iterate over rows
+ for i in df.index:
+ location = str(df.loc[i, 'Location']).strip()
+ # Encountered
+ if (location in commonTools.endNames):
+ break
+
+ location=location.strip().lower()
+
+ # temporary dictionary1 and dictionary2
+ tempStr = {}
+ tempdict = {}
+
+ # All columns are mandatory except customer contacts and tags
+ if (str(df.loc[i, 'Location']).lower() == 'nan' or \
+ str(df.loc[i, 'Project']).lower() == 'nan' or \
+ str(df.loc[i, 'ADB Display Name']).lower() == 'nan' or \
+ #str(df.loc[i, 'ODB Network Details']).lower() == 'nan' or \
+ #str(df.loc[i, 'ODB Network Subnet Details']).lower() == 'nan' or \
+ str(df.loc[i, 'DB Version']).lower() == 'nan'):
+ print("\nAll fields are mandatory. Please enter a value and try again !!")
+ print("\n** Exiting **")
+ exit(1)
+
+
+ for columnname in dfcolumns:
+ # Column value
+ columnvalue = str(df[columnname][i]).strip()
+
+ # Check for boolean/null in column values
+ columnvalue = commonTools.check_columnvalue(columnvalue)
+
+ # Check for multivalued columns
+ tempdict = commonTools.check_multivalues_columnvalue(columnvalue,columnname,tempdict)
+
+
+ # Process Defined and Freeform Tags
+ if columnname.lower() in gcpCommonTools.tagColumns:
+ tempdict = gcpCommonTools.split_tag_values(columnname, columnvalue, tempdict)
+
+ if columnname == "ADB Display Name":
+ display_name = columnvalue.strip()
+ display_tf_name = commonTools.check_tf_variable(display_name)
+ autonomous_database_id = re.sub(r'[^a-z0-9-]', '', display_name.lower())
+ tempdict = {'display_name': display_name, 'display_tf_name': display_tf_name,
+ 'autonomous_database_id': autonomous_database_id}
+ if columnname == "Autonomous Database ID":
+ columnvalue=columnvalue.strip()
+ if columnvalue!="":
+ tempdict = {'autonomous_database_id': columnvalue}
+
+
+ if columnname == 'Database Workload':
+ autonomous_value = columnvalue.strip().lower()
+ tempdict = {'db_workload': autonomous_value}
+
+ if columnname == "ODB Network Details":
+ if columnvalue!="":
+ values = columnvalue.split("::")
+ if len(values) < 3:
+ print("Invalid Value for ODB Network Details. Exiting!!")
+ exit(1)
+
+ if values[0].strip().upper() == "CREATE":
+ if len(values) != 5:
+ print("Invalid Value for ODB Network Details. Exiting!!")
+ exit(1)
+ create_odb_network = "true"
+ odb_network_project = values[1].strip()
+ vpc_network_name = values[2].strip()
+ odb_network_id = values[3].strip()
+ odb_network_gcp_oracle_zone = values[4].strip()
+
+ else:
+ create_odb_network = "false"
+ odb_network_project = values[0].strip()
+ vpc_network_name = values[1].strip()
+ odb_network_id = values[2].strip()
+ if (len(values) == 4):
+ odb_network_gcp_oracle_zone = values[3].strip()
+ else:
+ odb_network_gcp_oracle_zone = ""
+
+ tempdict = {'create_odb_network': create_odb_network, 'odb_network_project': odb_network_project,
+ 'vpc_network_name': vpc_network_name, 'odb_network_id': odb_network_id,
+ 'odb_network_gcp_oracle_zone': odb_network_gcp_oracle_zone}
+
+ if columnname == "ODB Network Subnet Details":
+ if columnvalue != "":
+ values = columnvalue.split("::")
+ if len(values) < 1:
+ print("Invalid Value for ODB Network Subnets Details. Exiting!!")
+ exit(1)
+
+ if values[0].strip().upper() == "CREATE":
+ if len(values) != 3:
+ print("Invalid Value for ODB Network Subnets Details. Exiting!!")
+ exit(1)
+ create_odb_network_subnets = "true"
+ odb_client_subnet_id = values[1].strip()
+ client_subnet_cidr = values[2].strip()
+
+
+ else:
+ create_odb_network_subnets = "false"
+ odb_client_subnet_id = values[0].strip()
+ client_subnet_cidr = ""
+
+
+ tempdict = {'create_odb_network_subnets': create_odb_network_subnets,
+ 'odb_client_subnet_id': odb_client_subnet_id, 'client_subnet_cidr': client_subnet_cidr}
+
+
+ columnname = commonTools.check_column_headers(columnname)
+ tempStr[columnname] = str(columnvalue).strip()
+ tempStr.update(tempdict)
+
+ # Write all info to TF string
+ tfStr=tfStr + template.render(tempStr)
+
+
+ if(tfStr!=''):
+ outfile = outdir + "/" + auto_tfvars_filename
+ commonTools.backup_file(outdir, resource, auto_tfvars_filename)
+ src = "##Add New ADB @GCP here##"
+ tfStr= template.render(count=0).replace(src, tfStr + "\n" + src)
+ tfStr = "".join([s for s in tfStr.strip().splitlines(True) if s.strip("\r\n").strip()])
+ oname=open(outfile,'w')
+ oname.write(tfStr)
+ oname.close()
+ print(outfile + " containing TF for ADB @GCP has been created")
diff --git a/cd3_automation_toolkit/gcpcloud/python/database/export_adb_gcp.py b/cd3_automation_toolkit/gcpcloud/python/database/export_adb_gcp.py
new file mode 100644
index 000000000..d7e713a91
--- /dev/null
+++ b/cd3_automation_toolkit/gcpcloud/python/database/export_adb_gcp.py
@@ -0,0 +1,371 @@
+#!/usr/bin/python3
+# Copyright (c) 2025, 2026, Oracle and/or its affiliates. All rights reserved.
+# This script will Export ADB @GCP resources into CD3 (existing worksheet) and write Terraform/tofu import commands
+# Author: Ulaganathan N
+# Oracle Consulting
+###############################################################################
+
+import os
+import sys
+import subprocess as sp
+
+from google.api_core.exceptions import GoogleAPIError
+
+sys.path.append(os.getcwd()+"/..")
+from common.python.commonTools import *
+import gcpcloud.python.gcpCommonTools as gcpCommonTools
+from typing import Dict, List, Optional
+from google.cloud import oracledatabase_v1
+
+# Global declaration
+importCommands: Dict[str, str] = {}
+
+
+def pick_first_not_none(*values):
+ """Return the first non-None value from a list of values."""
+ for v in values:
+ if v is not None:
+ return v
+ return None
+
+
+def normalize_enum_token(value):
+ """
+ Return the enum/member token expected by tfvars as a string.
+ For Example:
+ 'WorkloadType.OLTP' -> 'OLTP'
+ """
+ if value is None:
+ return ""
+ s = str(value)
+ return s.split(".", 1)[1] if "." in s else s
+
+
+def _format_rg_vnet_subnet_from_id(net_id: str) -> str:
+ """Return 'resourceGroup@vnet::subnet' from a subnet ARM ID."""
+ if not net_id:
+ return ""
+ parts = net_id.strip("/").split("/")
+ try:
+ rg = parts[parts.index("resourceGroups") + 1]
+ vnet = parts[parts.index("virtualNetworks") + 1]
+ subnet = parts[parts.index("subnets") + 1]
+ return f"{rg}@{vnet}::{subnet}"
+ except (ValueError, IndexError):
+ return ""
+
+
+def _get_rg_from_id(resource_id: str) -> str:
+ # ARM ID format: /subscriptions//resourceGroups//providers/.../autonomousDatabases/
+ try:
+ parts = resource_id.split("/")
+ idx = parts.index("resourceGroups")
+ return parts[idx + 1]
+ except Exception:
+ return ""
+
+
+def _list_adbs(active_projects,regions):
+ """Yield ADB @GCP resources across pall projects"""
+ for project in active_projects:
+ for region in regions:
+ response=[]
+ parent_path = f"projects/{project}/locations/{region}"
+ try:
+ request = oracledatabase_v1.ListAutonomousDatabasesRequest(parent=parent_path)
+ response = client.list_autonomous_databases(request)
+ except Exception as e:
+ print("Skipping Region: "+region)
+ pass
+ count = 0
+ for adb in response:
+ print("Fetching for "+str(request))
+ yield adb
+ count += 1
+
+
+
+def print_adbs_gcp(adb, values_for_column: Dict[str, List], state: Dict, tf_or_tofu: str, ):
+ """Populate CD3 columns for a single ADB @GCP and queue Terraform import commands."""
+
+ props = getattr(adb, "properties", None) or adb
+
+ # Resource names and IDs
+ adb_name = getattr(adb, "name", "")
+ parts=adb_name.split("/")
+ project=parts[parts.index("projects") + 1]
+ location=parts[parts.index("locations") + 1]
+
+
+ # Contacts (list of objects with .email)
+
+ contacts_csv = ""
+
+ try:
+ contacts = getattr(props, "customer_contacts", None)
+ if contacts:
+ emails = []
+ for c in contacts:
+ email = getattr(c, "email", None) or getattr(c, "contact", None)
+ if email:
+ emails.append(email)
+ contacts_csv = ",".join(emails)
+ except Exception:
+ contacts_csv = ""
+
+
+ # Tags (dict) — kept as metadata only if the sheet has 'Common Tags' column(not used yet)
+ #common_tags = gcpCommonTools._flatten_tags(getattr(adb, "tags", None))
+
+ common_tags=""
+
+ odb_network_details=""
+ subnet_id=""
+ try:
+ network=adb.odb_subnet
+ parts = network.strip("/").split("/")
+ network_project_id=parts[parts.index("projects") + 1]
+ odb_network_id=parts[parts.index("odbNetworks") + 1]
+ subnet_id = parts[parts.index("odbSubnets") + 1]
+
+ # Retrieve the ODB Network resource
+ response = client.get_odb_network(name=adb.odb_network)
+ # Access the mapped GCP VPC network URI
+ gcp_vpc_uri = response.network
+ parts = gcp_vpc_uri.strip("/").split("/")
+ gcp_vpc= parts[parts.index("networks") + 1]
+
+ odb_network_details = network_project_id+"::"+gcp_vpc+"::"+odb_network_id
+
+ except Exception as e:
+ print("Network Details empty for "+adb.display_name)
+
+
+
+ compute_count = (
+ getattr(props, "compute_count", None)
+ or getattr(props, "computeCount", None)
+ or ""
+ )
+ ocpu_cores = (
+ getattr(props, "cpu_core_count", None)
+ or getattr(props, "ocpuCoreCount", None)
+ or ""
+ )
+
+ database_name = adb.database
+ display_name = adb.display_name
+ name=adb.name
+ parts = name.strip("/").split("/")
+ autonomous_database_id = parts[parts.index("autonomousDatabases") + 1]
+
+
+ db_version = getattr(props, "db_version", None) or ""
+ db_edition = getattr(props, "db_edition", None)
+ db_edition = db_edition.name
+ storage_tbs = getattr(props, "data_storage_size_tb", None) or ""
+ storage_gbs = ""
+ if storage_tbs == "":
+ storage_gbs = getattr(props, "data_storage_size_gb", None) or ""
+ workload= getattr(props, "db_workload", None)
+ workload = workload.name
+
+ private_endpoint_ip = getattr(props, "private_endpoint_ip", None) or ""
+ private_endpoint_label = getattr(props, "private_endpoint_label", None) or ""
+
+ license_type = getattr(props, "license_type", None)
+ license_type=license_type.name
+
+ backup_retention_days = (getattr(props, "backup_retention_period_days", None) or getattr(props, "backupRetentionDays",
+ None) or "")
+ # Character sets
+ char_set = pick_first_not_none(
+ getattr(props, "character_set", None),
+ getattr(props, "characterSet", None),
+ )
+ nchar_set = pick_first_not_none(
+ getattr(props, "n_character_set", None),
+ getattr(props, "ncharacterSet", None),
+ )
+
+ maintenance_schedule_type = getattr(props, "maintenance_schedule_type", None)
+ maintenance_schedule_type = maintenance_schedule_type.name
+
+ # Auto-scaling flags
+ auto_scaling_storage = pick_first_not_none(
+ getattr(props, "is_storage_auto_scaling_enabled", None))
+ auto_scaling_enabled = pick_first_not_none(
+ getattr(props, "is_auto_scaling_enabled", None))
+
+ # mTLS requirement
+ mtls_required = pick_first_not_none(
+ getattr(props, "mtls_connection_required", None),
+ getattr(props, "isMtlsConnectionRequired", None),
+ )
+
+ module_name = "adb-gcp"
+ resource_type = "google_oracle_database_autonomous_database"
+ resource_name_in_module = "autonomous_database" # Need to change if tf module uses a different name
+
+ adb_tf_name = location +"_"+project+"_"+autonomous_database_id
+
+ # module.[""]..
+ tf_address = f'module.{module_name}["{adb_tf_name}"].{resource_type}.{resource_name_in_module}'
+
+ # Avoid duplicate imports by checking current state addresses
+ if tf_address not in state.get("resources", []):
+ # Wrap ADDRESS in single quotes to avoid escaping the ["] in a POSIX shell
+ importCommands["global"] += f"\n{tf_or_tofu} import '{tf_address}' {getattr(adb, 'name', '')}"
+
+ # Populate CD3 columns as per provided header list (write only if column exists)
+ for col_header in values_for_column:
+ if col_header in ("Location"):
+ values_for_column[col_header].append(location)
+ elif col_header == "Project":
+ # If the sheet still has Region, fill it from GCP location (no per-region scripting)
+ values_for_column[col_header].append(project)
+ elif col_header == "ADB Display Name":
+ values_for_column[col_header].append(display_name)
+ elif col_header == "Autonomous Database ID":
+ values_for_column[col_header].append(autonomous_database_id)
+ elif col_header == "ODB Network Details":
+ values_for_column[col_header].append(odb_network_details)
+ elif col_header == "ODB Network Subnet Details":
+ values_for_column[col_header].append(subnet_id)
+ elif col_header == "Private Endpoint IP":
+ values_for_column[col_header].append(private_endpoint_ip)
+ elif col_header == "Private Endpoint Label":
+ values_for_column[col_header].append(private_endpoint_label)
+ elif col_header == "Maintenance Schedule Type":
+ values_for_column[col_header].append(maintenance_schedule_type)
+ elif col_header == "Database Name":
+ values_for_column[col_header].append(database_name)
+ elif col_header == "DB Version":
+ values_for_column[col_header].append(db_version)
+ elif col_header == "DB Edition":
+ values_for_column[col_header].append(db_edition)
+ elif col_header == "Admin Password":
+ values_for_column[col_header].append("Rand0mPaswd#123") # never retrievable
+
+ elif col_header == "Compute Count":
+ values_for_column[col_header].append(compute_count)
+
+ elif col_header in ("Data Storage Size TB"):
+ values_for_column[col_header].append(storage_tbs)
+ elif col_header in ("Data Storage Size GB"):
+ values_for_column[col_header].append(storage_gbs)
+ elif col_header == "Database Workload":
+ if workload == "DW":
+ workload = "LakeHouse"
+ elif workload == "AJD":
+ workload = "JSON"
+ elif workload == "OLTP":
+ workload = "ATP"
+ elif workload == "APEX":
+ workload = "APEX"
+ values_for_column[col_header].append(workload.upper())
+ elif col_header == "License Type":
+ values_for_column[col_header].append(license_type)
+ elif col_header == "Backup Retention Period In Days":
+ values_for_column[col_header].append(backup_retention_days)
+ elif col_header == 'Character Set':
+ values_for_column[col_header].append(char_set)
+ elif col_header == 'nCharacter Set':
+ values_for_column[col_header].append(nchar_set)
+ elif col_header == "Is Storage Auto Scaling Enabled":
+ values_for_column[col_header].append(auto_scaling_storage)
+ elif col_header == "Is Auto Scaling Enabled":
+ values_for_column[col_header].append(auto_scaling_enabled)
+ elif col_header == "MTLS Connection Required":
+ values_for_column[col_header].append(mtls_required)
+ elif col_header == "Email":
+ values_for_column[col_header].append(contacts_csv)
+ elif col_header == "Common Tags":
+ values_for_column[col_header].append(common_tags)
+ else:
+ values_for_column[col_header].append("")
+ '''
+ elif col_header.lower() in azrCommonTools.tagColumns:
+ try:
+ values_for_column = commonTools.export_tags(adb, col_header, values_for_column)
+ except Exception:
+ values_for_column[col_header].append("")
+
+ else:
+ # Extra/custom columns via Excel_Columns mapping
+ try:
+ oci_objs = [adb]
+ values_for_column = commonTools.export_extra_columns(oci_objs, col_header, sheet_dict,
+ values_for_column)
+ except Exception:
+ values_for_column[col_header].append("")
+ '''
+
+
+
+def export_adb_gcp(inputfile: str, outdir: str,credentials,active_projects: [List[str]],regions: [List[str]],) :
+ """
+ Export ADB @GCP resources into CD3 (existing worksheet) and write Terraform/tofu import commands.
+ - No region/service_dir/export_tags/compartment scoping.
+ - Single import script at outdir/gcp folder.
+ """
+ global importCommands, sheet_dict, client
+
+ tf_or_tofu = "terraform"
+
+ # Validate input Excel
+ cd3file = inputfile
+ if '.xls' not in cd3file:
+ print("\nAcceptable cd3 format: .xlsx")
+ sys.exit(1)
+ sheetName = "ADB-GCP"
+ # Read CD3
+ df, values_for_column = commonTools.read_cd3(cd3file, sheetName)
+
+ # Get dict for columns from Excel_Columns
+
+ print("\nCD3 excel file should not be opened during export process!!!")
+ print("Tab- ADB-GCP will be overwritten during export process!!!\n")
+
+ # Prepare a single import commands script at outdir (subscription scope)
+
+ resource = 'import_' + sheetName.lower()
+ file_name = 'import_commands_' + sheetName.lower() + '.sh'
+ script_file = os.path.join(outdir, file_name)
+ if os.path.exists(script_file):
+ commonTools.backup_file(outdir, resource, file_name)
+ os.makedirs(outdir, exist_ok=True)
+ importCommands["global"] = ""
+
+
+ client = oracledatabase_v1.OracleDatabaseClient(credentials=credentials)
+ print("\nFetching details of ADB @GCP...")
+
+ # Build state resources (to avoid duplicate import lines) at outdir
+ state = {'path': outdir, 'resources': []}
+ try:
+ tf_state_list = [tf_or_tofu, "state", "list"]
+ byteOutput = sp.check_output(tf_state_list, cwd=state["path"], stderr=sp.DEVNULL)
+ output = byteOutput.decode('UTF-8').rstrip()
+ for item in output.split('\n'):
+ state["resources"].append(item.replace("\"", "\\\""))
+ except Exception:
+ pass
+
+ # Iterate ADBs
+
+ for adb in _list_adbs(active_projects,regions):
+ print_adbs_gcp(adb, values_for_column, state, tf_or_tofu)
+
+ # Write back to CD3
+ commonTools.write_to_cd3(values_for_column, cd3file, sheetName)
+ # Region count if present, else any main column (e.g., ADB Display Name)
+ count_col = "ADB Display Name" if "ADB Display Name" in values_for_column else next(iter(values_for_column.keys()))
+ print("{0} ADB @GCP exported into CD3.\n".format(len(values_for_column.get(count_col, []))))
+
+ # Write import script
+ init_commands = f'\n######### Writing import for ADB @GCP #########\n\n#!/bin/bash\n{tf_or_tofu} init'
+ if importCommands.get("global"):
+ importCommands["global"] += f'\n{tf_or_tofu} plan\n'
+ with open(script_file, 'a', encoding='utf-8') as importCommandsfile:
+ importCommandsfile.write(init_commands + importCommands["global"])
diff --git a/cd3_automation_toolkit/gcpcloud/python/database/export_exa_infra_gcp.py b/cd3_automation_toolkit/gcpcloud/python/database/export_exa_infra_gcp.py
new file mode 100644
index 000000000..c3e92a2b8
--- /dev/null
+++ b/cd3_automation_toolkit/gcpcloud/python/database/export_exa_infra_gcp.py
@@ -0,0 +1,279 @@
+#!/usr/bin/python3
+# Copyright (c) 2025, 2026, Oracle and/or its affiliates. All rights reserved.
+# This script will Export ADB @GCP resources into CD3 (existing worksheet) and write Terraform/tofu import commands
+# Author: Ulaganathan N
+# Oracle Consulting
+###############################################################################
+
+import os
+import sys
+import subprocess as sp
+
+from google.api_core.exceptions import GoogleAPIError
+
+sys.path.append(os.getcwd()+"/..")
+from common.python.commonTools import *
+import gcpcloud.python.gcpCommonTools as gcpCommonTools
+from typing import Dict, List, Optional
+from google.cloud import oracledatabase_v1
+
+# Global declaration
+importCommands: Dict[str, str] = {}
+
+
+def pick_first_not_none(*values):
+ """Return the first non-None value from a list of values."""
+ for v in values:
+ if v is not None:
+ return v
+ return None
+
+
+def normalize_enum_token(value):
+ """
+ Return the enum/member token expected by tfvars as a string.
+ For Example:
+ 'WorkloadType.OLTP' -> 'OLTP'
+ """
+ if value is None:
+ return ""
+ s = str(value)
+ return s.split(".", 1)[1] if "." in s else s
+
+
+def _format_rg_vnet_subnet_from_id(net_id: str) -> str:
+ """Return 'resourceGroup@vnet::subnet' from a subnet ARM ID."""
+ if not net_id:
+ return ""
+ parts = net_id.strip("/").split("/")
+ try:
+ rg = parts[parts.index("resourceGroups") + 1]
+ vnet = parts[parts.index("virtualNetworks") + 1]
+ subnet = parts[parts.index("subnets") + 1]
+ return f"{rg}@{vnet}::{subnet}"
+ except (ValueError, IndexError):
+ return ""
+
+
+def _get_rg_from_id(resource_id: str) -> str:
+ # ARM ID format: /subscriptions//resourceGroups//providers/.../autonomousDatabases/
+ try:
+ parts = resource_id.split("/")
+ idx = parts.index("resourceGroups")
+ return parts[idx + 1]
+ except Exception:
+ return ""
+
+
+def _list_exa_infra(active_projects,regions):
+ """Yield ADB @GCP resources across pall projects"""
+ for project in active_projects:
+ for region in regions:
+ response=[]
+ parent_path = f"projects/{project}/locations/{region}"
+ try:
+ request = oracledatabase_v1.ListCloudExadataInfrastructuresRequest(parent=parent_path)
+ response = client.list_cloud_exadata_infrastructures(request)
+ except Exception as e:
+ print("Skipping Region: "+region)
+ pass
+ count = 0
+ for exa_infra in response:
+ print("Fetching for "+str(request))
+ yield exa_infra
+ count += 1
+
+
+
+def print_exa_infra_gcp(exa_infra, values_for_column: Dict[str, List], state: Dict, tf_or_tofu: str, ):
+ """Populate CD3 columns for a single ADB @GCP and queue Terraform import commands."""
+
+
+ print(exa_infra)
+ print("==================")
+ props = getattr(exa_infra, "properties", None) or exa_infra
+ print(props)
+
+ # Resource names and IDs
+ exa_infra_name = getattr(exa_infra, "name", "")
+ parts=exa_infra_name.split("/")
+ project=parts[parts.index("projects") + 1]
+ location=parts[parts.index("locations") + 1]
+
+
+ # Contacts (list of objects with .email)
+ contacts_csv = ""
+ try:
+ contacts = getattr(props, "customer_contacts", None)
+ if contacts:
+ emails = []
+ for c in contacts:
+ email = getattr(c, "email", None) or getattr(c, "contact", None)
+ if email:
+ emails.append(email)
+ contacts_csv = ",".join(emails)
+ except Exception:
+ contacts_csv = ""
+
+
+ # Tags (dict) — kept as metadata only if the sheet has 'Common Tags' column(not used yet)
+ #common_tags = gcpCommonTools._flatten_tags(getattr(adb, "tags", None))
+
+ common_tags=""
+ display_name = exa_infra.display_name
+
+ shape = getattr(props, "shape", None)
+ compute_count = getattr(props, "compute_count", None)
+ storage_count = getattr(props, "storage_count", None)
+ maintenance_window = getattr(props, "maintenance_window", None)
+ preference = maintenance_window.preference.name
+ months = maintenance_window.months
+ months=",".join(str(num) for num in months)
+ weeks_of_month = maintenance_window.weeks_of_month
+ weeks_of_month=",".join(str(num) for num in weeks_of_month)
+ days_of_week = maintenance_window.days_of_week
+ days_of_week = ",".join(str(num) for num in days_of_week)
+ hours_of_day = maintenance_window.hours_of_day
+ hours_of_day = ",".join(str(num) for num in hours_of_day)
+ lead_time_week = maintenance_window.lead_time_week
+ if lead_time_week ==0:
+ lead_time_week=""
+
+ patching_mode = maintenance_window.patching_mode.name
+ gcp_oracle_zone = exa_infra.gcp_oracle_zone
+
+ module_name = "exa-infra-gcp"
+ resource_type = "google_oracle_database_cloud_exadata_infrastructure"
+ resource_name_in_module = "exadata_infrastructure"
+
+ exa_infra_tf_name = commonTools.check_tf_variable(display_name)
+
+ # module.[""]..
+ tf_address = f'module.{module_name}["{exa_infra_tf_name}"].{resource_type}.{resource_name_in_module}'
+
+ # Avoid duplicate imports by checking current state addresses
+ if tf_address not in state.get("resources", []):
+ # Wrap ADDRESS in single quotes to avoid escaping the ["] in a POSIX shell
+ importCommands["global"] += f"\n{tf_or_tofu} import '{tf_address}' {getattr(exa_infra, 'name', '')}"
+
+ # Populate CD3 columns as per provided header list (write only if column exists)
+ for col_header in values_for_column:
+ if col_header in ("Location"):
+ values_for_column[col_header].append(location)
+ elif col_header == "Project":
+ values_for_column[col_header].append(project)
+ elif col_header == "GCP Oracle Zone":
+ values_for_column[col_header].append(gcp_oracle_zone)
+ elif col_header == "Exadata Infra Display Name":
+ values_for_column[col_header].append(display_name)
+ elif col_header == "Shape":
+ values_for_column[col_header].append(shape)
+ elif col_header == "Database Servers":
+ values_for_column[col_header].append(compute_count)
+ elif col_header == "Storage Servers":
+ values_for_column[col_header].append(storage_count)
+ elif col_header == "Maintenance Method":
+ values_for_column[col_header].append(patching_mode)
+ elif col_header == "Preference":
+ values_for_column[col_header].append(preference)
+ elif col_header == "Months":
+ values_for_column[col_header].append(months)
+ elif col_header == "Weeks of Month":
+ values_for_column[col_header].append(weeks_of_month)
+ elif col_header == "Hours of Day":
+ values_for_column[col_header].append(hours_of_day)
+ elif col_header == "Days of Week":
+ values_for_column[col_header].append(days_of_week)
+ elif col_header == "Lead Time in Weeks":
+ values_for_column[col_header].append(lead_time_week)
+ elif col_header == "Email":
+ values_for_column[col_header].append(contacts_csv)
+ elif col_header == "Common Tags":
+ values_for_column[col_header].append(common_tags)
+ else:
+ values_for_column[col_header].append("")
+ '''
+ elif col_header.lower() in azrCommonTools.tagColumns:
+ try:
+ values_for_column = commonTools.export_tags(adb, col_header, values_for_column)
+ except Exception:
+ values_for_column[col_header].append("")
+
+ else:
+ # Extra/custom columns via Excel_Columns mapping
+ try:
+ oci_objs = [adb]
+ values_for_column = commonTools.export_extra_columns(oci_objs, col_header, sheet_dict,
+ values_for_column)
+ except Exception:
+ values_for_column[col_header].append("")
+ '''
+
+
+
+def export_exa_infra_gcp(inputfile: str, outdir: str,credentials,active_projects: [List[str]],regions: [List[str]],) :
+ """
+ Export Exa-Infra @GCP resources into CD3 (existing worksheet) and write Terraform/tofu import commands.
+ - No region/service_dir/export_tags/compartment scoping.
+ - Single import script at outdir/gcp folder.
+ """
+ global importCommands, sheet_dict, client
+
+ tf_or_tofu = "terraform"
+
+ # Validate input Excel
+ cd3file = inputfile
+ if '.xls' not in cd3file:
+ print("\nAcceptable cd3 format: .xlsx")
+ sys.exit(1)
+ sheetName = "EXA-Infra-GCP"
+ # Read CD3
+ df, values_for_column = commonTools.read_cd3(cd3file, sheetName)
+
+ # Get dict for columns from Excel_Columns
+
+ print("\nCD3 excel file should not be opened during export process!!!")
+ print("Tab- EXA-INFRA-GCP will be overwritten during export process!!!\n")
+
+ # Prepare a single import commands script at outdir (subscription scope)
+
+ resource = 'import_' + sheetName.lower()
+ file_name = 'import_commands_' + sheetName.lower() + '.sh'
+ script_file = os.path.join(outdir, file_name)
+ if os.path.exists(script_file):
+ commonTools.backup_file(outdir, resource, file_name)
+ os.makedirs(outdir, exist_ok=True)
+ importCommands["global"] = ""
+
+
+ client = oracledatabase_v1.OracleDatabaseClient(credentials=credentials)
+ print("\nFetching details of Exa Infra @GCP...")
+
+ # Build state resources (to avoid duplicate import lines) at outdir
+ state = {'path': outdir, 'resources': []}
+ try:
+ tf_state_list = [tf_or_tofu, "state", "list"]
+ byteOutput = sp.check_output(tf_state_list, cwd=state["path"], stderr=sp.DEVNULL)
+ output = byteOutput.decode('UTF-8').rstrip()
+ for item in output.split('\n'):
+ state["resources"].append(item.replace("\"", "\\\""))
+ except Exception:
+ pass
+
+ # Iterate Exa Infra
+
+ for exa_infra in _list_exa_infra(active_projects,regions):
+ print_exa_infra_gcp(exa_infra, values_for_column, state, tf_or_tofu)
+
+ # Write back to CD3
+ commonTools.write_to_cd3(values_for_column, cd3file, sheetName)
+ # Region count if present, else any main column (e.g., ADB Display Name)
+ count_col = "Exadata Infra Display Name" if "Exadata Infra Display Name" in values_for_column else next(iter(values_for_column.keys()))
+ print("{0} Exa Infra @GCP exported into CD3.\n".format(len(values_for_column.get(count_col, []))))
+
+ # Write import script
+ init_commands = f'\n######### Writing import for Exa Infra @GCP #########\n\n#!/bin/bash\n{tf_or_tofu} init'
+ if importCommands.get("global"):
+ importCommands["global"] += f'\n{tf_or_tofu} plan\n'
+ with open(script_file, 'a', encoding='utf-8') as importCommandsfile:
+ importCommandsfile.write(init_commands + importCommands["global"])
\ No newline at end of file
diff --git a/cd3_automation_toolkit/gcpcloud/python/database/templates/adb-gcp-template b/cd3_automation_toolkit/gcpcloud/python/database/templates/adb-gcp-template
new file mode 100644
index 000000000..93ba83a48
--- /dev/null
+++ b/cd3_automation_toolkit/gcpcloud/python/database/templates/adb-gcp-template
@@ -0,0 +1,119 @@
+{% if count == 0 %}
+# Copyright (c) 2024, Oracle and/or its affiliates. All rights reserved.
+# Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl.
+#
+############################
+# ADB @GCP
+# ADB @GCP - tfvars
+############################
+
+gcp_oci_adb = {
+ ##Add New ADB @GCP here##
+}
+{% else %}
+ {{ location }}_{{ project }}_{{ autonomous_database_id }} = {
+ location = "{{ location }}"
+ project = "{{ project }}"
+ autonomous_database_id = "{{ autonomous_database_id }}"
+ database = "{{ database_name }}"
+ display_name = "{{ display_name }}"
+ admin_password = "{{ admin_password }}"
+
+ {% if create_odb_network %}
+ create_odb_network = {{ create_odb_network }}
+ {% endif %}
+ {% if create_odb_network_subnets %}
+ create_odb_network_subnets = {{ create_odb_network_subnets }}
+ {% endif %}
+ {% if odb_network_project %}
+ odb_network_project = "{{ odb_network_project }}"
+ {% endif %}
+ {% if vpc_network_name %}
+ vpc_network_name = "{{ vpc_network_name }}"
+ {% endif %}
+ {% if odb_network_gcp_oracle_zone %}
+ odb_network_gcp_oracle_zone = "{{ odb_network_gcp_oracle_zone }}"
+ {% endif %}
+ {% if client_subnet_cidr %}
+ client_subnet_cidr = "{{ client_subnet_cidr }}"
+ {% endif %}
+ {% if odb_network_id %}
+ odb_network_id = "{{ odb_network_id }}"
+ {% endif %}
+ {% if odb_client_subnet_id %}
+ odb_client_subnet_id = "{{ odb_client_subnet_id }}"
+ {% endif %}
+ {% if private_endpoint_ip %}
+ private_endpoint_ip = "{{ private_endpoint_ip }}"
+ {% endif %}
+ {% if private_endpoint_label %}
+ private_endpoint_label = "{{ private_endpoint_label }}"
+ {% endif %}
+
+ compute_count = {{ compute_count }}
+ {% if data_storage_size_tb %}
+ data_storage_size_tb = {{ data_storage_size_tb }}
+ {% endif %}
+ {% if data_storage_size_gb %}
+ data_storage_size_gb = {{ data_storage_size_gb }}
+ {% endif %}
+ db_version = "{{ db_version }}"
+ {% if db_workload.lower() == 'lakehouse' %}
+ db_workload = "DW"
+ {% elif db_workload.lower() == 'atp' %}
+ db_workload = "OLTP"
+ {% elif db_workload.lower() == 'json' %}
+ db_workload = "AJD"
+ {% elif db_workload.lower() == 'apex' %}
+ db_workload = "APEX"
+ {% else %}
+ db_workload = "{{ db_workload }}"
+ {% endif %}
+ db_edition = "{{ db_edition }}"
+ {% if character_set != "" and character_set != "nan" %}
+ character_set = "{{ character_set }}"
+ {% endif %}
+ {% if ncharacter_set != "" and ncharacter_set != "nan" %}
+ n_character_set = "{{ ncharacter_set }}"
+ {% endif %}
+ license_type = "{{ license_type }}"
+
+ {% if is_auto_scaling_enabled %}
+ is_auto_scaling_enabled = {{ is_auto_scaling_enabled }}
+ {% endif %}
+ {% if is_storage_auto_scaling_enabled %}
+ is_storage_auto_scaling_enabled = {{ is_storage_auto_scaling_enabled }}
+ {% endif %}
+
+ {% if maintenance_schedule_type != "" and maintenance_schedule_type != "nan" %}
+ maintenance_schedule_type = "{{ maintenance_schedule_type }}"
+ {% endif %}
+ {% if mtls_connection_required != "" and mtls_connection_required != "nan" %}
+ mtls_connection_required = "{{ mtls_connection_required }}"
+ {% endif %}
+ {% if backup_retention_period_days != null %}
+ backup_retention_period_days = {{ backup_retention_period_days }}
+ {% endif %}
+ email = "{{ email }}"
+
+ {# ##Do not modify below this line## #}
+ {# #}
+ {# ###Section for adding Labels### #}
+ {% if labels and labels != 'nan' and labels != '' and labels != [['nan']] %}
+ {% if labels[0] %}
+ labels = {
+ {% for label in labels %}
+ {% if not loop.last %}
+ "{{ label[0] }}" = "{{ label[1] }}",
+ {% else %}
+ "{{ label[0] }}" = "{{ label[1] }}"
+ {% endif %}
+ {% endfor %}
+ }
+ {% endif %}
+ {% endif %}
+
+ {# ###Section for adding Labels ends here### #}
+ },
+
+{% endif %}
diff --git a/cd3_automation_toolkit/gcpcloud/python/gcpCommonTools.py b/cd3_automation_toolkit/gcpcloud/python/gcpCommonTools.py
index 3dc648938..a720efeb6 100644
--- a/cd3_automation_toolkit/gcpcloud/python/gcpCommonTools.py
+++ b/cd3_automation_toolkit/gcpcloud/python/gcpCommonTools.py
@@ -2,21 +2,51 @@
from typing import Dict, Optional
import os
from google.oauth2 import service_account
+from google.api_core.exceptions import GoogleAPIError
+from google.cloud import resourcemanager_v3
class gcpCommonTools():
- tagColumns = {'labels'}
+ tagColumns = {'labels', 'Labels'}
def authenticate(self,config_file):
# GCP credential & client
try:
+ if config_file=='':
+ print("\nCannot run export workflow as authentication parameters are missing!!\n")
+ exit()
credentials = service_account.Credentials.from_service_account_file(config_file)
except Exception as e:
print("\n")
print(str(e))
print(f"\nError reading credentials file. Exiting!!!\n")
exit(1)
- return credentials
+
+ projects_client = resourcemanager_v3.ProjectsClient(credentials=credentials)
+ from google.cloud import oracledatabase_v1
+
+ shared_client = oracledatabase_v1.OracleDatabaseClient(credentials=credentials)
+ active_project_ids = []
+
+ try:
+ # Search across all accessible projects
+ search_request = resourcemanager_v3.SearchProjectsRequest()
+ project_pages = projects_client.search_projects(request=search_request)
+ regions=[]
+ for project in project_pages:
+ # Only target live, active projects (exclude deleted/pending delete)
+ if project.state == resourcemanager_v3.Project.State.ACTIVE:
+ active_project_ids.append(project.project_id)
+
+ from google.cloud import compute_v1
+ client = compute_v1.RegionsClient(credentials=credentials)
+ for region in client.list(project=project.project_id):
+ regions.append(region.name)
+
+ except GoogleAPIError as e:
+ print(f"Error reading project hierarchy: {e}")
+ print("Falling back. Please verify organization viewer permissions.")
+ return credentials,active_project_ids,regions
def split_tag_values(columnname, columnvalue, tempdict):
columnvalue = columnvalue.replace("\n", "")
diff --git a/cd3_automation_toolkit/gcpcloud/terraform/adb-gcp.tf b/cd3_automation_toolkit/gcpcloud/terraform/adb-gcp.tf
new file mode 100644
index 000000000..054ee7515
--- /dev/null
+++ b/cd3_automation_toolkit/gcpcloud/terraform/adb-gcp.tf
@@ -0,0 +1,34 @@
+# Copyright (c) 2024, Oracle and/or its affiliates. All rights reserved.
+# Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl.
+#
+
+##################################
+# Module Block - ADB @GCP
+# Create ADB
+##################################
+
+/*
+data "google_oracle_database_odb_network" "gcp_odb_network" {
+
+ location = "us-east4"
+ project = "oradb-489018"
+ odb_network_id = "nework1"
+}
+
+output "odb_network1" {
+ description = "Exadata Infrastructure ID"
+ value = data.google_oracle_database_odb_network.gcp_odb_network.id
+}
+*/
+module "adb-gcp" {
+
+ depends_on = [module.gcp_network]
+ source = "./modules/gcp-oci-adb"
+ for_each = var.gcp_oci_adb != null ? var.gcp_oci_adb : {}
+
+ adb_config = each.value
+ odb_network_id = each.value.odb_network_id!=null ? "projects/${each.value.odb_network_project}/locations/${each.value.location}/odbNetworks/${each.value.odb_network_id}" : null
+ odb_client_subnet_id = each.value.odb_network_id!=null && each.value.odb_client_subnet_id!=null ? "projects/${each.value.odb_network_project}/locations/${each.value.location}/odbNetworks/${each.value.odb_network_id}/odbSubnets/${each.value.odb_client_subnet_id}" : null
+
+ labels = each.value.labels
+}
diff --git a/cd3_automation_toolkit/gcpcloud/terraform/modules/gcp-oci-adb/main.tf b/cd3_automation_toolkit/gcpcloud/terraform/modules/gcp-oci-adb/main.tf
new file mode 100644
index 000000000..db071e492
--- /dev/null
+++ b/cd3_automation_toolkit/gcpcloud/terraform/modules/gcp-oci-adb/main.tf
@@ -0,0 +1,47 @@
+resource "google_oracle_database_autonomous_database" "autonomous_database"{
+
+ location = var.adb_config.location
+ project = var.adb_config.project
+ autonomous_database_id = var.adb_config.autonomous_database_id
+ database = var.adb_config.database
+ display_name = var.adb_config.display_name
+ admin_password = var.adb_config.admin_password
+ odb_network = var.odb_network_id
+ odb_subnet = var.odb_client_subnet_id
+ properties {
+ compute_count = var.adb_config.compute_count
+ cpu_core_count = var.adb_config.cpu_core_count
+ data_storage_size_tb = var.adb_config.data_storage_size_tb
+ data_storage_size_gb = var.adb_config.data_storage_size_gb
+ db_version = var.adb_config.db_version
+ db_workload = var.adb_config.db_workload
+ db_edition = var.adb_config.db_edition
+ n_character_set = var.adb_config.n_character_set
+ character_set = var.adb_config.character_set
+ license_type = var.adb_config.license_type
+
+ private_endpoint_ip = var.adb_config.private_endpoint_ip
+ private_endpoint_label = var.adb_config.private_endpoint_label
+
+ is_auto_scaling_enabled = var.adb_config.is_auto_scaling_enabled
+ is_storage_auto_scaling_enabled = var.adb_config.is_storage_auto_scaling_enabled
+
+ maintenance_schedule_type = var.adb_config.maintenance_schedule_type
+ mtls_connection_required = var.adb_config.mtls_connection_required
+ backup_retention_period_days = var.adb_config.backup_retention_period_days
+ customer_contacts {
+ email = var.adb_config.email
+ }
+ }
+ deletion_protection = "true"
+ lifecycle {
+ ignore_changes = [
+ properties["db_edition"],
+ cidr,
+ admin_password,
+ network,
+ terraform_labels
+ ]
+ }
+
+}
\ No newline at end of file
diff --git a/cd3_automation_toolkit/gcpcloud/terraform/modules/gcp-oci-adb/variables.tf b/cd3_automation_toolkit/gcpcloud/terraform/modules/gcp-oci-adb/variables.tf
new file mode 100644
index 000000000..5e57c8c2a
--- /dev/null
+++ b/cd3_automation_toolkit/gcpcloud/terraform/modules/gcp-oci-adb/variables.tf
@@ -0,0 +1,61 @@
+variable "labels" {
+ description = "A mapping of tags which should be assigned to the Cloud Exadata Infrastructure."
+ type = map(string)
+ default = null
+}
+
+variable "adb_config" {
+ description = "ADB configuration object"
+ type = object({
+ location = string
+ project = string
+ autonomous_database_id = string
+ database = string
+ display_name = string
+ admin_password = string
+
+ create_odb_network = bool
+ create_odb_network_subnets = bool
+ odb_network_project = optional(string)
+ vpc_network_name = optional(string)
+ odb_network_gcp_oracle_zone = optional(string)
+ client_subnet_cidr = optional(string)
+ odb_network_id = optional(string)
+ odb_client_subnet_id = optional(string)
+
+ compute_count = number
+ cpu_core_count = optional(number)
+ data_storage_size_tb = optional(number)
+ data_storage_size_gb = optional(number)
+ db_version = string
+ db_workload = string
+ db_edition = string
+ n_character_set = string
+ character_set = string
+ license_type = string
+
+ private_endpoint_ip = optional(string)
+ private_endpoint_label = optional(string)
+
+ is_auto_scaling_enabled = optional(bool, false)
+ is_storage_auto_scaling_enabled = optional(bool, false)
+
+ maintenance_schedule_type = optional(string)
+ mtls_connection_required = optional(string)
+ backup_retention_period_days = optional(number)
+ email = optional(string)
+ })
+
+}
+
+variable "odb_network_id" {
+ description = "ODB Network ID"
+ type = string
+ default = null
+}
+
+variable "odb_client_subnet_id" {
+ description = "ODB Client Subnet ID"
+ type = string
+ default = null
+}
diff --git a/cd3_automation_toolkit/gcpcloud/terraform/odb-network-gcp.tf b/cd3_automation_toolkit/gcpcloud/terraform/odb-network-gcp.tf
index 4ff725d33..bc7ff1431 100644
--- a/cd3_automation_toolkit/gcpcloud/terraform/odb-network-gcp.tf
+++ b/cd3_automation_toolkit/gcpcloud/terraform/odb-network-gcp.tf
@@ -16,10 +16,10 @@ module "gcp_network" {
odb_network_id = each.value.odb_network_id
odb_network_gcp_oracle_zone = each.value.odb_network_gcp_oracle_zone
odb_client_subnet_id = each.value.odb_client_subnet_id
- odb_backup_subnet_id = each.value.odb_backup_subnet_id
+ odb_backup_subnet_id = try(each.value.odb_backup_subnet_id,null)
create_odb_network = each.value.create_odb_network
create_odb_network_subnets = each.value.create_odb_network_subnets
client_subnet_cidr = each.value.client_subnet_cidr
- backup_subnet_cidr = each.value.backup_subnet_cidr
+ backup_subnet_cidr = try(each.value.backup_subnet_cidr, null)
}
\ No newline at end of file
diff --git a/cd3_automation_toolkit/gcpcloud/terraform/variables.tf b/cd3_automation_toolkit/gcpcloud/terraform/variables.tf
index 840c6af2e..9dfac15a1 100644
--- a/cd3_automation_toolkit/gcpcloud/terraform/variables.tf
+++ b/cd3_automation_toolkit/gcpcloud/terraform/variables.tf
@@ -8,6 +8,48 @@
#
############################
variable "gcp_oci_adb" {
+
+ description = "Map of ADB configurations."
+ type = map(object({
+ location = string
+ project = string
+ autonomous_database_id = string
+ database = string
+ display_name = string
+ admin_password = string
+
+ create_odb_network = optional(bool,false)
+ create_odb_network_subnets = optional(bool,false)
+ odb_network_project = optional(string)
+ vpc_network_name = optional(string)
+ odb_network_gcp_oracle_zone = optional(string)
+ client_subnet_cidr = optional(string)
+ odb_network_id = optional(string)
+ odb_client_subnet_id = optional(string)
+
+ compute_count = number
+ cpu_core_count = optional(number)
+ data_storage_size_tb = optional(number)
+ data_storage_size_gb = optional(number)
+ db_version = string
+ db_workload = string
+ db_edition = string
+ n_character_set = optional(string)
+ character_set = optional(string)
+ license_type = string
+
+ private_endpoint_ip = optional(string)
+ private_endpoint_label = optional(string)
+
+ is_auto_scaling_enabled = optional(bool, false)
+ is_storage_auto_scaling_enabled = optional(bool, false)
+
+ maintenance_schedule_type = optional(string)
+ mtls_connection_required = optional(string)
+ backup_retention_period_days = optional(number)
+ email = optional(string)
+ labels = optional(map(string), {})
+ }))
default = {}
}
@@ -107,7 +149,7 @@ variable "gcp_oci_exa_vmclusters" {
sparse_diskgroup_enabled = bool
# TAGS
- labels = map(string)
+ labels = optional(map(string), {})
}))
default = {}
}
diff --git a/cd3_automation_toolkit/ocicloud/python/Excel_Columns b/cd3_automation_toolkit/ocicloud/python/Excel_Columns
index c265eb283..9bc711a1f 100644
--- a/cd3_automation_toolkit/ocicloud/python/Excel_Columns
+++ b/cd3_automation_toolkit/ocicloud/python/Excel_Columns
@@ -171,7 +171,6 @@
"Cluster Name" : "name",
"Cluster Kubernetes Version" : "kubernetes_version",
"Network Type" : "",
- "Pod Security Policies Enforced" : "is_pod_security_policy_enabled",
"Load Balancer Subnets" : "service_lb_subnet_ids",
"API Endpoint Subnet" : "subnet_id",
"API Endpoint Pub Address" : "public_endpoint",
diff --git a/cd3_automation_toolkit/ocicloud/python/cd3Services.py b/cd3_automation_toolkit/ocicloud/python/cd3Services.py
index cc37af936..5b292e7d0 100644
--- a/cd3_automation_toolkit/ocicloud/python/cd3Services.py
+++ b/cd3_automation_toolkit/ocicloud/python/cd3Services.py
@@ -9,8 +9,9 @@
import shutil
import sys
class cd3Services():
-
+ global current_path
#Get OCI Cloud Regions
+ current_path = os.path.dirname(os.path.abspath(__file__))
regions_list = ""
def fetch_regions(self,config,signer):
#config = oci.config.from_file(file_location=configFileName)
@@ -61,7 +62,7 @@ def fetch_regions(self,config,signer):
line = cd3key + ":" + name
tempStr = tempStr + line + '\n'
- with open('/cd3user/oci_tools/cd3_automation_toolkit/ocicloud/python/OCI_Regions', 'w+') as f:
+ with open(current_path+'/OCI_Regions', 'w+') as f:
f.write(tempStr)
f.close()
print("Updated OCI_Regions file !!!\n")
diff --git a/cd3_automation_toolkit/ocicloud/python/compute/create_terraform_instances.py b/cd3_automation_toolkit/ocicloud/python/compute/create_terraform_instances.py
index 36d14b648..b0e5ceaf6 100755
--- a/cd3_automation_toolkit/ocicloud/python/compute/create_terraform_instances.py
+++ b/cd3_automation_toolkit/ocicloud/python/compute/create_terraform_instances.py
@@ -20,6 +20,147 @@
from jinja2 import Environment, FileSystemLoader
+
+
+def _secondary_vnic_column_key(columnname):
+ return commonTools.check_column_headers(str(columnname))
+
+
+def _get_secondary_vnic_column(values, columnname):
+ return values.get(commonTools.check_column_headers(columnname), "")
+
+def _split_secondary_vnic_values(columnvalue):
+ if columnvalue == "" or str(columnvalue).strip().lower() == "nan":
+ return []
+ return [str(value).strip() for value in str(columnvalue).split(";")]
+
+
+def _split_secondary_vnic_boolean_values(columnvalue):
+ # CD3 exports Excel booleans as TRUE/FALSE, but Terraform requires lowercase boolean literals.
+ return [
+ value.lower() if value.lower() in ("true", "false") else value
+ for value in _split_secondary_vnic_values(columnvalue)
+ ]
+
+
+def _parse_network_details(value, default_compartment_name, row_num, column_label):
+ if "ocid1.subnet.oc" in value:
+ return {
+ "network_compartment_id": "root",
+ "vcn_name": "",
+ "subnet_id": value,
+ }
+ if len(value.split("@")) == 2:
+ network_compartment_id = commonTools.check_tf_variable(value.split("@")[0].strip())
+ vcn_subnet_name = value.split("@")[1].strip()
+ else:
+ network_compartment_id = commonTools.check_tf_variable(default_compartment_name)
+ vcn_subnet_name = value
+ if "::" not in vcn_subnet_name:
+ print("Invalid " + column_label + " format specified for row " + str(row_num) + ". Exiting!!!")
+ exit(1)
+ return {
+ "network_compartment_id": network_compartment_id,
+ "vcn_name": vcn_subnet_name.split("::")[0].strip(),
+ "subnet_id": vcn_subnet_name.split("::")[1].strip(),
+ }
+
+
+def _parse_secondary_vnic_network_details(columnvalue, default_network_details, default_compartment_name, count, row_num):
+ values = _split_secondary_vnic_values(columnvalue)
+ network_details = []
+ for index in range(count):
+ value = values[index] if index < len(values) else ""
+ if value == "":
+ value = default_network_details
+ network_details.append(_parse_network_details(value, default_compartment_name, row_num, "Secondary VNIC Network Details"))
+ return network_details
+
+
+def _parse_secondary_vnic_nsgs(columnvalue):
+ nsg_values = []
+ for value in _split_secondary_vnic_values(columnvalue):
+ if value == "":
+ nsg_values.append([])
+ else:
+ nsg_values.append([nsg.strip() for nsg in value.split(",") if nsg.strip() != ""])
+ return nsg_values
+
+
+def _parse_secondary_vnic_tags(columnvalue):
+ tag_values = []
+ for value in _split_secondary_vnic_values(columnvalue):
+ if value == "":
+ tag_values.append([])
+ continue
+ tags = []
+ for tag_value in value.split(","):
+ if "=" in tag_value:
+ tags.append([tag_value.split("=", 1)[0].strip(), tag_value.split("=", 1)[1].strip()])
+ tag_values.append(tags)
+ return tag_values
+
+
+def _get_secondary_vnic_value(values, index, default=""):
+ return values[index] if index < len(values) else default
+
+
+def _build_secondary_vnics(df, row_index, dfcolumns, row_num):
+ secondary_vnic_columns = [column for column in dfcolumns if _secondary_vnic_column_key(column).startswith("secondary_vnic_")]
+ if len(secondary_vnic_columns) == 0:
+ return {}
+
+ values = {}
+ max_count = 0
+ for columnname in secondary_vnic_columns:
+ columnvalue = commonTools.check_columnvalue(str(df[columnname][row_index]).strip())
+ values[_secondary_vnic_column_key(columnname)] = columnvalue
+ max_count = max(max_count, len(_split_secondary_vnic_values(columnvalue)))
+
+ if max_count == 0:
+ return {}
+
+ default_compartment_name = str(df.loc[row_index, 'Compartment Name']).strip()
+ default_network_details = str(df.loc[row_index, 'Network Details']).strip()
+ networks = _parse_secondary_vnic_network_details(
+ _get_secondary_vnic_column(values, "Secondary VNIC Network Details"),
+ default_network_details,
+ default_compartment_name,
+ max_count,
+ row_num
+ )
+ secondary_vnics = {}
+
+ value_lists = {
+ "private_ip": _split_secondary_vnic_values(_get_secondary_vnic_column(values, "Secondary VNIC IP Addresses")),
+ "nsg_ids": _parse_secondary_vnic_nsgs(_get_secondary_vnic_column(values, "Secondary VNIC NSGs")),
+ "skip_source_dest_check": _split_secondary_vnic_boolean_values(_get_secondary_vnic_column(values, "Secondary VNIC Skip Source Dest Check")),
+ "assign_public_ip": _split_secondary_vnic_boolean_values(_get_secondary_vnic_column(values, "Secondary VNIC Pub Address")),
+ "display_name": _split_secondary_vnic_values(_get_secondary_vnic_column(values, "Secondary VNIC Display Names")),
+ "hostname_label": _split_secondary_vnic_values(_get_secondary_vnic_column(values, "Secondary VNIC Hostname Labels")),
+ "assign_private_dns_record": _split_secondary_vnic_boolean_values(_get_secondary_vnic_column(values, "Secondary VNIC Private DNS")),
+ "physical_nic_index": _split_secondary_vnic_values(_get_secondary_vnic_column(values, "Secondary VNIC Physical NIC Index")),
+ "defined_tags": _parse_secondary_vnic_tags(_get_secondary_vnic_column(values, "Secondary VNIC Defined Tags")),
+ "freeform_tags": _parse_secondary_vnic_tags(_get_secondary_vnic_column(values, "Secondary VNIC Freeform Tags")),
+ }
+
+ for index, network in enumerate(networks):
+ vnic = dict(network)
+ vnic["display_name"] = _get_secondary_vnic_value(value_lists["display_name"], index)
+ vnic["private_ip"] = _get_secondary_vnic_value(value_lists["private_ip"], index)
+ vnic["nsg_ids"] = _get_secondary_vnic_value(value_lists["nsg_ids"], index, [])
+ vnic["skip_source_dest_check"] = _get_secondary_vnic_value(value_lists["skip_source_dest_check"], index)
+ vnic["assign_public_ip"] = _get_secondary_vnic_value(value_lists["assign_public_ip"], index)
+ vnic["hostname_label"] = _get_secondary_vnic_value(value_lists["hostname_label"], index)
+ vnic["assign_private_dns_record"] = _get_secondary_vnic_value(value_lists["assign_private_dns_record"], index)
+ vnic["physical_nic_index"] = _get_secondary_vnic_value(value_lists["physical_nic_index"], index)
+ vnic["defined_tags"] = _get_secondary_vnic_value(value_lists["defined_tags"], index, [])
+ vnic["freeform_tags"] = _get_secondary_vnic_value(value_lists["freeform_tags"], index, [])
+ secondary_vnics["vnic_" + str(index + 1)] = vnic
+
+ return secondary_vnics
+
+
# If input is CD3 excel file
# Execution of the code begins here
def create_terraform_instances(inputfile, outdir, service_dir, prefix, ct):
@@ -87,6 +228,9 @@ def create_terraform_instances(inputfile, outdir, service_dir, prefix, ct):
# temporary dictionary1 and dictionary2
tempStr = {}
tempdict = {}
+ secondary_vnics = _build_secondary_vnics(df, i, dfcolumns, i + 3)
+ if secondary_vnics != {}:
+ tempStr.update({'secondary_vnics': secondary_vnics})
# Check if values are entered for mandatory fields
if (str(df.loc[i, 'Region']).lower() == 'nan' or str(df.loc[i, 'Display Name']).lower() == 'nan' or str(
@@ -104,6 +248,8 @@ def create_terraform_instances(inputfile, outdir, service_dir, prefix, ct):
source_details=[]
for columnname in dfcolumns:
+ if _secondary_vnic_column_key(columnname).startswith("secondary_vnic_"):
+ continue
# Column value
columnvalue = str(df[columnname][i]).strip()
diff --git a/cd3_automation_toolkit/ocicloud/python/compute/export_instances_nonGreenField.py b/cd3_automation_toolkit/ocicloud/python/compute/export_instances_nonGreenField.py
index 74502df66..8553ec730 100644
--- a/cd3_automation_toolkit/ocicloud/python/compute/export_instances_nonGreenField.py
+++ b/cd3_automation_toolkit/ocicloud/python/compute/export_instances_nonGreenField.py
@@ -15,7 +15,7 @@
def adding_columns_values(region, ad, fd, vs, publicip, privateip, os_dname, shape, key_name, c_name,
bkp_policy_name, nsgs, d_host, instance_data, values_for_column_instances, bdet,
cpcn, shape_config, vnic_info, vnic_defined_tags, vnic_freeform_tags, launch_options,avail_config,ins_options,
- platform_config, plugin_config):
+ platform_config, plugin_config, secondary_vnics):
for col_header in values_for_column_instances.keys():
if (col_header == "Region"):
values_for_column_instances[col_header].append(region)
@@ -53,6 +53,29 @@ def adding_columns_values(region, ad, fd, vs, publicip, privateip, os_dname, sha
elif (col_header == "VNIC Display Name"):
values_for_column_instances[col_header].append(vnic_info.display_name)
+ elif (col_header == "Secondary VNIC Network Details"):
+ values_for_column_instances[col_header].append(secondary_vnics.get("network_details", ""))
+ elif (col_header == "Secondary VNIC IP Addresses"):
+ values_for_column_instances[col_header].append(secondary_vnics.get("ip_addresses", ""))
+ elif (col_header == "Secondary VNIC NSGs"):
+ values_for_column_instances[col_header].append(secondary_vnics.get("nsgs", ""))
+ elif (col_header == "Secondary VNIC Skip Source Dest Check"):
+ values_for_column_instances[col_header].append(secondary_vnics.get("skip_source_dest_check", ""))
+ elif (col_header == "Secondary VNIC Pub Address"):
+ values_for_column_instances[col_header].append(secondary_vnics.get("pub_address", ""))
+ elif (col_header == "Secondary VNIC Display Names"):
+ values_for_column_instances[col_header].append(secondary_vnics.get("display_names", ""))
+ elif (col_header == "Secondary VNIC Hostname Labels"):
+ values_for_column_instances[col_header].append(secondary_vnics.get("hostname_labels", ""))
+ elif (col_header == "Secondary VNIC Private DNS"):
+ values_for_column_instances[col_header].append(secondary_vnics.get("private_dns", ""))
+ elif (col_header == "Secondary VNIC Physical NIC Index"):
+ values_for_column_instances[col_header].append(secondary_vnics.get("physical_nic_index", ""))
+ elif (col_header == "Secondary VNIC Defined Tags"):
+ values_for_column_instances[col_header].append(secondary_vnics.get("defined_tags", ""))
+ elif (col_header == "Secondary VNIC Freeform Tags"):
+ values_for_column_instances[col_header].append(secondary_vnics.get("freeform_tags", ""))
+
elif(col_header.lower().startswith("plugin")):
col_temp = col_header.lower().replace("plugin","")
col_temp=col_temp.strip()
@@ -86,6 +109,11 @@ def find_vnic(ins_id, compartment_id):
return net
+def _secondary_vnic_sort_key(vnic_attachment):
+ """Keep Excel and Terraform VNIC order stable across OCI API responses."""
+ return str(getattr(vnic_attachment, "time_created", "")), vnic_attachment.id
+
+
def __get_instances_info(compartment_name, compartment_id, reg_name, display_names, ad_names, export_tags, ct,state):
config.__setitem__("region", ct.region_dict[reg_name])
compute = oci.core.ComputeClient(config=config, retry_strategy=oci.retry.DEFAULT_RETRY_STRATEGY,signer=signer)
@@ -200,11 +228,34 @@ def __get_instances_info(compartment_name, compartment_id, reg_name, display_nam
# VNIC Details
ins_vnic = find_vnic(ins_id, compartment_id)
vnic_info=None
+ secondary_vnics = {
+ "network_details": [],
+ "ip_addresses": [],
+ "nsgs": [],
+ "skip_source_dest_check": [],
+ "pub_address": [],
+ "display_names": [],
+ "hostname_labels": [],
+ "private_dns": [],
+ "physical_nic_index": [],
+ "defined_tags": [],
+ "freeform_tags": [],
+ }
+ vnic_details = []
for lnic in ins_vnic.data:
+ vnic_details.append((lnic, network.get_vnic(vnic_id=lnic.vnic_id).data))
+
+ primary_vnic_details = [details for details in vnic_details if details[1].is_primary]
+ secondary_vnic_details = sorted(
+ (details for details in vnic_details if not details[1].is_primary),
+ key=lambda details: _secondary_vnic_sort_key(details[0]),
+ )
+ ordered_vnic_details = secondary_vnic_details + primary_vnic_details
+ secondary_vnic_index = 0
+ for lnic, current_vnic_info in ordered_vnic_details:
# print(lnic)
subnet_id = lnic.subnet_id
- vnic_id = lnic.vnic_id
- vnic_info = network.get_vnic(vnic_id=vnic_id).data
+ vnic_info = current_vnic_info
# print(vnic_info)
vnic_defined_tags = ""
@@ -220,6 +271,43 @@ def __get_instances_info(compartment_name, compartment_id, reg_name, display_nam
#Get only primary VNIC details
if (vnic_info.is_primary == False):
+ secondary_vnic_index += 1
+ tf_resource = f'module.instances[\\"{tf_name}\\"].oci_core_vnic_attachment.secondary_vnic_attachment[\\"vnic_{secondary_vnic_index}\\"]'
+ if tf_resource not in state["resources"]:
+ importCommands[reg_name] += f'\n{tf_or_tofu} import "{tf_resource}" {str(lnic.id)}'
+
+ subnet_info = network.get_subnet(subnet_id=subnet_id)
+ subnet_name = subnet_info.data.display_name
+ vcn_info = network.get_vcn(vcn_id=subnet_info.data.vcn_id)
+ network_compartment_name = compartment_name
+ for comp_name, comp_id in ct.ntk_compartment_ids.items():
+ if comp_id == vcn_info.data.compartment_id:
+ network_compartment_name = comp_name
+ secondary_vnics["network_details"].append(network_compartment_name + "@" + vcn_info.data.display_name + "::" + subnet_name)
+ secondary_vnics["ip_addresses"].append(vnic_info.private_ip or "")
+ secondary_vnics["skip_source_dest_check"].append(str(vnic_info.skip_source_dest_check).upper() if vnic_info.skip_source_dest_check is not None else "")
+ secondary_vnics["pub_address"].append("TRUE" if vnic_info.public_ip is not None else "FALSE")
+ secondary_vnics["display_names"].append(vnic_info.display_name or "")
+ secondary_vnics["hostname_labels"].append(vnic_info.hostname_label or "")
+ secondary_vnics["private_dns"].append(str(vnic_info.is_dns_record_enabled).upper() if getattr(vnic_info, "is_dns_record_enabled", None) is not None else "")
+ secondary_vnics["physical_nic_index"].append(str(lnic.nic_index) if getattr(lnic, "nic_index", None) is not None else "")
+
+ secondary_nsg_names = []
+ for nsg_id in vnic_info.nsg_ids:
+ nsg_info = network.get_network_security_group(nsg_id)
+ secondary_nsg_names.append(nsg_info.data.display_name)
+ secondary_vnics["nsgs"].append(",".join(secondary_nsg_names))
+
+ secondary_defined_tags = []
+ for namespace, tagkey in vnic_info.defined_tags.items():
+ for tag, value in tagkey.items():
+ secondary_defined_tags.append(namespace + "." + tag + "=" + value)
+ secondary_vnics["defined_tags"].append(",".join(secondary_defined_tags))
+
+ secondary_freeform_tags = []
+ for tag, value in vnic_info.freeform_tags.items():
+ secondary_freeform_tags.append(tag + "=" + value)
+ secondary_vnics["freeform_tags"].append(",".join(secondary_freeform_tags))
continue
subnet_info = network.get_subnet(subnet_id=subnet_id)
subnet_name = subnet_info.data.display_name
@@ -297,7 +385,7 @@ def __get_instances_info(compartment_name, compartment_id, reg_name, display_nam
ins_shape, key_name, compartment_name, bkp_policy_name, nsg_names, dedicated_host,
ins, values_for_column_instances, bdet, cpcn, shape_config, vnic_info,
vnic_defined_tags, vnic_freeform_tags, launch_options,avail_config,ins_options,
- platform_data, plugin_config)
+ platform_data, plugin_config, {key: ';'.join(value) for key, value in secondary_vnics.items()})
# Execution of the code begins here
@@ -416,4 +504,3 @@ def export_instances(inputfile, outdir, service_dir,config1, signer1, ct, export
commonTools.write_to_cd3(values_for_column_instances, cd3file, "Instances")
print("{0} Instance Details exported into CD3.\n".format(len(values_for_column_instances["Region"])))
-
diff --git a/cd3_automation_toolkit/ocicloud/python/compute/templates/instances-template b/cd3_automation_toolkit/ocicloud/python/compute/templates/instances-template
index 0768c88f9..0c5cc6f32 100644
--- a/cd3_automation_toolkit/ocicloud/python/compute/templates/instances-template
+++ b/cd3_automation_toolkit/ocicloud/python/compute/templates/instances-template
@@ -344,6 +344,57 @@ instances = {
vnic_display_name = "{{ vnic_display_name }}"
{% endif %}
+
+ {% if secondary_vnics %}
+ secondary_vnics = {
+ {% for vnic_name, vnic in secondary_vnics.items() %}
+ {{ vnic_name }} = {
+ network_compartment_id = "{{ vnic.network_compartment_id }}"
+ vcn_name = "{{ vnic.vcn_name }}"
+ subnet_id = "{{ vnic.subnet_id }}"
+ {% if vnic.display_name %}
+ display_name = "{{ vnic.display_name }}"
+ {% endif %}
+ {% if vnic.private_ip %}
+ private_ip = "{{ vnic.private_ip }}"
+ {% endif %}
+ {% if vnic.assign_public_ip %}
+ assign_public_ip = {{ vnic.assign_public_ip }}
+ {% endif %}
+ {% if vnic.hostname_label %}
+ hostname_label = "{{ vnic.hostname_label }}"
+ {% endif %}
+ {% if vnic.assign_private_dns_record %}
+ assign_private_dns_record = {{ vnic.assign_private_dns_record }}
+ {% endif %}
+ {% if vnic.skip_source_dest_check %}
+ skip_source_dest_check = {{ vnic.skip_source_dest_check }}
+ {% endif %}
+ {% if vnic.physical_nic_index %}
+ physical_nic_index = {{ vnic.physical_nic_index }}
+ {% endif %}
+ {% if vnic.nsg_ids %}
+ nsg_ids = [{% for nsg in vnic.nsg_ids %}"{{ nsg }}"{% if not loop.last %}, {% endif %}{% endfor %}]
+ {% endif %}
+ {% if vnic.defined_tags %}
+ defined_tags = {
+ {% for tags in vnic.defined_tags %}
+ "{{ tags[0] }}" = "{{ tags[1] }}"
+ {% endfor %}
+ }
+ {% endif %}
+ {% if vnic.freeform_tags %}
+ freeform_tags = {
+ {% for tags in vnic.freeform_tags %}
+ "{{ tags[0] }}" = "{{ tags[1] }}"
+ {% endfor %}
+ }
+ {% endif %}
+ }
+ {% endfor %}
+ }
+ {% endif %}
+
## Section for adding VNIC Defined and Freeform Tags
{% if vnic_defined_tags and vnic_defined_tags != 'nan' and vnic_defined_tags != '' %}
{% if vnic_defined_tags[0] %}
diff --git a/cd3_automation_toolkit/ocicloud/python/database/create_terraform_adb.py b/cd3_automation_toolkit/ocicloud/python/database/create_terraform_adb.py
index d2d067d06..74da666b3 100644
--- a/cd3_automation_toolkit/ocicloud/python/database/create_terraform_adb.py
+++ b/cd3_automation_toolkit/ocicloud/python/database/create_terraform_adb.py
@@ -133,13 +133,9 @@ def create_terraform_adb(inputfile, outdir, service_dir, prefix, ct):
if columnname == 'Database Workload':
columnvalue = columnvalue.strip()
- autonomous_value = commonTools.check_tf_variable(columnvalue).lower()
+ autonomous_value = commonTools.check_tf_variable(columnvalue)
tempdict = {'autonomous_value': autonomous_value}
- if columnname == 'Database Workload':
- columnvalue = columnvalue.strip()
- autonomous_value = commonTools.check_tf_variable(columnvalue).lower()
- tempdict = {'autonomous_value': autonomous_value}
if columnname == 'Customer Contacts':
columnvalue = columnvalue.strip().split(",")
diff --git a/cd3_automation_toolkit/ocicloud/python/database/export_exa_infra_cdb_nonGreenField.py b/cd3_automation_toolkit/ocicloud/python/database/export_exa_infra_cdb_nonGreenField.py
index 479140b46..d8d66c289 100644
--- a/cd3_automation_toolkit/ocicloud/python/database/export_exa_infra_cdb_nonGreenField.py
+++ b/cd3_automation_toolkit/ocicloud/python/database/export_exa_infra_cdb_nonGreenField.py
@@ -212,7 +212,11 @@ def export_exa_cdb(inputfile, outdir, service_dir, config, signer, ct, export_co
if exa_dbhome.vm_cluster_id:
exa_cdbs = oci.pagination.list_call_get_all_results(db_client.list_databases,compartment_id=ct.ntk_compartment_ids[ntk_compartment_name],db_home_id=exa_dbhome.id, lifecycle_state="AVAILABLE")
for exa_cdb in exa_cdbs.data:
- vm_cluster = db_client.get_cloud_vm_cluster(cloud_vm_cluster_id=exa_cdb.vm_cluster_id)
+ try:
+ vm_cluster = db_client.get_cloud_vm_cluster(cloud_vm_cluster_id=exa_cdb.vm_cluster_id)
+ except Exception as e:
+ print("error with db home " + exa_dbhome.display_name)
+
exadata_infrastructure = db_client.get_cloud_exadata_infrastructure(cloud_exadata_infrastructure_id=vm_cluster.data.cloud_exadata_infrastructure_id)
exa_infra_compartment_name = next((name for name, ocid in ct.ntk_compartment_ids.items() if
ocid == exadata_infrastructure.data.compartment_id), None)
@@ -233,6 +237,8 @@ def export_exa_cdb(inputfile, outdir, service_dir, config, signer, ct, export_co
if check == False:
continue
+ print(exa_cdb)
+ print("------")
print_exa_cdb(region,exadata_infrastructure,exa_infra_compartment_name,vm_cluster,exa_dbhome, exa_cdb,values_for_column, ntk_compartment_name,state)
commonTools.write_to_cd3(values_for_column, cd3file, sheetName)
diff --git a/cd3_automation_toolkit/ocicloud/python/database/export_exa_infra_dbhome_nonGreenField.py b/cd3_automation_toolkit/ocicloud/python/database/export_exa_infra_dbhome_nonGreenField.py
index c25b4f55b..079e01622 100644
--- a/cd3_automation_toolkit/ocicloud/python/database/export_exa_infra_dbhome_nonGreenField.py
+++ b/cd3_automation_toolkit/ocicloud/python/database/export_exa_infra_dbhome_nonGreenField.py
@@ -99,12 +99,16 @@ def export_exa_dbhome(inputfile, outdir, service_dir, config, signer, ct, export
region = reg.capitalize()
db_client = oci.database.DatabaseClient(config=config,retry_strategy=oci.retry.DEFAULT_RETRY_STRATEGY, signer=signer)
-
for ntk_compartment_name in export_compartments:
exa_dbhomes = oci.pagination.list_call_get_all_results(db_client.list_db_homes,compartment_id=ct.ntk_compartment_ids[ntk_compartment_name], lifecycle_state="AVAILABLE")
for exa_dbhome in exa_dbhomes.data:
if exa_dbhome.vm_cluster_id:
- vm_cluster = db_client.get_cloud_vm_cluster(cloud_vm_cluster_id=exa_dbhome.vm_cluster_id)
+ print(exa_dbhome.display_name)
+ try:
+ vm_cluster = db_client.get_cloud_vm_cluster(cloud_vm_cluster_id=exa_dbhome.vm_cluster_id)
+ except Exception as e:
+ print("error with db home "+exa_dbhome.display_name)
+
exadata_infrastructure = db_client.get_cloud_exadata_infrastructure(cloud_exadata_infrastructure_id=vm_cluster.data.cloud_exadata_infrastructure_id)
#print("exa_data",exadata_infrastructure.data)
#print("comp_data",ct.ntk_compartment_ids)
diff --git a/cd3_automation_toolkit/ocicloud/python/database/export_exa_infra_pdb_nonGreenField.py b/cd3_automation_toolkit/ocicloud/python/database/export_exa_infra_pdb_nonGreenField.py
index bdf7efacf..d9252f1ed 100644
--- a/cd3_automation_toolkit/ocicloud/python/database/export_exa_infra_pdb_nonGreenField.py
+++ b/cd3_automation_toolkit/ocicloud/python/database/export_exa_infra_pdb_nonGreenField.py
@@ -110,7 +110,11 @@ def export_exa_pdb(inputfile, outdir, service_dir, config, signer, ct, export_co
if exa_dbhome.vm_cluster_id:
exa_cdbs = oci.pagination.list_call_get_all_results(db_client.list_databases,compartment_id=ct.ntk_compartment_ids[ntk_compartment_name],db_home_id=exa_dbhome.id, lifecycle_state="AVAILABLE")
for exa_cdb in exa_cdbs.data:
- vm_cluster = db_client.get_cloud_vm_cluster(cloud_vm_cluster_id=exa_cdb.vm_cluster_id)
+ try:
+ vm_cluster = db_client.get_cloud_vm_cluster(cloud_vm_cluster_id=exa_cdb.vm_cluster_id)
+ except Exception as e:
+ print("error with db home " + exa_dbhome.display_name)
+
exadata_infrastructure = db_client.get_cloud_exadata_infrastructure(cloud_exadata_infrastructure_id=vm_cluster.data.cloud_exadata_infrastructure_id)
exa_infra_compartment_name = next((name for name, ocid in ct.ntk_compartment_ids.items() if
ocid == exadata_infrastructure.data.compartment_id), None)
diff --git a/cd3_automation_toolkit/ocicloud/python/database/templates/adb-template b/cd3_automation_toolkit/ocicloud/python/database/templates/adb-template
index fdfc67c47..8f118121b 100644
--- a/cd3_automation_toolkit/ocicloud/python/database/templates/adb-template
+++ b/cd3_automation_toolkit/ocicloud/python/database/templates/adb-template
@@ -94,13 +94,13 @@ adb = {
db_version = "{{ db_version.strip() }}"
{% endif %}
- {% if autonomous_value == 'adw' %}
+ {% if autonomous_value.lower() == 'adw' %}
db_workload = "DW"
- {% elif autonomous_value == 'atp' %}
+ {% elif autonomous_value.lower() == 'atp' %}
db_workload = "OLTP"
- {% elif autonomous_value == 'json' %}
+ {% elif autonomous_value.lower() == 'json' %}
db_workload = "AJD"
- {% elif autonomous_value == 'apex' %}
+ {% elif autonomous_value.lower() == 'apex' %}
db_workload = "APEX"
{% else %}
db_workload = "{{ autonomous_value }}"
diff --git a/cd3_automation_toolkit/ocicloud/python/developerservices/oke/create_terraform_oke.py b/cd3_automation_toolkit/ocicloud/python/developerservices/oke/create_terraform_oke.py
index 3dbc26992..d4383d069 100644
--- a/cd3_automation_toolkit/ocicloud/python/developerservices/oke/create_terraform_oke.py
+++ b/cd3_automation_toolkit/ocicloud/python/developerservices/oke/create_terraform_oke.py
@@ -21,6 +21,112 @@
from jinja2 import Environment, FileSystemLoader
+def _normalize_node_capacity_type(value, row_num):
+ value = str(value or "").strip()
+ value_l = value.lower()
+ if value_l in ("", "nan", "none", "on-demand", "on_demand", "on demand", "ondemand"):
+ return "ON_DEMAND"
+ if value_l in ("preemptible", "preemptible_capacity", "preemptible capacity"):
+ return "PREEMPTIBLE"
+ if value_l in ("capacity_reservation", "capacity reservation", "capacityreservation"):
+ return "CAPACITY_RESERVATION"
+ print(
+ "\nNode Capacity Type must be ON_DEMAND, PREEMPTIBLE, or CAPACITY_RESERVATION. "
+ "Please fix it for row : {} and try again.".format(row_num)
+ )
+ print("\n** Exiting **")
+ exit(1)
+
+
+def _normalize_preemptible_preserve_boot_volume(value, row_num):
+ value = str(value or "").strip().lower()
+ if value in ("", "nan", "none", "false", "f", "no", "n", "0"):
+ return "false"
+ if value in ("true", "t", "yes", "y", "1"):
+ return "true"
+ print(
+ "\nPreserve_Boot_Volume in Node Capacity Type must be TRUE or FALSE. "
+ "Please fix it for row : {} and try again.".format(row_num)
+ )
+ print("\n** Exiting **")
+ exit(1)
+
+
+def _parse_node_capacity_config(value, row_num):
+ value = str(value or "").strip()
+ if value.lower() in ("", "nan", "none"):
+ return "ON_DEMAND", "false", ""
+
+ parts = [part.strip() for part in value.split(":")]
+ if len(parts) > 2:
+ print(
+ "\nNode Capacity Type format must be ON_DEMAND, PREEMPTIBLE:, "
+ "or CAPACITY_RESERVATION:. Please fix it for row : {} and try again.".format(row_num)
+ )
+ print("\n** Exiting **")
+ exit(1)
+
+ node_capacity_type = _normalize_node_capacity_type(parts[0], row_num)
+ preserve_boot_volume = "false"
+ capacity_reservation_id = ""
+ if len(parts) == 2:
+ if node_capacity_type == "PREEMPTIBLE":
+ preserve_boot_volume = _normalize_preemptible_preserve_boot_volume(parts[1], row_num)
+ elif node_capacity_type == "CAPACITY_RESERVATION":
+ capacity_reservation_id = parts[1]
+ if not capacity_reservation_id.lower().startswith("ocid1.capacityreservation."):
+ print(
+ "\nCAPACITY_RESERVATION in Node Capacity Type requires a valid capacity reservation OCID. "
+ "Please fix it for row : {} and try again.".format(row_num)
+ )
+ print("\n** Exiting **")
+ exit(1)
+ else:
+ print(
+ "\nON_DEMAND does not accept an additional value in Node Capacity Type. "
+ "Please fix it for row : {} and try again.".format(row_num)
+ )
+ print("\n** Exiting **")
+ exit(1)
+ elif node_capacity_type == "CAPACITY_RESERVATION":
+ print(
+ "\nCAPACITY_RESERVATION in Node Capacity Type requires a capacity reservation OCID. "
+ "Please fix it for row : {} and try again.".format(row_num)
+ )
+ print("\n** Exiting **")
+ exit(1)
+
+ return node_capacity_type, preserve_boot_volume, capacity_reservation_id
+
+
+def _get_node_capacity_configs(value, placement_count, row_num):
+ """Return one normalized capacity configuration for each placement."""
+ value = str(value or "").strip()
+ if value.lower() in ("", "nan", "none"):
+ configs = [""]
+ else:
+ configs = [config.strip() for config in value.split(";")]
+ if any(not config for config in configs):
+ print(
+ "\nNode Capacity Type contains an empty semicolon-separated value. "
+ "Please fix it for row : {} and try again.".format(row_num)
+ )
+ print("\n** Exiting **")
+ exit(1)
+
+ if len(configs) == 1:
+ configs *= placement_count
+ elif len(configs) != placement_count:
+ print(
+ "\nNode Capacity Type must contain one value or the same number of semicolon-separated "
+ "values as Node Placement Configs. Please fix it for row : {} and try again.".format(row_num)
+ )
+ print("\n** Exiting **")
+ exit(1)
+
+ return [_parse_node_capacity_config(config, row_num) for config in configs]
+
+
######
# Required Inputs-CD3 excel file, Config file AND outdir
######
@@ -48,15 +154,16 @@ def create_terraform_oke(inputfile, outdir, service_dir, prefix, ct):
df = df.dropna(how='all')
df = df.reset_index(drop=True)
+ # Kubernetes PodSecurityPolicy is deprecated and intentionally removed from OKE generation.
# fill the empty values with that in previous row.
dffill = df[
- ['Region', 'Compartment Name', 'Cluster Name', 'Cluster Kubernetes Version','Network Type', 'Pod Security Policies Enforced',
+ ['Region', 'Compartment Name', 'Cluster Name', 'Cluster Kubernetes Version','Network Type',
'Load Balancer Network Details', 'API Endpoint Network Details']]
- dffill = dffill.fillna(method='ffill')
+ dffill = dffill.ffill()
#Drop unnecessary columns
dfdrop = df[
- ['Region', 'Compartment Name', 'Cluster Name', 'Cluster Kubernetes Version','Network Type', 'Pod Security Policies Enforced',
+ ['Region', 'Compartment Name', 'Cluster Name', 'Cluster Kubernetes Version','Network Type',
'Load Balancer Network Details', 'API Endpoint Network Details']]
dfdrop = df.drop(dfdrop, axis=1)
df = pd.concat([dffill, dfdrop], axis=1)
@@ -105,11 +212,10 @@ def create_terraform_oke(inputfile, outdir, service_dir, prefix, ct):
str(df.loc[i, 'Cluster Name']).lower() == 'nan' or \
str(df.loc[i, 'Network Type']).lower() == 'nan' or \
str(df.loc[i, 'Cluster Kubernetes Version']).lower() == 'nan' or \
- str(df.loc[i, 'Pod Security Policies Enforced']).lower() == 'nan' or \
str(df.loc[i, 'Load Balancer Network Details']).lower() == 'nan' or \
str(df.loc[i, 'API Endpoint Network Details']).lower() == 'nan':
print(
- "\nRegion, Compartment Name, Cluster Name, Network Type, Cluster Kubernetes Version, Pod Security Policies, Load Balancer Network Details, API Endpoint Network Details fields are mandatory. Please enter a value and try again !!\n\nPlease fix it for row : {}".format(
+ "\nRegion, Compartment Name, Cluster Name, Network Type, Cluster Kubernetes Version, Load Balancer Network Details, API Endpoint Network Details fields are mandatory. Please enter a value and try again !!\n\nPlease fix it for row : {}".format(
i + 3))
print("\n** Exiting **")
exit(1)
@@ -181,15 +287,29 @@ def create_terraform_oke(inputfile, outdir, service_dir, prefix, ct):
if columnname == 'Node Placement Configs':
if columnvalue.lower() != 'nan' and columnvalue.lower() != '':
- configs=columnvalue.split(";")
+ row_node_capacity_config = df.loc[i, 'Node Capacity Type'] if 'Node Capacity Type' in dfcolumns else ''
+ configs = [config.strip() for config in columnvalue.split(";")]
+ if any(not config for config in configs):
+ print(
+ "Node Placement Configs contains an empty semicolon-separated value in row "
+ + str(i + 3) + ". Exiting!!!"
+ )
+ exit(1)
+ capacity_configs = _get_node_capacity_configs(
+ row_node_capacity_config, len(configs), i + 3
+ )
j=0
placement_configs = {}
for config in configs:
- if config == "":
- break
j=j+1
#split on : and preserve ::
config_data = re.split(r'(?::"
+ " in row " + str(i + 3) + ". Exiting!!!"
+ )
+ exit(1)
network_details = config_data[0]
if ("ocid1.subnet.oc" in network_details):
@@ -218,13 +338,12 @@ def create_terraform_oke(inputfile, outdir, service_dir, prefix, ct):
fd=','.join(['"' + x.upper() + '"' for x in FD.split(',')])
else:
fd=""
- cr=""
- if (len(config_data)==4):
- cr = str(config_data[3]).lower()
- if cr =='none':
- cr=""
+ node_capacity_type, preserve_boot_volume, cr = capacity_configs[j - 1]
- placement_configs["node_"+str(j)]=[network_compartment_id, vcn_name, subnet_id, ad, fd,cr]
+ placement_configs["node_"+str(j)] = [
+ network_compartment_id, vcn_name, subnet_id, ad, fd, cr,
+ node_capacity_type, preserve_boot_volume
+ ]
tempdict = {'placement_configs' : placement_configs}
tempStr.update(tempdict)
if columnname == "Compartment Name":
diff --git a/cd3_automation_toolkit/ocicloud/python/developerservices/oke/export_oke_nonGreenField.py b/cd3_automation_toolkit/ocicloud/python/developerservices/oke/export_oke_nonGreenField.py
index d9bc96f4d..435d22b1c 100644
--- a/cd3_automation_toolkit/ocicloud/python/developerservices/oke/export_oke_nonGreenField.py
+++ b/cd3_automation_toolkit/ocicloud/python/developerservices/oke/export_oke_nonGreenField.py
@@ -21,6 +21,32 @@
sys.path.append(os.getcwd() + "/..")
+def _is_preemptible_config(config):
+ return getattr(config, "preemptible_node_config", None) is not None
+
+
+def _preserve_boot_volume_on_preemption(config):
+ preemptible_config = getattr(config, "preemptible_node_config", None)
+ preemption_action = getattr(preemptible_config, "preemption_action", None)
+ return str(getattr(preemption_action, "is_preserve_boot_volume", False)).lower()
+
+
+def _node_capacity_config_cell_value(configs):
+ values = []
+ for config in configs or []:
+ if _is_preemptible_config(config):
+ values.append("PREEMPTIBLE:" + _preserve_boot_volume_on_preemption(config))
+ elif getattr(config, "capacity_reservation_id", None):
+ values.append("CAPACITY_RESERVATION:" + config.capacity_reservation_id)
+ else:
+ values.append("ON_DEMAND")
+ if not values:
+ return None
+ if len(set(values)) == 1:
+ return values[0]
+ return ";".join(values)
+
+
def print_oke(values_for_column_oke, reg, compartment_name, compartment_name_nodepool,nodepool_count, nodepool_info,cluster_info,network,nodepool_type,ct):
image_policy_config = cluster_info.image_policy_config
for col_header in values_for_column_oke.keys():
@@ -58,12 +84,6 @@ def print_oke(values_for_column_oke, reg, compartment_name, compartment_name_nod
values_for_column_oke[col_header].append("FLANNEL_OVERLAY")
else:
values_for_column_oke[col_header].append(None)
- elif col_header == 'Pod Security Policies Enforced':
- if nodepool_count <= 1:
- values_for_column_oke[col_header].append(
- cluster_info.options.admission_controller_options.is_pod_security_policy_enabled)
- else:
- values_for_column_oke[col_header].append(None)
elif col_header == 'Load Balancer Network Details':
if nodepool_count <=1:
subnets = []
@@ -212,7 +232,7 @@ def print_oke(values_for_column_oke, reg, compartment_name, compartment_name_nod
elif (col_header == "Node Placement Configs"):
if (nodepool_info != None):
- excel_data = ""
+ placement_values = []
if (nodepool_type=='managed'):
configs = nodepool_info.node_config_details.placement_configs
elif (nodepool_type == 'virtual'):
@@ -239,25 +259,30 @@ def print_oke(values_for_column_oke, reg, compartment_name, compartment_name_nod
ad = AD(okead)
if (nodepool_type == 'managed'):
fd=config.fault_domains
- cr = config.capacity_reservation_id
- if cr == None:
- cr="None"
elif (nodepool_type == 'virtual'):
fd = config.fault_domain
- cr="None"
+ if fd==[]:
+ fd=None
if fd != None:
fd_s = ','.join(str(x) for x in fd)
okefd = fd_s
else:
okefd = "None"
- excel_data=excel_data+nw+":"+str(ad)+":"+okefd+":"+cr+";"
+ placement_values.append(nw+":"+str(ad)+":"+okefd)
- values_for_column_oke[col_header].append(excel_data)
+ values_for_column_oke[col_header].append(";".join(placement_values))
else:
values_for_column_oke[col_header].append(None)
+ elif (col_header == "Node Capacity Type"):
+ if (nodepool_info != None and nodepool_type == 'managed'):
+ configs = nodepool_info.node_config_details.placement_configs or []
+ values_for_column_oke[col_header].append(_node_capacity_config_cell_value(configs))
+ else:
+ values_for_column_oke[col_header].append(None)
+
elif (col_header == "Max Pods Per Node"):
if (nodepool_info != None and nodepool_type=='managed'):
if nodepool_info.node_config_details.node_pool_pod_network_option_details.cni_type == "OCI_VCN_IP_NATIVE":
@@ -606,6 +631,7 @@ def export_oke(inputfile, outdir,service_dir, config, signer, ct, export_compart
mykeys = {}
for keys, values in tempsshDict.items():
reg_name = keys.split("::")[0]
+ values = values.replace("\n", "\\n")
if (reg == reg_name):
key_name = "\"" + keys[len(reg_name) + 2:] + "\""
mykeys[key_name] = values
@@ -656,4 +682,3 @@ def export_oke(inputfile, outdir,service_dir, config, signer, ct, export_compart
commonTools.write_to_cd3(values_for_column_oke, cd3file, "OKE")
print("{0} OKE clusters exported into CD3.\n".format(total_resources))
-
diff --git a/cd3_automation_toolkit/ocicloud/python/developerservices/oke/templates/cluster-template b/cd3_automation_toolkit/ocicloud/python/developerservices/oke/templates/cluster-template
index 34f9079de..5a96b2988 100644
--- a/cd3_automation_toolkit/ocicloud/python/developerservices/oke/templates/cluster-template
+++ b/cd3_automation_toolkit/ocicloud/python/developerservices/oke/templates/cluster-template
@@ -36,12 +36,6 @@ clusters = {
nsg_ids = [{{ api_endpoint_nsgs }}]
{% endif %}
endpoint_subnet_id = "{{ api_endpoint_subnet }}"
- {% if pod_security_policies_enforced == 'true' %}
- is_pod_security_policy_enabled = "true"
- {% else %}
- is_pod_security_policy_enabled = "false"
- {% endif %}
-
{% if is_policy_enabled == 'true' %}
is_policy_enabled = "true"
{% endif %}
@@ -162,4 +156,4 @@ clusters = {
},
-{% endif %}
\ No newline at end of file
+{% endif %}
diff --git a/cd3_automation_toolkit/ocicloud/python/developerservices/oke/templates/nodepool-template b/cd3_automation_toolkit/ocicloud/python/developerservices/oke/templates/nodepool-template
index a4b24a714..f6762be25 100644
--- a/cd3_automation_toolkit/ocicloud/python/developerservices/oke/templates/nodepool-template
+++ b/cd3_automation_toolkit/ocicloud/python/developerservices/oke/templates/nodepool-template
@@ -57,8 +57,18 @@ nodepools = {
{% if value[4] %}
fault_domains = [{{ value[4] }}]
{% endif %}
- {% if value[5] %}
- capacity_reservation_id = {{ value[5] }}
+ {% if value[6] == "ON_DEMAND" %}
+ capacity_reservation_id = ""
+ {% elif value[6] == "CAPACITY_RESERVATION" %}
+ capacity_reservation_id = "{{ value[5] }}"
+ {% endif %}
+ {% if value[6] and value[6] == "PREEMPTIBLE" %}
+ preemptible_node_config = {
+ preemption_action = {
+ type = "TERMINATE"
+ is_preserve_boot_volume = {{ value[7] }}
+ }
+ }
{% endif %}
}
{% endfor %}
@@ -168,4 +178,4 @@ nodepools = {
{# ###Section for adding Defined and Freeform Tags ends here### #}
},
-{% endif %}
\ No newline at end of file
+{% endif %}
diff --git a/cd3_automation_toolkit/ocicloud/python/identity/users/export_users_nonGreenField.py b/cd3_automation_toolkit/ocicloud/python/identity/users/export_users_nonGreenField.py
index e07a9cfdc..c819278cc 100644
--- a/cd3_automation_toolkit/ocicloud/python/identity/users/export_users_nonGreenField.py
+++ b/cd3_automation_toolkit/ocicloud/python/identity/users/export_users_nonGreenField.py
@@ -132,18 +132,21 @@ def export_users(inputfile, outdir, service_dir, config, signer, ct,export_domai
if defined_tags_info is not None:
defined_tags = defined_tags_info.defined_tags
- for tag in defined_tags:
- namespace = tag.namespace
- key = tag.key
- value = tag.value
- if namespace is not None and key is not None and value is not None:
- user_defined_tags.append(f"{namespace}.{key}={value}")
+ if defined_tags is not None:
+ for tag in defined_tags:
+ namespace = tag.namespace
+ key = tag.key
+ value = tag.value
+ if namespace is not None and key is not None and value is not None:
+ user_defined_tags.append(f"{namespace}.{key}={value}")
user_defined_tags = ";".join(user_defined_tags) if user_defined_tags else ""
user_info = user
#if user_info.urn_ietf_params_scim_schemas_oracle_idcs_extension_user_user.is_federated_user != "True" and user_info.active !="False":
if (user_info.active != "False" and (user_info.urn_ietf_params_scim_schemas_oracle_idcs_extension_user_user is None or user_info.urn_ietf_params_scim_schemas_oracle_idcs_extension_user_user.is_federated_user != "True")):
+ if not user_info.name:
+ continue
username = user_info.user_name
family_name = user_info.name.family_name
given_name = user_info.name.given_name
diff --git a/cd3_automation_toolkit/ocicloud/python/network/Global/create_rpc_resources.py b/cd3_automation_toolkit/ocicloud/python/network/Global/create_rpc_resources.py
index 622e21361..2266fdf55 100755
--- a/cd3_automation_toolkit/ocicloud/python/network/Global/create_rpc_resources.py
+++ b/cd3_automation_toolkit/ocicloud/python/network/Global/create_rpc_resources.py
@@ -24,6 +24,23 @@
oci_regions_path = Path(__file__).resolve().parents[2] / "OCI_Regions"
+def resolve_oci_region_name(region):
+ """Return the canonical OCI region name for a CD3 region name or region ID."""
+ normalized_region = str(region).strip().lower()
+
+ with open(oci_regions_path, encoding='utf8') as regions_file:
+ for line in regions_file:
+ region_details = line.strip().split(':', 1)
+ if len(region_details) != 2:
+ continue
+
+ region_name, region_id = (value.strip() for value in region_details)
+ if normalized_region in (region_name.lower(), region_id.lower()):
+ return region_id
+
+ raise ValueError(f"Invalid OCI region '{region}'")
+
+
def find_subscribed_regions(inputfile, outdir, service_dir, prefix, config, signer, auth_mechanism):
subs_region_list = []
new_subs_region_list = []
@@ -184,7 +201,20 @@ def create_rpc_resource(inputfile, outdir, service_dir, prefix, auth_mechanism,
requester_drg_name = str(df.loc[i, 'DRG Name']).strip()
target_details = str(df.loc[i, 'Attached To'])
- attached_to = target_details.replace(target_details.split("::")[1], target_details.split("::")[1].lower())
+ target_details_parts = target_details.split("::")
+ if len(target_details_parts) != 3:
+ print(f"\nERROR!!! Invalid Attached To value '{target_details}' in DRGs sheet. "
+ "Expected RPC::::..Exiting!")
+ exit(1)
+
+ try:
+ accepter_region = resolve_oci_region_name(target_details_parts[1])
+ except ValueError as error:
+ print(f"\nERROR!!! {error} in Attached To value '{target_details}' in DRGs sheet..Exiting!")
+ exit(1)
+
+ accepter_drg_name = target_details_parts[2].strip()
+ attached_to = f"RPC::{accepter_region}::{accepter_drg_name}"
existing_line = f"{attached_to}::{requester_drg_name}"
# match = re.findall(existing_line, fo)
@@ -239,20 +269,11 @@ def create_rpc_resource(inputfile, outdir, service_dir, prefix, auth_mechanism,
tempdict = {'requester_compartment_name': compartment_var_name}
if columnname == 'Attached To':
- accepter_compartment_var_name = columnvalue.strip().split("::")
- accepter_region = accepter_compartment_var_name[1]
- accepter_region = next(
- line.split(':')[1].strip() for line in open(oci_regions_path) if
- line.startswith(f"{accepter_region}:"))
- accepter_drg_name = accepter_compartment_var_name[2]
tempdict = {'accepter_region': accepter_region.lower(),
'accepter_drg_name': accepter_drg_name}
if columnname == 'Region':
- requester_region = columnvalue.strip().lower()
- requester_region = next(
- line.split(':')[1].strip() for line in open(oci_regions_path) if
- line.startswith(f"{requester_region}:"))
+ requester_region = resolve_oci_region_name(columnvalue)
tempdict = {'requester_region': requester_region}
diff --git a/cd3_automation_toolkit/ocicloud/python/network/basenetwork/templates/nsg-rule-template b/cd3_automation_toolkit/ocicloud/python/network/basenetwork/templates/nsg-rule-template
index 2c7a14722..506af5090 100644
--- a/cd3_automation_toolkit/ocicloud/python/network/basenetwork/templates/nsg-rule-template
+++ b/cd3_automation_toolkit/ocicloud/python/network/basenetwork/templates/nsg-rule-template
@@ -84,7 +84,7 @@ EOF
{% endif %}
}]
{% else %}
- tcp = []
+ tcp = [{}]
{% endif %}
{% elif protocol.lower() == 'udp' %}
diff --git a/cd3_automation_toolkit/ocicloud/python/storage/objectstorage/export_terraform_oss.py b/cd3_automation_toolkit/ocicloud/python/storage/objectstorage/export_terraform_oss.py
index a6a41ff60..9c042092f 100644
--- a/cd3_automation_toolkit/ocicloud/python/storage/objectstorage/export_terraform_oss.py
+++ b/cd3_automation_toolkit/ocicloud/python/storage/objectstorage/export_terraform_oss.py
@@ -253,7 +253,7 @@ def export_buckets(inputfile, outdir, service_dir, config, signer, ct, export_co
if rt_time_rule_locked != 'None' and rt_time_rule_locked != '':
date_obj = parser.parse(rt_time_rule_locked)
# rt_time_rule_locked = date_obj.strftime("%Y-%m-%dT%H:%M:%S.%f")[:-3] + "Z"
- rt_time_rule_locked = (date_obj.strftime("%Y-%m-%dT%H:%M:%S.%f")[:-3]).rstrip("0") + "Z"
+ rt_time_rule_locked = (date_obj.strftime("%Y-%m-%dT%H:%M:%S.%f")[:-3]).rstrip("0").rstrip(".") + "Z"
rt_data = rt_name + "::" + rt_time_amount + "::" + rt_time_unit + "::" + rt_time_rule_locked
else:
rt_data = rt_name + "::" + rt_time_amount + "::" + rt_time_unit
diff --git a/cd3_automation_toolkit/ocicloud/terraform/compartment.tf b/cd3_automation_toolkit/ocicloud/terraform/compartment.tf
new file mode 100644
index 000000000..ec9629fd9
--- /dev/null
+++ b/cd3_automation_toolkit/ocicloud/terraform/compartment.tf
@@ -0,0 +1,154 @@
+# Copyright (c) 2024, Oracle and/or its affiliates. All rights reserved.
+# Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl.
+#
+############################
+# Module Block - Identity
+# Create Compartments
+############################
+
+module "iam-compartments" {
+ source = "./modules/identity/iam-compartment"
+ for_each = var.compartments.root != null ? var.compartments.root : {}
+
+ # insert the 4 required variables here
+ tenancy_ocid = var.tenancy_ocid
+ compartment_id = each.value.parent_compartment_id != null ? (length(regexall("ocid1.compartment.oc*", each.value.parent_compartment_id)) > 0 ? each.value.parent_compartment_id : try(var.compartment_ocids[each.value.parent_compartment_id], zipmap(data.oci_identity_compartments.compartments.compartments.*.name, data.oci_identity_compartments.compartments.compartments.*.id)[each.value.parent_compartment_id])) : var.tenancy_ocid
+ compartment_name = each.value.name
+ compartment_description = each.value.description
+ enable_delete = each.value.enable_delete
+
+ #Optional
+ defined_tags = each.value.defined_tags
+ freeform_tags = each.value.freeform_tags
+}
+
+module "sub-compartments-level1" {
+ source = "./modules/identity/iam-compartment"
+ for_each = var.compartments.compartment_level1 != null ? var.compartments.compartment_level1 : {}
+
+ depends_on = [module.iam-compartments]
+ # insert the 4 required variables here
+ tenancy_ocid = var.tenancy_ocid
+ compartment_id = length(regexall("ocid1.compartment.oc*", each.value.parent_compartment_id)) > 0 ? each.value.parent_compartment_id : try(var.compartment_ocids[each.value.parent_compartment_id], zipmap(data.oci_identity_compartments.compartments.compartments.*.name, data.oci_identity_compartments.compartments.compartments.*.id)[each.value.parent_compartment_id], merge(module.iam-compartments.*...)[each.value.parent_compartment_id]["compartment_tf_id"])
+ compartment_name = each.value.name
+ compartment_description = each.value.description
+ enable_delete = each.value.enable_delete
+
+ #Optional
+ defined_tags = each.value.defined_tags
+ freeform_tags = each.value.freeform_tags
+}
+
+module "sub-compartments-level2" {
+ source = "./modules/identity/iam-compartment"
+ for_each = var.compartments.compartment_level2 != null ? var.compartments.compartment_level2 : {}
+
+ depends_on = [module.sub-compartments-level1]
+ # insert the 4 required variables here
+ tenancy_ocid = var.tenancy_ocid
+ compartment_id = length(regexall("ocid1.compartment.oc*", each.value.parent_compartment_id)) > 0 ? each.value.parent_compartment_id : try(var.compartment_ocids[each.value.parent_compartment_id], zipmap(data.oci_identity_compartments.compartments.compartments.*.name, data.oci_identity_compartments.compartments.compartments.*.id)[each.value.parent_compartment_id], merge(module.sub-compartments-level1.*...)[each.value.parent_compartment_id]["compartment_tf_id"])
+ compartment_name = each.value.name
+ compartment_description = each.value.description
+
+ enable_delete = each.value.enable_delete
+
+ #Optional
+ defined_tags = each.value.defined_tags
+ freeform_tags = each.value.freeform_tags
+}
+
+module "sub-compartments-level3" {
+ source = "./modules/identity/iam-compartment"
+ for_each = var.compartments.compartment_level3 != null ? var.compartments.compartment_level3 : {}
+
+ depends_on = [module.sub-compartments-level2]
+ # insert the 4 required variables here
+ tenancy_ocid = var.tenancy_ocid
+ compartment_id = length(regexall("ocid1.compartment.oc*", each.value.parent_compartment_id)) > 0 ? each.value.parent_compartment_id : try(var.compartment_ocids[each.value.parent_compartment_id], zipmap(data.oci_identity_compartments.compartments.compartments.*.name, data.oci_identity_compartments.compartments.compartments.*.id)[each.value.parent_compartment_id], merge(module.sub-compartments-level2.*...)[each.value.parent_compartment_id]["compartment_tf_id"])
+ compartment_name = each.value.name
+ compartment_description = each.value.description
+ enable_delete = each.value.enable_delete
+
+ #Optional
+ defined_tags = each.value.defined_tags
+ freeform_tags = each.value.freeform_tags
+}
+
+module "sub-compartments-level4" {
+ source = "./modules/identity/iam-compartment"
+ for_each = var.compartments.compartment_level4 != null ? var.compartments.compartment_level4 : {}
+
+ depends_on = [module.sub-compartments-level3]
+ # insert the 4 required variables here
+ tenancy_ocid = var.tenancy_ocid
+ compartment_id = length(regexall("ocid1.compartment.oc*", each.value.parent_compartment_id)) > 0 ? each.value.parent_compartment_id : try(var.compartment_ocids[each.value.parent_compartment_id], zipmap(data.oci_identity_compartments.compartments.compartments.*.name, data.oci_identity_compartments.compartments.compartments.*.id)[each.value.parent_compartment_id], merge(module.sub-compartments-level3.*...)[each.value.parent_compartment_id]["compartment_tf_id"])
+ compartment_name = each.value.name
+ compartment_description = each.value.description
+ enable_delete = each.value.enable_delete
+
+ #Optional
+ defined_tags = each.value.defined_tags
+ freeform_tags = each.value.freeform_tags
+}
+
+module "sub-compartments-level5" {
+ source = "./modules/identity/iam-compartment"
+ for_each = var.compartments.compartment_level5 != null ? var.compartments.compartment_level5 : {}
+
+ depends_on = [module.sub-compartments-level4]
+ # insert the 4 required variables here
+ tenancy_ocid = var.tenancy_ocid
+ compartment_id = length(regexall("ocid1.compartment.oc*", each.value.parent_compartment_id)) > 0 ? each.value.parent_compartment_id : try(var.compartment_ocids[each.value.parent_compartment_id], zipmap(data.oci_identity_compartments.compartments.compartments.*.name, data.oci_identity_compartments.compartments.compartments.*.id)[each.value.parent_compartment_id], merge(module.sub-compartments-level4.*...)[each.value.parent_compartment_id]["compartment_tf_id"])
+ compartment_name = each.value.name
+ compartment_description = each.value.description
+ enable_delete = each.value.enable_delete
+
+ #Optional
+ defined_tags = each.value.defined_tags
+ freeform_tags = each.value.freeform_tags
+}
+
+/*
+output "root_compartments_map" {
+ description = "Compartment ocid"
+ // This allows the compartment ID to be retrieved from the resource if it exists, and if not to use the data source.
+ #value = element(concat(oci_identity_compartment.this.*.id, tolist([""])), 0)
+ value = [ for k,v in merge(module.iam-compartments.*...) : v.compartment_tf_id]
+}
+
+output "sub_compartments_level1_map" {
+ description = "Compartment ocid"
+ // This allows the compartment ID to be retrieved from the resource if it exists, and if not to use the data source.
+ #value = element(concat(oci_identity_compartment.this.*.id, tolist([""])), 0)
+ value = [ for k,v in merge(module.sub-compartments-level1.*...) : v.compartment_tf_id]
+}
+
+output "sub_compartments_level2_map" {
+ description = "Compartment ocid"
+ // This allows the compartment ID to be retrieved from the resource if it exists, and if not to use the data source.
+ #value = element(concat(oci_identity_compartment.this.*.id, tolist([""])), 0)
+ value = [ for k,v in merge(module.sub-compartments-level2.*...) : v.compartment_tf_id]
+}
+
+output "sub_compartments_level3_map" {
+ description = "Compartment ocid"
+ // This allows the compartment ID to be retrieved from the resource if it exists, and if not to use the data source.
+ #value = element(concat(oci_identity_compartment.this.*.id, tolist([""])), 0)
+ value = [ for k,v in merge(module.sub-compartments-level3.*...) : v.compartment_tf_id]
+}
+
+output "sub_compartments_level4_map" {
+ description = "Compartment ocid"
+ // This allows the compartment ID to be retrieved from the resource if it exists, and if not to use the data source.
+ #value = element(concat(oci_identity_compartment.this.*.id, tolist([""])), 0)
+ value = [ for k,v in merge(module.sub-compartments-level4.*...) : v.compartment_tf_id]
+}
+
+output "sub_compartments_level5_map" {
+ description = "Compartment ocid"
+ // This allows the compartment ID to be retrieved from the resource if it exists, and if not to use the data source.
+ #value = element(concat(oci_identity_compartment.this.*.id, tolist([""])), 0)
+ value = [ for k,v in merge(module.sub-compartments-level5.*...) : v.compartment_tf_id]
+}
+*/
+
diff --git a/cd3_automation_toolkit/ocicloud/terraform/group.tf b/cd3_automation_toolkit/ocicloud/terraform/group.tf
new file mode 100644
index 000000000..8f1ba1a60
--- /dev/null
+++ b/cd3_automation_toolkit/ocicloud/terraform/group.tf
@@ -0,0 +1,43 @@
+############################
+# Module Block - Identity
+# Create Groups
+############################
+
+module "iam-groups" {
+
+ source = "./modules/identity/iam-group"
+ for_each = var.groups
+ depends_on = [module.iam-users]
+ tenancy_ocid = var.tenancy_ocid
+ group_name = each.value.group_name
+ group_description = each.value.group_description
+ matching_rule = each.value.matching_rule
+ members = lookup(each.value, "members", [])
+
+ #Optional
+ defined_tags = each.value.defined_tags
+ freeform_tags = each.value.freeform_tags
+}
+
+module "groups" {
+
+ depends_on = [module.users]
+
+ source = "./modules/identity/identity-domain-group"
+ for_each = var.identity_domain_groups
+
+ group_name = each.value.group_name
+ group_description = each.value.group_description != null ? each.value.group_description : null
+ matching_rule = each.value.matching_rule
+ compartment_id = each.value.domain_compartment_id != "root" ? (length(regexall("ocid1.compartment.oc*", each.value.domain_compartment_id)) > 0 ? each.value.domain_compartment_id : var.compartment_ocids[each.value.domain_compartment_id]) : var.tenancy_ocid
+ identity_domain = data.oci_identity_domains.iam_domains[each.value.idcs_endpoint].domains[0]
+ tenancy_ocid = var.tenancy_ocid
+ members = each.value.members != null ? each.value.members : []
+ domain_users = local.domain_users_map[each.value.idcs_endpoint]
+ #Optional
+ user_can_request_access = each.value.user_can_request_access
+ defined_tags = each.value.defined_tags
+ freeform_tags_key = each.value.freeform_tags != null ? each.value.freeform_tags.key : null
+ freeform_tags_value = each.value.freeform_tags != null ? each.value.freeform_tags.value : null
+
+}
\ No newline at end of file
diff --git a/cd3_automation_toolkit/ocicloud/terraform/identity.tf b/cd3_automation_toolkit/ocicloud/terraform/identity.tf
deleted file mode 100644
index 6869da819..000000000
--- a/cd3_automation_toolkit/ocicloud/terraform/identity.tf
+++ /dev/null
@@ -1,393 +0,0 @@
-# Copyright (c) 2024, Oracle and/or its affiliates. All rights reserved.
-# Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl.
-#
-############################
-# Module Block - Identity
-# Create Compartments
-############################
-
-module "iam-compartments" {
- source = "./modules/identity/iam-compartment"
- for_each = var.compartments.root != null ? var.compartments.root : {}
-
- # insert the 4 required variables here
- tenancy_ocid = var.tenancy_ocid
- compartment_id = each.value.parent_compartment_id != null ? (length(regexall("ocid1.compartment.oc*", each.value.parent_compartment_id)) > 0 ? each.value.parent_compartment_id : try(var.compartment_ocids[each.value.parent_compartment_id], zipmap(data.oci_identity_compartments.compartments.compartments.*.name, data.oci_identity_compartments.compartments.compartments.*.id)[each.value.parent_compartment_id])) : var.tenancy_ocid
- compartment_name = each.value.name
- compartment_description = each.value.description
- enable_delete = each.value.enable_delete
-
- #Optional
- defined_tags = each.value.defined_tags
- freeform_tags = each.value.freeform_tags
-}
-
-module "sub-compartments-level1" {
- source = "./modules/identity/iam-compartment"
- for_each = var.compartments.compartment_level1 != null ? var.compartments.compartment_level1 : {}
-
- depends_on = [module.iam-compartments]
- # insert the 4 required variables here
- tenancy_ocid = var.tenancy_ocid
- compartment_id = length(regexall("ocid1.compartment.oc*", each.value.parent_compartment_id)) > 0 ? each.value.parent_compartment_id : try(var.compartment_ocids[each.value.parent_compartment_id], zipmap(data.oci_identity_compartments.compartments.compartments.*.name, data.oci_identity_compartments.compartments.compartments.*.id)[each.value.parent_compartment_id], merge(module.iam-compartments.*...)[each.value.parent_compartment_id]["compartment_tf_id"])
- compartment_name = each.value.name
- compartment_description = each.value.description
- enable_delete = each.value.enable_delete
-
- #Optional
- defined_tags = each.value.defined_tags
- freeform_tags = each.value.freeform_tags
-}
-
-module "sub-compartments-level2" {
- source = "./modules/identity/iam-compartment"
- for_each = var.compartments.compartment_level2 != null ? var.compartments.compartment_level2 : {}
-
- depends_on = [module.sub-compartments-level1]
- # insert the 4 required variables here
- tenancy_ocid = var.tenancy_ocid
- compartment_id = length(regexall("ocid1.compartment.oc*", each.value.parent_compartment_id)) > 0 ? each.value.parent_compartment_id : try(var.compartment_ocids[each.value.parent_compartment_id], zipmap(data.oci_identity_compartments.compartments.compartments.*.name, data.oci_identity_compartments.compartments.compartments.*.id)[each.value.parent_compartment_id], merge(module.sub-compartments-level1.*...)[each.value.parent_compartment_id]["compartment_tf_id"])
- compartment_name = each.value.name
- compartment_description = each.value.description
-
- enable_delete = each.value.enable_delete
-
- #Optional
- defined_tags = each.value.defined_tags
- freeform_tags = each.value.freeform_tags
-}
-
-module "sub-compartments-level3" {
- source = "./modules/identity/iam-compartment"
- for_each = var.compartments.compartment_level3 != null ? var.compartments.compartment_level3 : {}
-
- depends_on = [module.sub-compartments-level2]
- # insert the 4 required variables here
- tenancy_ocid = var.tenancy_ocid
- compartment_id = length(regexall("ocid1.compartment.oc*", each.value.parent_compartment_id)) > 0 ? each.value.parent_compartment_id : try(var.compartment_ocids[each.value.parent_compartment_id], zipmap(data.oci_identity_compartments.compartments.compartments.*.name, data.oci_identity_compartments.compartments.compartments.*.id)[each.value.parent_compartment_id], merge(module.sub-compartments-level2.*...)[each.value.parent_compartment_id]["compartment_tf_id"])
- compartment_name = each.value.name
- compartment_description = each.value.description
- enable_delete = each.value.enable_delete
-
- #Optional
- defined_tags = each.value.defined_tags
- freeform_tags = each.value.freeform_tags
-}
-
-module "sub-compartments-level4" {
- source = "./modules/identity/iam-compartment"
- for_each = var.compartments.compartment_level4 != null ? var.compartments.compartment_level4 : {}
-
- depends_on = [module.sub-compartments-level3]
- # insert the 4 required variables here
- tenancy_ocid = var.tenancy_ocid
- compartment_id = length(regexall("ocid1.compartment.oc*", each.value.parent_compartment_id)) > 0 ? each.value.parent_compartment_id : try(var.compartment_ocids[each.value.parent_compartment_id], zipmap(data.oci_identity_compartments.compartments.compartments.*.name, data.oci_identity_compartments.compartments.compartments.*.id)[each.value.parent_compartment_id], merge(module.sub-compartments-level3.*...)[each.value.parent_compartment_id]["compartment_tf_id"])
- compartment_name = each.value.name
- compartment_description = each.value.description
- enable_delete = each.value.enable_delete
-
- #Optional
- defined_tags = each.value.defined_tags
- freeform_tags = each.value.freeform_tags
-}
-
-module "sub-compartments-level5" {
- source = "./modules/identity/iam-compartment"
- for_each = var.compartments.compartment_level5 != null ? var.compartments.compartment_level5 : {}
-
- depends_on = [module.sub-compartments-level4]
- # insert the 4 required variables here
- tenancy_ocid = var.tenancy_ocid
- compartment_id = length(regexall("ocid1.compartment.oc*", each.value.parent_compartment_id)) > 0 ? each.value.parent_compartment_id : try(var.compartment_ocids[each.value.parent_compartment_id], zipmap(data.oci_identity_compartments.compartments.compartments.*.name, data.oci_identity_compartments.compartments.compartments.*.id)[each.value.parent_compartment_id], merge(module.sub-compartments-level4.*...)[each.value.parent_compartment_id]["compartment_tf_id"])
- compartment_name = each.value.name
- compartment_description = each.value.description
- enable_delete = each.value.enable_delete
-
- #Optional
- defined_tags = each.value.defined_tags
- freeform_tags = each.value.freeform_tags
-}
-
-/*
-output "root_compartments_map" {
- description = "Compartment ocid"
- // This allows the compartment ID to be retrieved from the resource if it exists, and if not to use the data source.
- #value = element(concat(oci_identity_compartment.this.*.id, tolist([""])), 0)
- value = [ for k,v in merge(module.iam-compartments.*...) : v.compartment_tf_id]
-}
-
-output "sub_compartments_level1_map" {
- description = "Compartment ocid"
- // This allows the compartment ID to be retrieved from the resource if it exists, and if not to use the data source.
- #value = element(concat(oci_identity_compartment.this.*.id, tolist([""])), 0)
- value = [ for k,v in merge(module.sub-compartments-level1.*...) : v.compartment_tf_id]
-}
-
-output "sub_compartments_level2_map" {
- description = "Compartment ocid"
- // This allows the compartment ID to be retrieved from the resource if it exists, and if not to use the data source.
- #value = element(concat(oci_identity_compartment.this.*.id, tolist([""])), 0)
- value = [ for k,v in merge(module.sub-compartments-level2.*...) : v.compartment_tf_id]
-}
-
-output "sub_compartments_level3_map" {
- description = "Compartment ocid"
- // This allows the compartment ID to be retrieved from the resource if it exists, and if not to use the data source.
- #value = element(concat(oci_identity_compartment.this.*.id, tolist([""])), 0)
- value = [ for k,v in merge(module.sub-compartments-level3.*...) : v.compartment_tf_id]
-}
-
-output "sub_compartments_level4_map" {
- description = "Compartment ocid"
- // This allows the compartment ID to be retrieved from the resource if it exists, and if not to use the data source.
- #value = element(concat(oci_identity_compartment.this.*.id, tolist([""])), 0)
- value = [ for k,v in merge(module.sub-compartments-level4.*...) : v.compartment_tf_id]
-}
-
-output "sub_compartments_level5_map" {
- description = "Compartment ocid"
- // This allows the compartment ID to be retrieved from the resource if it exists, and if not to use the data source.
- #value = element(concat(oci_identity_compartment.this.*.id, tolist([""])), 0)
- value = [ for k,v in merge(module.sub-compartments-level5.*...) : v.compartment_tf_id]
-}
-*/
-
-
-############################
-# Module Block - Identity
-# Create Groups
-############################
-
-module "iam-groups" {
-
- source = "./modules/identity/iam-group"
- for_each = var.groups
- depends_on = [module.iam-users]
- tenancy_ocid = var.tenancy_ocid
- group_name = each.value.group_name
- group_description = each.value.group_description
- matching_rule = each.value.matching_rule
- members = lookup(each.value, "members", [])
-
- #Optional
- defined_tags = each.value.defined_tags
- freeform_tags = each.value.freeform_tags
-}
-
-/*
-output "group_id_map" {
- value = [ for k,v in merge(module.iam-groups.*...) : v.group_id_map]
-}
-
-
-output "dynamic_group_id_map" {
- value = [ for k,v in merge(module.iam-groups.*...) : v.dynamic_group_id_map]
-}
-*/
-
-############################
-# Module Block - Identity
-# Create Policies
-############################
-
-module "iam-policies" {
- source = "./modules/identity/iam-policy"
- for_each = var.policies
-
- depends_on = [module.iam-groups]
- tenancy_ocid = var.tenancy_ocid
- policy_name = each.value.name
- policy_compartment_id = each.value.compartment_id != "root" ? (length(regexall("ocid1.compartment.oc*", each.value.compartment_id)) > 0 ? each.value.compartment_id : var.compartment_ocids[each.value.compartment_id]) : var.tenancy_ocid
- policy_description = each.value.policy_description
- policy_statements = each.value.policy_statements
-
- #Optional
- defined_tags = each.value.defined_tags
- freeform_tags = each.value.freeform_tags
- policy_version_date = each.value.policy_version_date
-}
-
-/*
-output "policies_id_map" {
- value = [ for k,v in merge(module.iam-policies.*...) : v.policies_id_map]
-}
-*/
-
-############################
-# Module Block - Identity
-# Create Users
-############################
-
-module "iam-users" {
- source = "./modules/identity/iam-user"
- #depends_on = [module.iam-groups]
- for_each = var.users
- user_name = each.value.name
- user_description = each.value.description
- user_email = each.value.email
- tenancy_ocid = var.tenancy_ocid
- enabled_capabilities = each.value.enabled_capabilities != null ? each.value.enabled_capabilities : null
-
- #Optional
- defined_tags = each.value.defined_tags
- freeform_tags = each.value.freeform_tags
-}
-
-
-
-############################
-# Module - Network Source
-# Create Network Source
-############################
-
-#locals {
-#
-#vcns = flatten ([
-#for key, val in var.networkSources : [
-# for k,virtual_source in val.virtual_source_list != null ? val.virtual_source_list : [] :{
-# vcn_name = virtual_source.vcn_name.0
-# network_compartment = virtual_source.network_compartment_id.0
-# }
-# ]
-#])
-#}
-
-#data "oci_core_vcns" "oci_vcns_networksource" {
-#
-# for_each = { for vcn in local.vcns : vcn.vcn_name => vcn... }
-# display_name = each.key
-# compartment_id = var.compartment_ocids[each.value[0].network_compartment]
-#}
-
-module "iam-network-sources" {
- source = "./modules/identity/iam-network-sources"
- for_each = var.networkSources
- name = each.value.name
- description = each.value.description
- tenancy_ocid = var.tenancy_ocid
-
- #Optional
- public_source_list = each.value.public_source_list != null ? each.value.public_source_list : null
- #virtual_source_list = each.value.virtual_source_list != null ? each.value.virtual_source_list : null
- virtual_source_list = { for k, v in each.value.virtual_source_list != null ? each.value.virtual_source_list : [] : k =>
- {
- #vcn_id = data.oci_core_vcns.oci_vcns_networksource[v.vcn_name.0].virtual_networks.*.id[0]
- ip_ranges = v.ip_ranges
- } }
- #vcn_comp_map = each.value.vcn_comp_map != null ? each.value.vcn_comp_map : null
- defined_tags = try(each.value.defined_tags, null)
- freeform_tags = try(each.value.freeform_tags, null)
-}
-############################
-# Module Block - Identity
-# Create Identity Domain Groups
-############################
-locals {
- # get all the domains used for users
- users_unique_idcs_endpoints = {
- for k, v in var.identity_domain_users :
- v.idcs_endpoint => {
- idcs_endpoint = v.idcs_endpoint
- domain_compartment_id = v.domain_compartment_id
- }...
- }
- # get all the domains used for groups
- groups_unique_idcs_endpoints = {
- for k, v in var.identity_domain_groups :
- v.idcs_endpoint => {
- idcs_endpoint = v.idcs_endpoint
- domain_compartment_id = v.domain_compartment_id
- }...
- }
- # get unique domains used across users and groups
- domains_distinct = { for k, v in merge(local.groups_unique_idcs_endpoints, local.users_unique_idcs_endpoints) : k => distinct(v)[0]... }
-
- # users in each domain used in groups
- domain_users_map = {
- for k, v in local.domains_distinct :
- k => {
- for user in data.oci_identity_domains_users.users[k].users :
- user.user_name => user.id
- } if contains(keys(local.groups_unique_idcs_endpoints), k)
- }
-
-}
-# output "domain_distinct" {
-# value = local.domains_distinct
-# }
-# output "groups_unique_idcs_endpoints" {
-# value = local.groups_unique_idcs_endpoints
-# }
-
-# domain data for unique domin used across users and groups
-data "oci_identity_domains" "iam_domains" {
- for_each = local.domains_distinct
- # Required
- compartment_id = var.compartment_ocids[each.value[0].domain_compartment_id]
- # Optional
- display_name = each.key
-}
-
-# user data for each used domain
-data "oci_identity_domains_users" "users" {
- for_each = { for k, v in local.domains_distinct : k => v if contains(keys(local.groups_unique_idcs_endpoints), k) }
- idcs_endpoint = data.oci_identity_domains.iam_domains[each.value[0].idcs_endpoint].domains[0].url
-}
-
-# output "user_map" {
-# value = local.domain_users_map
-# }
-module "groups" {
-
- depends_on = [module.users]
-
- source = "./modules/identity/identity-domain-group"
- for_each = var.identity_domain_groups
-
- group_name = each.value.group_name
- group_description = each.value.group_description != null ? each.value.group_description : null
- matching_rule = each.value.matching_rule
- compartment_id = each.value.domain_compartment_id != "root" ? (length(regexall("ocid1.compartment.oc*", each.value.domain_compartment_id)) > 0 ? each.value.domain_compartment_id : var.compartment_ocids[each.value.domain_compartment_id]) : var.tenancy_ocid
- identity_domain = data.oci_identity_domains.iam_domains[each.value.idcs_endpoint].domains[0]
- tenancy_ocid = var.tenancy_ocid
- members = each.value.members != null ? each.value.members : []
- domain_users = local.domain_users_map[each.value.idcs_endpoint]
- #Optional
- user_can_request_access = each.value.user_can_request_access
- defined_tags = each.value.defined_tags
- freeform_tags_key = each.value.freeform_tags != null ? each.value.freeform_tags.key : null
- freeform_tags_value = each.value.freeform_tags != null ? each.value.freeform_tags.value : null
-
-}
-
-############################
-# Module Block - Identity
-# Create Identity Domain Users
-############################
-
-module "users" {
- source = "./modules/identity/identity-domain-user"
- #depends_on = [module.iam-groups]
- for_each = var.identity_domain_users
- user_name = each.value.user_name
- family_name = each.value.name.family_name
- given_name = each.value.name.given_name
- middle_name = each.value.name.middle_name
- honorific_prefix = each.value.name.honorific_prefix
- display_name = each.value.display_name
- identity_domain = data.oci_identity_domains.iam_domains[each.value.idcs_endpoint].domains[0]
- compartment_id = each.value.domain_compartment_id != "root" ? (length(regexall("ocid1.compartment.oc*", each.value.domain_compartment_id)) > 0 ? each.value.domain_compartment_id : var.compartment_ocids[each.value.domain_compartment_id]) : var.tenancy_ocid
- description = each.value.description
- email = each.value.email
- recovery_email = each.value.recovery_email
- tenancy_ocid = var.tenancy_ocid
- groups = each.value.groups != null ? each.value.groups : null
- home_phone_number = each.value.home_phone_number
- mobile_phone_number = each.value.mobile_phone_number
- enabled_capabilities = each.value.enabled_capabilities
-
- #Optional
- defined_tags = each.value.defined_tags
- freeform_tags_key = each.value.freeform_tags != null ? each.value.freeform_tags.key : null
- freeform_tags_value = each.value.freeform_tags != null ? each.value.freeform_tags.value : null
-}
\ No newline at end of file
diff --git a/cd3_automation_toolkit/ocicloud/terraform/instance.tf b/cd3_automation_toolkit/ocicloud/terraform/instance.tf
index 604c83087..86cd04c54 100755
--- a/cd3_automation_toolkit/ocicloud/terraform/instance.tf
+++ b/cd3_automation_toolkit/ocicloud/terraform/instance.tf
@@ -79,4 +79,9 @@ module "instances" {
vnic_defined_tags = each.value.vnic_defined_tags
vnic_freeform_tags = each.value.vnic_freeform_tags
vnic_display_name = each.value.vnic_display_name
+ secondary_vnics = each.value.secondary_vnics != null ? {
+ for vnic_key, vnic in each.value.secondary_vnics : vnic_key => merge(vnic, {
+ network_compartment_id = try(vnic.network_compartment_id, null) != null ? (length(regexall("ocid1.compartment.oc*", vnic.network_compartment_id)) > 0 ? vnic.network_compartment_id : var.compartment_ocids[vnic.network_compartment_id]) : null
+ })
+ } : {}
}
diff --git a/cd3_automation_toolkit/ocicloud/terraform/modules/compute/instance/data.tf b/cd3_automation_toolkit/ocicloud/terraform/modules/compute/instance/data.tf
index 5ed71dab7..affa45762 100755
--- a/cd3_automation_toolkit/ocicloud/terraform/modules/compute/instance/data.tf
+++ b/cd3_automation_toolkit/ocicloud/terraform/modules/compute/instance/data.tf
@@ -9,6 +9,31 @@
locals {
nsg_ids = var.nsg_ids != null ? flatten(tolist([for nsg in var.nsg_ids : (length(regexall("ocid1.networksecuritygroup.oc*", nsg)) > 0 ? [nsg] : data.oci_core_network_security_groups.network_security_groups[nsg].network_security_groups[*].id)])) : null
+ secondary_vnic_vcns = {
+ for vnic_key, vnic in var.secondary_vnics : vnic_key => {
+ network_compartment_id = try(vnic.network_compartment_id, null)
+ vcn_name = try(vnic.vcn_name, "")
+ } if try(vnic.vcn_name, "") != ""
+ }
+
+ secondary_vnic_nsg_keys = flatten([
+ for vnic_key, vnic in var.secondary_vnics : [
+ for nsg in coalesce(try(vnic.nsg_ids, []), []) : {
+ key = "${vnic_key}::${nsg}"
+ vnic_key = vnic_key
+ nsg = nsg
+ network_compartment_id = try(vnic.network_compartment_id, null)
+ vcn_name = try(vnic.vcn_name, "")
+ }
+ ]
+ ])
+
+ secondary_vnic_nsg_ids = {
+ for vnic_key, vnic in var.secondary_vnics : vnic_key => flatten([
+ for nsg in coalesce(try(vnic.nsg_ids, []), []) : length(regexall("ocid1.networksecuritygroup.oc*", nsg)) > 0 ? [nsg] : data.oci_core_network_security_groups.secondary_vnic_network_security_groups["${vnic_key}::${nsg}"].network_security_groups[*].id
+ ])
+ }
+
ADs = [
for ad in data.oci_identity_availability_domains.ads.availability_domains : ad.name
]
@@ -66,6 +91,27 @@ data "oci_core_network_security_groups" "network_security_groups" {
vcn_id = data.oci_core_vcns.oci_vcns_instances[var.vcn_names[0]].virtual_networks.*.id[0]
}
+
+data "oci_core_vcns" "secondary_vnic_vcns" {
+ for_each = local.secondary_vnic_vcns
+ compartment_id = try(length(regexall("ocid1.compartment.oc*", each.value.network_compartment_id)) > 0, false) ? each.value.network_compartment_id : var.network_compartment_id != null ? var.network_compartment_id : var.compartment_id
+ display_name = each.value.vcn_name
+}
+
+data "oci_core_subnets" "secondary_vnic_subnets" {
+ for_each = { for vnic_key, vnic in var.secondary_vnics : vnic_key => vnic if length(regexall("ocid1.subnet.oc*", vnic.subnet_id)) == 0 }
+ compartment_id = try(length(regexall("ocid1.compartment.oc*", each.value.network_compartment_id)) > 0, false) ? each.value.network_compartment_id : var.network_compartment_id != null ? var.network_compartment_id : var.compartment_id
+ display_name = each.value.subnet_id
+ vcn_id = try(each.value.vcn_name, "") != "" ? data.oci_core_vcns.secondary_vnic_vcns[each.key].virtual_networks.*.id[0] : null
+}
+
+data "oci_core_network_security_groups" "secondary_vnic_network_security_groups" {
+ for_each = { for item in local.secondary_vnic_nsg_keys : item.key => item if length(regexall("ocid1.networksecuritygroup.oc*", item.nsg)) == 0 }
+ compartment_id = try(length(regexall("ocid1.compartment.oc*", each.value.network_compartment_id)) > 0, false) ? each.value.network_compartment_id : var.network_compartment_id != null ? var.network_compartment_id : var.compartment_id
+ display_name = each.value.nsg
+ vcn_id = each.value.vcn_name != "" ? data.oci_core_vcns.secondary_vnic_vcns[each.value.vnic_key].virtual_networks.*.id[0] : data.oci_core_vcns.oci_vcns_instances[var.vcn_names[0]].virtual_networks.*.id[0]
+}
+
#data "oci_core_boot_volumes" "all_boot_volumes" {
# depends_on = [oci_core_instance.instance]
# count = var.boot_tf_policy != null ? 1 : 0
diff --git a/cd3_automation_toolkit/ocicloud/terraform/modules/compute/instance/main.tf b/cd3_automation_toolkit/ocicloud/terraform/modules/compute/instance/main.tf
index 3355d255f..4260b345c 100755
--- a/cd3_automation_toolkit/ocicloud/terraform/modules/compute/instance/main.tf
+++ b/cd3_automation_toolkit/ocicloud/terraform/modules/compute/instance/main.tf
@@ -133,6 +133,27 @@ resource "oci_core_instance" "instance" {
}
+resource "oci_core_vnic_attachment" "secondary_vnic_attachment" {
+ for_each = var.secondary_vnics
+ instance_id = oci_core_instance.instance.id
+
+ create_vnic_details {
+ assign_private_dns_record = try(each.value.assign_private_dns_record, null)
+ assign_public_ip = try(each.value.assign_public_ip, false)
+ defined_tags = try(each.value.defined_tags, {})
+ display_name = try(each.value.display_name, "") != "" ? each.value.display_name : "${var.display_name}-${each.key}"
+ freeform_tags = try(each.value.freeform_tags, {})
+ hostname_label = try(each.value.hostname_label, null)
+ nsg_ids = try(each.value.nsg_ids, []) != [] ? local.secondary_vnic_nsg_ids[each.key] : null
+ private_ip = try(each.value.private_ip, null)
+ skip_source_dest_check = try(each.value.skip_source_dest_check, false)
+ subnet_id = length(regexall("ocid1.subnet.oc*", each.value.subnet_id)) > 0 ? each.value.subnet_id : data.oci_core_subnets.secondary_vnic_subnets[each.key].subnets.*.id[0]
+ }
+
+ nic_index = try(each.value.physical_nic_index, null)
+}
+
+
resource "null_resource" "ansible-remote-exec" {
count = var.remote_execute == null ? 0 : ((length(regexall(".yaml", local.remote_execute_script)) > 0) || (length(regexall(".yml", local.remote_execute_script)) > 0) ? 1 : 0)
diff --git a/cd3_automation_toolkit/ocicloud/terraform/modules/compute/instance/variables.tf b/cd3_automation_toolkit/ocicloud/terraform/modules/compute/instance/variables.tf
index dba44a73d..002f7eb45 100755
--- a/cd3_automation_toolkit/ocicloud/terraform/modules/compute/instance/variables.tf
+++ b/cd3_automation_toolkit/ocicloud/terraform/modules/compute/instance/variables.tf
@@ -317,6 +317,13 @@ variable "vnic_freeform_tags" {
default = {}
}
+
+variable "secondary_vnics" {
+ type = map(any)
+ description = "Secondary VNIC attachments for the instance"
+ default = {}
+}
+
variable "ssh_private_key_file_path" {
type = string
default = null
diff --git a/cd3_automation_toolkit/ocicloud/terraform/modules/oke/cluster/main.tf b/cd3_automation_toolkit/ocicloud/terraform/modules/oke/cluster/main.tf
index 96dde2e80..6fdf59e5d 100644
--- a/cd3_automation_toolkit/ocicloud/terraform/modules/oke/cluster/main.tf
+++ b/cd3_automation_toolkit/ocicloud/terraform/modules/oke/cluster/main.tf
@@ -38,10 +38,6 @@ resource "oci_containerengine_cluster" "cluster" {
is_kubernetes_dashboard_enabled = var.is_kubernetes_dashboard_enabled
is_tiller_enabled = var.is_tiller_enabled
}
- admission_controller_options {
- is_pod_security_policy_enabled = var.is_pod_security_policy_enabled
- }
-
kubernetes_network_config {
pods_cidr = var.pods_cidr
services_cidr = var.services_cidr
diff --git a/cd3_automation_toolkit/ocicloud/terraform/modules/oke/cluster/variables.tf b/cd3_automation_toolkit/ocicloud/terraform/modules/oke/cluster/variables.tf
index 6ed663973..c35672995 100644
--- a/cd3_automation_toolkit/ocicloud/terraform/modules/oke/cluster/variables.tf
+++ b/cd3_automation_toolkit/ocicloud/terraform/modules/oke/cluster/variables.tf
@@ -83,11 +83,6 @@ variable "is_tiller_enabled" {
description = "Whether tiller is ebabled"
}
-variable "is_pod_security_policy_enabled" {
- type = bool
- description = "Whether a pod security needs to be enabled for the nodepool"
-}
-
variable "pods_cidr" {
type = string
description = "The pod CIDR value"
@@ -142,4 +137,3 @@ variable "lb_freeform_tags" {
type = map(any)
default = {}
}
-
diff --git a/cd3_automation_toolkit/ocicloud/terraform/modules/oke/nodepool/main.tf b/cd3_automation_toolkit/ocicloud/terraform/modules/oke/nodepool/main.tf
index 259f5c423..f403fc1d4 100644
--- a/cd3_automation_toolkit/ocicloud/terraform/modules/oke/nodepool/main.tf
+++ b/cd3_automation_toolkit/ocicloud/terraform/modules/oke/nodepool/main.tf
@@ -28,6 +28,15 @@ resource "oci_containerengine_node_pool" "nodepool" {
subnet_id = placement_configs.value.subnet_id
fault_domains = placement_configs.value.fault_domains
capacity_reservation_id = placement_configs.value.capacity_reservation_id
+ dynamic "preemptible_node_config" {
+ for_each = try(placement_configs.value.preemptible_node_config, null) != null ? [placement_configs.value.preemptible_node_config] : []
+ content {
+ preemption_action {
+ type = preemptible_node_config.value.preemption_action.type
+ is_preserve_boot_volume = try(preemptible_node_config.value.preemption_action.is_preserve_boot_volume, false)
+ }
+ }
+ }
}
}
node_pool_pod_network_option_details {
diff --git a/cd3_automation_toolkit/ocicloud/terraform/networksource.tf b/cd3_automation_toolkit/ocicloud/terraform/networksource.tf
new file mode 100644
index 000000000..52f1e0459
--- /dev/null
+++ b/cd3_automation_toolkit/ocicloud/terraform/networksource.tf
@@ -0,0 +1,48 @@
+# Copyright (c) 2024, Oracle and/or its affiliates. All rights reserved.
+# Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl.
+#
+
+
+############################
+# Module - Network Source
+# Create Network Source
+############################
+
+#locals {
+#
+#vcns = flatten ([
+#for key, val in var.networkSources : [
+# for k,virtual_source in val.virtual_source_list != null ? val.virtual_source_list : [] :{
+# vcn_name = virtual_source.vcn_name.0
+# network_compartment = virtual_source.network_compartment_id.0
+# }
+# ]
+#])
+#}
+
+#data "oci_core_vcns" "oci_vcns_networksource" {
+#
+# for_each = { for vcn in local.vcns : vcn.vcn_name => vcn... }
+# display_name = each.key
+# compartment_id = var.compartment_ocids[each.value[0].network_compartment]
+#}
+
+module "iam-network-sources" {
+ source = "./modules/identity/iam-network-sources"
+ for_each = var.networkSources
+ name = each.value.name
+ description = each.value.description
+ tenancy_ocid = var.tenancy_ocid
+
+ #Optional
+ public_source_list = each.value.public_source_list != null ? each.value.public_source_list : null
+ #virtual_source_list = each.value.virtual_source_list != null ? each.value.virtual_source_list : null
+ virtual_source_list = { for k, v in each.value.virtual_source_list != null ? each.value.virtual_source_list : [] : k =>
+ {
+ #vcn_id = data.oci_core_vcns.oci_vcns_networksource[v.vcn_name.0].virtual_networks.*.id[0]
+ ip_ranges = v.ip_ranges
+ } }
+ #vcn_comp_map = each.value.vcn_comp_map != null ? each.value.vcn_comp_map : null
+ defined_tags = try(each.value.defined_tags, null)
+ freeform_tags = try(each.value.freeform_tags, null)
+}
diff --git a/cd3_automation_toolkit/ocicloud/terraform/oke.tf b/cd3_automation_toolkit/ocicloud/terraform/oke.tf
index d726b9743..1b68a4c44 100644
--- a/cd3_automation_toolkit/ocicloud/terraform/oke.tf
+++ b/cd3_automation_toolkit/ocicloud/terraform/oke.tf
@@ -32,6 +32,7 @@ locals {
availability_domain = vv.availability_domain
fault_domains = vv.fault_domains
capacity_reservation_id = vv.capacity_reservation_id
+ preemptible_node_config = try(vv.preemptible_node_config, null)
}
}
}
@@ -151,7 +152,6 @@ module "clusters" {
is_public_ip_enabled = each.value.is_public_ip_enabled
nsg_ids = each.value.nsg_ids
endpoint_subnet_id = length(regexall("ocid1.subnet.oc*", each.value.endpoint_subnet_id)) > 0 ? each.value.endpoint_subnet_id : data.oci_core_subnets.oci_subnets_endpoint[each.key].subnets.*.id[0]
- is_pod_security_policy_enabled = each.value.is_pod_security_policy_enabled
pods_cidr = each.value.pods_cidr
services_cidr = each.value.services_cidr
service_lb_subnet_ids = each.value.service_lb_subnet_ids
@@ -223,4 +223,4 @@ module "virtual-nodepools" {
node_freeform_tags = each.value.node_freeform_tags
nodepool_defined_tags = each.value.nodepool_defined_tags
nodepool_freeform_tags = each.value.nodepool_freeform_tags
-}
\ No newline at end of file
+}
diff --git a/cd3_automation_toolkit/ocicloud/terraform/policy.tf b/cd3_automation_toolkit/ocicloud/terraform/policy.tf
new file mode 100644
index 000000000..e446ad133
--- /dev/null
+++ b/cd3_automation_toolkit/ocicloud/terraform/policy.tf
@@ -0,0 +1,26 @@
+# Copyright (c) 2024, Oracle and/or its affiliates. All rights reserved.
+# Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl.
+#
+
+
+############################
+# Module Block - Identity
+# Create Policies
+############################
+
+module "iam-policies" {
+ source = "./modules/identity/iam-policy"
+ for_each = var.policies
+
+ #depends_on = [module.iam-groups]
+ tenancy_ocid = var.tenancy_ocid
+ policy_name = each.value.name
+ policy_compartment_id = each.value.compartment_id != "root" ? (length(regexall("ocid1.compartment.oc*", each.value.compartment_id)) > 0 ? each.value.compartment_id : var.compartment_ocids[each.value.compartment_id]) : var.tenancy_ocid
+ policy_description = each.value.policy_description
+ policy_statements = each.value.policy_statements
+
+ #Optional
+ defined_tags = each.value.defined_tags
+ freeform_tags = each.value.freeform_tags
+ policy_version_date = each.value.policy_version_date
+}
diff --git a/cd3_automation_toolkit/ocicloud/terraform/user.tf b/cd3_automation_toolkit/ocicloud/terraform/user.tf
new file mode 100644
index 000000000..5d8cb8416
--- /dev/null
+++ b/cd3_automation_toolkit/ocicloud/terraform/user.tf
@@ -0,0 +1,131 @@
+# Copyright (c) 2024, Oracle and/or its affiliates. All rights reserved.
+# Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl.
+#
+
+
+############################
+# Module Block - Identity
+# Create Users
+############################
+
+module "iam-users" {
+ source = "./modules/identity/iam-user"
+ #depends_on = [module.iam-groups]
+ for_each = var.users
+ user_name = each.value.name
+ user_description = each.value.description
+ user_email = each.value.email
+ tenancy_ocid = var.tenancy_ocid
+ enabled_capabilities = each.value.enabled_capabilities != null ? each.value.enabled_capabilities : null
+
+ #Optional
+ defined_tags = each.value.defined_tags
+ freeform_tags = each.value.freeform_tags
+}
+
+
+
+
+/*
+output "group_id_map" {
+ value = [ for k,v in merge(module.iam-groups.*...) : v.group_id_map]
+}
+
+
+output "dynamic_group_id_map" {
+ value = [ for k,v in merge(module.iam-groups.*...) : v.dynamic_group_id_map]
+}
+*/
+
+############################
+# Module Block - Identity
+# Create Identity Domain Groups
+############################
+locals {
+ # get all the domains used for users
+ users_unique_idcs_endpoints = {
+ for k, v in var.identity_domain_users :
+ v.idcs_endpoint => {
+ idcs_endpoint = v.idcs_endpoint
+ domain_compartment_id = v.domain_compartment_id
+ }...
+ }
+ # get all the domains used for groups
+ groups_unique_idcs_endpoints = {
+ for k, v in var.identity_domain_groups :
+ v.idcs_endpoint => {
+ idcs_endpoint = v.idcs_endpoint
+ domain_compartment_id = v.domain_compartment_id
+ }...
+ }
+ # get unique domains used across users and groups
+ domains_distinct = { for k, v in merge(local.groups_unique_idcs_endpoints, local.users_unique_idcs_endpoints) : k => distinct(v)[0]... }
+
+ # users in each domain used in groups
+ domain_users_map = {
+ for k, v in local.domains_distinct :
+ k => {
+ for user in data.oci_identity_domains_users.users[k].users :
+ user.user_name => user.id
+ } if contains(keys(local.groups_unique_idcs_endpoints), k)
+ }
+
+}
+# output "domain_distinct" {
+# value = local.domains_distinct
+# }
+# output "groups_unique_idcs_endpoints" {
+# value = local.groups_unique_idcs_endpoints
+# }
+
+# domain data for unique domin used across users and groups
+data "oci_identity_domains" "iam_domains" {
+ for_each = local.domains_distinct
+ # Required
+ compartment_id = var.compartment_ocids[each.value[0].domain_compartment_id]
+ # Optional
+ display_name = each.key
+}
+
+# user data for each used domain
+data "oci_identity_domains_users" "users" {
+ for_each = { for k, v in local.domains_distinct : k => v if contains(keys(local.groups_unique_idcs_endpoints), k) }
+ idcs_endpoint = data.oci_identity_domains.iam_domains[each.value[0].idcs_endpoint].domains[0].url
+}
+
+# output "user_map" {
+# value = local.domain_users_map
+# }
+
+
+############################
+# Module Block - Identity
+# Create Identity Domain Users
+############################
+
+module "users" {
+ source = "./modules/identity/identity-domain-user"
+ #depends_on = [module.iam-groups]
+ for_each = var.identity_domain_users
+ user_name = each.value.user_name
+ family_name = each.value.name.family_name
+ given_name = each.value.name.given_name
+ middle_name = each.value.name.middle_name
+ honorific_prefix = each.value.name.honorific_prefix
+ display_name = each.value.display_name
+ identity_domain = data.oci_identity_domains.iam_domains[each.value.idcs_endpoint].domains[0]
+ compartment_id = each.value.domain_compartment_id != "root" ? (length(regexall("ocid1.compartment.oc*", each.value.domain_compartment_id)) > 0 ? each.value.domain_compartment_id : var.compartment_ocids[each.value.domain_compartment_id]) : var.tenancy_ocid
+ description = each.value.description
+ email = each.value.email
+ recovery_email = each.value.recovery_email
+ tenancy_ocid = var.tenancy_ocid
+ groups = each.value.groups != null ? each.value.groups : null
+ home_phone_number = each.value.home_phone_number
+ mobile_phone_number = each.value.mobile_phone_number
+ enabled_capabilities = each.value.enabled_capabilities
+
+ #Optional
+ defined_tags = each.value.defined_tags
+ freeform_tags_key = each.value.freeform_tags != null ? each.value.freeform_tags.key : null
+ freeform_tags_value = each.value.freeform_tags != null ? each.value.freeform_tags.value : null
+}
\ No newline at end of file
diff --git a/cd3_automation_toolkit/ocicloud/terraform/variables_example.tf b/cd3_automation_toolkit/ocicloud/terraform/variables_example.tf
index c2154d27e..d2dd72562 100644
--- a/cd3_automation_toolkit/ocicloud/terraform/variables_example.tf
+++ b/cd3_automation_toolkit/ocicloud/terraform/variables_example.tf
@@ -905,6 +905,21 @@ variable "instances" {
vnic_freeform_tags = optional(map(any))
defined_tags = optional(map(any))
freeform_tags = optional(map(any))
+ secondary_vnics = optional(map(object({
+ network_compartment_id = optional(string)
+ vcn_name = optional(string)
+ subnet_id = string
+ display_name = optional(string)
+ private_ip = optional(string)
+ assign_public_ip = optional(bool)
+ hostname_label = optional(string)
+ assign_private_dns_record = optional(bool)
+ skip_source_dest_check = optional(bool)
+ physical_nic_index = optional(number)
+ nsg_ids = optional(list(string))
+ defined_tags = optional(map(any))
+ freeform_tags = optional(map(any))
+ })), {})
}))
default = {}
}
@@ -1822,7 +1837,6 @@ variable "clusters" {
is_public_ip_enabled = optional(bool)
nsg_ids = optional(list(string))
endpoint_subnet_id = string
- is_pod_security_policy_enabled = optional(bool)
pods_cidr = optional(string)
services_cidr = optional(string)
service_lb_subnet_ids = optional(list(string))
@@ -1853,6 +1867,12 @@ variable "nodepools" {
subnet_id = string
fault_domains = optional(list(string))
capacity_reservation_id = optional(string)
+ preemptible_node_config = optional(object({
+ preemption_action = object({
+ type = string
+ is_preserve_boot_volume = optional(bool)
+ })
+ }))
})))
kubernetes_version = string
is_pv_encryption_in_transit_enabled = optional(bool)
@@ -2360,4 +2380,3 @@ variable "fw_logs" {
# Add new variables here #
##########################
######################### END #########################
-
diff --git a/cd3_automation_toolkit/setUpCloud.py b/cd3_automation_toolkit/setUpCloud.py
index f7dae46f2..2792b8799 100644
--- a/cd3_automation_toolkit/setUpCloud.py
+++ b/cd3_automation_toolkit/setUpCloud.py
@@ -3,7 +3,7 @@
def main():
- if len(sys.argv) != 3:
+ if len(sys.argv) < 3:
print("Usage: python setUpCloud.py ")
print("Example: python setUpCloud.py azure setUpAzure.properties")
print("Example: python setUpCloud.py oci setUpOCI.properties")
@@ -11,10 +11,10 @@ def main():
return
cloud_provider = sys.argv[1].lower()
- argument = sys.argv[2]
+ #argument = sys.argv[2]
if cloud_provider == 'oci':
- script_name = 'setUpOCI.py'
+ script_name = 'user-scripts/setUpOCI.py'
elif cloud_provider == 'azure':
script_name = 'user-scripts/setUpAzure.py'
elif cloud_provider == 'gcp':
@@ -24,7 +24,7 @@ def main():
return
try:
- subprocess.run([sys.executable, script_name, argument], check=True)
+ subprocess.run([sys.executable, script_name, *sys.argv[2:]], check=True)
except subprocess.CalledProcessError as e:
pass
diff --git a/cd3_automation_toolkit/setUpGCP.properties b/cd3_automation_toolkit/setUpGCP.properties
index 7075f5001..6f945e643 100644
--- a/cd3_automation_toolkit/setUpGCP.properties
+++ b/cd3_automation_toolkit/setUpGCP.properties
@@ -1,23 +1,23 @@
[Default]
-#Input variables required to run setUpAzure script
+#Input variables required to run setUpGCP script
-#path to output directory where terraform files will be generated. eg. /cd3user/azure//terraform_files
+#path to output directory where terraform files will be generated. eg. /cd3user/gcp//terraform_files
outdir=
#prefix for output terraform files eg demo
prefix=
# Auth Params
-# auth mechanism for OCI APIs - api_key,instance_principal,session_token
+# auth mechanism for GCP APIs - api_key
auth_mechanism=api_key
-#input credentials file for service account eg /cd3user/gcp//gcp_api_private.pem
+#input credentials file for service account eg /cd3user/gcp//gcp_api_private.json
config_file=
-#path to cd3 excel eg /cd3user/azure//CD3-Customer.xlsx
+#path to cd3 excel eg /cd3user/gcp//CD3-Customer.xlsx
cd3file=
-#specify create_resources to create new resources in Azure(greenfield workflow)
-#specify export_resources to export resources from Azure(non-greenfield workflow)
+#specify create_resources to create new resources in GCP(greenfield workflow)
+#specify export_resources to export resources from GCP(non-greenfield workflow)
workflow_type=create_resources
\ No newline at end of file
diff --git a/cd3_automation_toolkit/shell_script.sh b/cd3_automation_toolkit/shell_script.sh
index c46e3822b..99b4f3e3b 100644
--- a/cd3_automation_toolkit/shell_script.sh
+++ b/cd3_automation_toolkit/shell_script.sh
@@ -19,6 +19,8 @@ pip install --user azure.mgmt.compute
pip install --user azure.mgmt.oracledatabase
pip install --user google-auth
pip install --user google-cloud-resource-manager
+pip install --user google-cloud-oracledatabase
+pip install --user google-cloud-compute
pip install --user pycryptodomex==3.10.1
pip install --user regex==2022.10.31
pip install --user numpy==1.26.4
diff --git a/cd3_automation_toolkit/user-scripts/connectGCP.py b/cd3_automation_toolkit/user-scripts/connectGCP.py
index 8c02163a3..ae6410409 100644
--- a/cd3_automation_toolkit/user-scripts/connectGCP.py
+++ b/cd3_automation_toolkit/user-scripts/connectGCP.py
@@ -11,9 +11,9 @@
import configparser
import datetime
import shutil
-import os,sys,logging
+import os, sys, logging
sys.path.append(os.getcwd())
-sys.path.append(os.getcwd()+"/..")
+sys.path.append(os.getcwd() + "/..")
from gcpcloud.python import *
from google.cloud import resourcemanager_v3
@@ -24,96 +24,93 @@
config = configparser.RawConfigParser()
config.read(args.propsfile)
-current_time=str(datetime.datetime.now())
-cloud="GCP"
+current_time = str(datetime.datetime.now())
+cloud = "GCP"
# Initialize Toolkit Variables
-user_dir = "/cd3user/"+cloud.lower()+"/"
+user_dir = "/cd3user/" + cloud.lower() + "/"
-toolkit_dir = os.path.dirname(os.path.abspath(__file__))+"/.."
-tf_modules_dir = toolkit_dir + "/"+cloud.lower()+"cloud/terraform"
-setupcloud_props_toolkit_file_path = toolkit_dir + "/setUp"+cloud+".properties"
+toolkit_dir = os.path.dirname(os.path.abspath(__file__)) + "/.."
+tf_modules_dir = toolkit_dir + "/" + cloud.lower() + "cloud/terraform"
+setupcloud_props_toolkit_file_path = toolkit_dir + "/setUp" + cloud + ".properties"
auto_keys_dir = user_dir + "/keys"
-
prefix = config.get('Default', 'prefix').strip()
if prefix == "" or prefix == "\n":
print("Invalid Prefix. Please try again......Exiting !!")
exit(1)
-prefixes=[]
-f = os.path.basename(__file__).rstrip("py")+"safe"
+prefixes = []
+f = os.path.basename(__file__).rstrip("py") + "safe"
safe_file = user_dir + f
if os.path.exists(safe_file):
- f=open(safe_file,"r")
+ f = open(safe_file, "r")
safe_file_lines = f.readlines()
for l in safe_file_lines:
if "SUCCESS" in l:
prefixes.append(l.split("\t")[0])
-if prefixes !=[]:
+if prefixes != []:
if prefix in prefixes:
- print("WARNING!!! Container has already been successfuly connected to "+cloud+" with same prefix. Please proceed only if you re-running the script for new project")
+ print("WARNING!!! Container has already been successfuly connected to " + cloud + " with same prefix. Please proceed only if you re-running the script for new project")
inp = input("\nDo you want to proceed (y/n):")
- if inp.lower()=="n":
+ if inp.lower() == "n":
exit(1)
# Initialize Tenancy Variables
-prefix_dir = user_dir +"/" + prefix
-config_files= prefix_dir +"/.config_files"
+prefix_dir = user_dir + "/" + prefix
+config_files = prefix_dir + "/.config_files"
terraform_files = prefix_dir + "/terraform_files"
gcp_provider_file = terraform_files + "/provider.tf"
-setupcloud_props_file_path = prefix_dir + "/"+prefix+"_setUp"+cloud+".properties"
+setupcloud_props_file_path = prefix_dir + "/" + prefix + "_setUp" + cloud + ".properties"
connected = 1
# Read Config file Variables
try:
- config_file=''
+ config_file = ''
config_file = config.get('Default', 'config_file').strip()
if config_file == "" or config_file == "\n":
- config_file = auto_keys_dir +"/gcp_api_private.json"
+ config_file = auto_keys_dir + "/gcp_api_private.json"
if not os.path.exists(config_file):
- connected=0
+ connected = 0
except Exception as e:
print(e)
print('Check if input properties exist and try again..exiting...')
exit(1)
-
if not os.path.exists(prefix_dir):
os.makedirs(prefix_dir)
if not os.path.exists(config_files):
os.makedirs(config_files)
-excel_name="CD3-"+cloud.upper()+"-template.xlsx"
+excel_name = "CD3-" + cloud.upper() + "-template.xlsx"
shutil.copy(os.path.join(toolkit_dir, 'example', excel_name), os.path.join(prefix_dir, excel_name))
# 1. Copy input properties file
-shutil.copy(args.propsfile,config_files+"/"+prefix+"_"+os.path.basename(args.propsfile))
+shutil.copy(args.propsfile, config_files + "/" + prefix + "_" + os.path.basename(args.propsfile))
# 2. Authenticate and Get Projects
if connected == 1:
gct = gcpCommonTools()
+ credentials, active_project_ids, regions = gct.authenticate(config_file)
# Copy onfig file to customer_tenancy_dir
filename = os.path.basename(config_file)
- _config_file=config_files + "/" + filename
+ _config_file = config_files + "/" + filename
shutil.copy(config_file, _config_file)
- os.chmod(_config_file,0o600)
- credentials = gct.authenticate(config_file)
+ os.chmod(_config_file, 0o600)
try:
- project_id=credentials.project_id
+ project_id = credentials.project_id
client = resourcemanager_v3.ProjectsClient(credentials=credentials)
# Request project details using the project ID
# The name parameter must follow the format: projects/{project_id}
request = resourcemanager_v3.GetProjectRequest(name=f"projects/{project_id}")
project = client.get_project(request=request)
-
# 'display_name' contains the human-readable Project Name
project_name = project.display_name
except Exception as e:
@@ -123,17 +120,16 @@
exit(1)
# 3. Generate setUpCloud.properties file
-print("Creating "+cloud+" specific setUp"+cloud+".properties.................")
+print("Creating " + cloud + " specific setUp" + cloud + ".properties.................")
with open(setupcloud_props_toolkit_file_path, 'r+') as setUpCloud_file:
setupcloud_props_toolkit_file_data = setUpCloud_file.read().rstrip()
-setupcloud_props_toolkit_file_data = setupcloud_props_toolkit_file_data.replace("outdir=", "outdir="+terraform_files)
-setupcloud_props_toolkit_file_data = setupcloud_props_toolkit_file_data.replace("prefix=", "prefix="+prefix)
+setupcloud_props_toolkit_file_data = setupcloud_props_toolkit_file_data.replace("outdir=", "outdir=" + terraform_files)
+setupcloud_props_toolkit_file_data = setupcloud_props_toolkit_file_data.replace("prefix=", "prefix=" + prefix)
setupcloud_props_toolkit_file_data = setupcloud_props_toolkit_file_data.replace("cd3file=","cd3file=" + os.path.join(prefix_dir,excel_name))
-
-if connected ==1:
- setupcloud_props_toolkit_file_data = setupcloud_props_toolkit_file_data.replace("config_file=", "config_file="+_config_file)
+if connected == 1:
+ setupcloud_props_toolkit_file_data = setupcloud_props_toolkit_file_data.replace("config_file=", "config_file=" + _config_file)
f = open(setupcloud_props_file_path, "w+")
f.write(setupcloud_props_toolkit_file_data)
@@ -141,11 +137,10 @@
# Copy modules dir to terraform_files folder
try:
- shutil.copytree(tf_modules_dir, terraform_files )
+ shutil.copytree(tf_modules_dir, terraform_files)
except FileExistsError as fe:
print(fe)
-
if connected == 1:
print("Creating project specific terraform provider file.................")
with open(gcp_provider_file, 'r+') as provider_file:
@@ -157,26 +152,25 @@
f.write(gcp_provider_file_data)
f.close()
-
# Logging information
-outfile = prefix_dir+'/'+os.path.basename(__file__).rstrip("py")+"out"
+outfile = prefix_dir + '/' + os.path.basename(__file__).rstrip("py") + "out"
logging.basicConfig(filename=outfile, format='%(message)s', filemode='w', level=logging.INFO)
print("==================================================================================================================================")
print("\nThe toolkit has been setup successfully. !!!\n")
f = open(safe_file, "a")
-data=prefix + "\t" + "SUCCESS\t"+current_time+"\n"
+data = prefix + "\t" + "SUCCESS\t" + current_time + "\n"
f.write(data)
f.close()
-logging.info("Prefix Specific Working Directory Path: "+prefix_dir+"\n")
+logging.info("Prefix Specific Working Directory Path: " + prefix_dir + "\n")
logging.info("\n######################################")
logging.info("Next Steps for using toolkit via CLI")
logging.info("######################################")
-logging.info("Modify "+prefix_dir + "/" +prefix+"_setUp"+cloud+".properties with input values for cd3file and workflow_type")
+logging.info("Modify " + prefix_dir + "/" + prefix + "_setUp" + cloud + ".properties with input values for cd3file and workflow_type")
logging.info("cd /cd3user/oci_tools/cd3_automation_toolkit")
-logging.info("python setUpCloud.py "+cloud.lower()+" "+prefix_dir + "/" +prefix+"_setUp"+cloud+".properties")
+logging.info("python setUpCloud.py " + cloud.lower() + " " + prefix_dir + "/" + prefix + "_setUp" + cloud + ".properties")
with open(outfile, 'r') as log_file:
data = log_file.read().rstrip()
diff --git a/cd3_automation_toolkit/user-scripts/connectOCI.py b/cd3_automation_toolkit/user-scripts/connectOCI.py
index a5abb2177..29b136a5a 100644
--- a/cd3_automation_toolkit/user-scripts/connectOCI.py
+++ b/cd3_automation_toolkit/user-scripts/connectOCI.py
@@ -1168,7 +1168,7 @@ def create_bucket(config, signer):
if service_dir=="" or service_dir == "\n":
continue
#if (service != 'identity' and service != 'tagging') or ((service == 'identity' or service == 'tagging') and region == home_region):
- home_region_services = ['identity', 'tagging', 'budget','quota']
+ home_region_services = ['identity','compartment','policy','user','group','networksource', 'tagging', 'budget','quota']
if (region != home_region) and (service in home_region_services):
os.remove(os.path.join(region_dir, f"{service}.tf"))
diff --git a/cd3_automation_toolkit/user-scripts/outdir_structure_file.properties b/cd3_automation_toolkit/user-scripts/outdir_structure_file.properties
index c0ea27945..b0baa03f7 100644
--- a/cd3_automation_toolkit/user-scripts/outdir_structure_file.properties
+++ b/cd3_automation_toolkit/user-scripts/outdir_structure_file.properties
@@ -8,7 +8,13 @@
# You will have to run createTenancy.py from scratch if you want to make any changes to the directory structure later.
# It is mandatory to specify the directory name for each service.
-identity=identity
+
+compartment=identity
+policy=identity
+# Same Directory must be specified for users and groups
+user=identity
+group=identity
+networksource=identity
tagging=tagging
network=network
quota=quota
diff --git a/cd3_automation_toolkit/user-scripts/setUpAzure.py b/cd3_automation_toolkit/user-scripts/setUpAzure.py
index 529cfb6e7..df319dee7 100644
--- a/cd3_automation_toolkit/user-scripts/setUpAzure.py
+++ b/cd3_automation_toolkit/user-scripts/setUpAzure.py
@@ -97,12 +97,6 @@ def create_db_at_azure(execute_all=False):
if not execute_all:
execute_options(options)
-'''
-def export_az_adb():
- export_az_oci_adb(inputfile, outdir, credentials)
-'''
-def export_az_oci_exa():
- export_az_oci_adb(inputfile, outdir, credentials)
def export_db_at_azure(execute_all=False):
options = [
@@ -132,8 +126,8 @@ def export_db_at_azure(execute_all=False):
#parser.add_argument('--add_filter', default=None)
#parser.add_argument('--devops', default=False)
args = parser.parse_args()
-setUpAz_props = configparser.RawConfigParser()
-setUpAz_props.read(args.propsfile)
+setUp_props = configparser.RawConfigParser()
+setUp_props.read(args.propsfile)
#devops = args.devops
#main_options = args.main_options.split(",")
#sub_options = args.sub_options.split(",")
@@ -141,16 +135,16 @@ def export_db_at_azure(execute_all=False):
#Read Config file Variables
try:
- workflow_type = setUpAz_props.get('Default', 'workflow_type').strip().lower()
+ workflow_type = setUp_props.get('Default', 'workflow_type').strip().lower()
if (workflow_type == 'export_resources'):
non_gf_tenancy = True
else:
non_gf_tenancy = False
- inputfile = setUpAz_props.get('Default','cd3file').strip()
- outdir = setUpAz_props.get('Default', 'outdir').strip()
- prefix = setUpAz_props.get('Default', 'prefix').strip()
+ inputfile = setUp_props.get('Default','cd3file').strip()
+ outdir = setUp_props.get('Default', 'outdir').strip()
+ prefix = setUp_props.get('Default', 'prefix').strip()
tf_or_tofu = "terraform"
if not outdir:
diff --git a/cd3_automation_toolkit/user-scripts/setUpGCP.py b/cd3_automation_toolkit/user-scripts/setUpGCP.py
index a0b3b1570..6a6528fae 100644
--- a/cd3_automation_toolkit/user-scripts/setUpGCP.py
+++ b/cd3_automation_toolkit/user-scripts/setUpGCP.py
@@ -80,8 +80,8 @@ def execute_options(options, *args, **kwargs):
option.callback(*args, **kwargs)
-#def create_adb_azure():
-# create_terraform_adb_azure(inputfile, outdir, prefix)
+def create_adb_gcp():
+ create_terraform_adb_gcp(inputfile, outdir, prefix)
def create_exa_gcp():
create_terraform_exa_infra_gcp(inputfile, outdir, prefix)
@@ -90,31 +90,34 @@ def create_exa_gcp():
def create_db_at_gcp(execute_all=False):
options = [
- #Option('Add/Modify/Delete ADB @Azure', create_adb_azure, 'Processing ADB-Azure Tab'),
+ Option('Add/Modify/Delete ADB @GCP', create_adb_gcp, 'Processing ADB-GCP Tab'),
Option('Add/Modify/Delete Exa @GCP', create_exa_gcp, 'Processing Exa-GCP Tabs')
]
options = show_options(options, quit=True, menu=True, index=1)
if not execute_all:
execute_options(options)
-'''
-def export_az_adb():
- export_az_oci_adb(inputfile, outdir, credentials)
-def export_az_oci_exa():
- export_az_oci_adb(inputfile, outdir, credentials)
+def export_adb_at_gcp(inputfile, outdir, credentials, active_projects,regions):
+ export_adb_gcp(inputfile, outdir, credentials, active_projects,regions)
+ create_terraform_adb_gcp(inputfile, outdir, prefix)
+
+def export_exa_infra_at_gcp(inputfile, outdir, credentials, active_projects,regions):
+ export_exa_infra_gcp(inputfile, outdir, credentials, active_projects,regions)
+ create_terraform_exa_infra_gcp(inputfile, outdir, prefix)
+ create_terraform_exa_vmclusters_gcp(inputfile, outdir, prefix)
-def export_db_at_azure(execute_all=False):
+def export_db_at_gcp(execute_all=False):
options = [
- Option('Export ADB @Azure', export_adb_azure, 'Exporting ADB-Azure'),
- # Option('Export Exa @Azure', export_az_oci_exa, 'Processing Exa-Azure')
+ Option('Export ADB @GCP', export_adb_at_gcp, 'Exporting ADB-GCP'),
+ #Option('Export Exa @GCP', export_exa_infra_at_gcp, 'Exporting Exa-Infra-GCP')
]
options = show_options(options, quit=True, menu=True, index=1)
if not execute_all:
- execute_options(options,inputfile, outdir, credentials)
+ execute_options(options,inputfile, outdir, credentials, active_projects,regions)
+
+
- create_terraform_adb_azure(inputfile, outdir, prefix)
-'''
#Execution starts here
global devops
@@ -132,8 +135,8 @@ def export_db_at_azure(execute_all=False):
#parser.add_argument('--add_filter', default=None)
#parser.add_argument('--devops', default=False)
args = parser.parse_args()
-setUpAz_props = configparser.RawConfigParser()
-setUpAz_props.read(args.propsfile)
+setUp_props = configparser.RawConfigParser()
+setUp_props.read(args.propsfile)
#devops = args.devops
#main_options = args.main_options.split(",")
#sub_options = args.sub_options.split(",")
@@ -141,16 +144,17 @@ def export_db_at_azure(execute_all=False):
#Read Config file Variables
try:
- workflow_type = setUpAz_props.get('Default', 'workflow_type').strip().lower()
+ workflow_type = setUp_props.get('Default', 'workflow_type').strip().lower()
if (workflow_type == 'export_resources'):
non_gf_tenancy = True
else:
non_gf_tenancy = False
- inputfile = setUpAz_props.get('Default','cd3file').strip()
- outdir = setUpAz_props.get('Default', 'outdir').strip()
- prefix = setUpAz_props.get('Default', 'prefix').strip()
+ inputfile = setUp_props.get('Default','cd3file').strip()
+ outdir = setUp_props.get('Default', 'outdir').strip()
+ prefix = setUp_props.get('Default', 'prefix').strip()
+ config_file = setUp_props.get('Default', 'config_file').strip()
tf_or_tofu = "terraform"
if not outdir:
@@ -183,7 +187,7 @@ def export_db_at_azure(execute_all=False):
if non_gf_tenancy:
ct = gcpCommonTools()
- credentials = ct.authenticate(args.propsfile)
+ credentials,active_projects,regions = ct.authenticate(config_file)
# verify_outdir_is_empty()
print("\nworkflow_type set to export_resources. Export existing GCP objects and Synch with TF state")
@@ -193,7 +197,7 @@ def export_db_at_azure(execute_all=False):
#export_tags_list = get_tags_list()
inputs = [
- #Option("Export DB @Azure", export_db_at_azure, "Export DB @Azure"),
+ Option("Export DB @GCP", export_db_at_gcp, "Export DB @GCP"),
]
diff --git a/cd3_automation_toolkit/setUpOCI.py b/cd3_automation_toolkit/user-scripts/setUpOCI.py
similarity index 98%
rename from cd3_automation_toolkit/setUpOCI.py
rename to cd3_automation_toolkit/user-scripts/setUpOCI.py
index 20a01eeaf..06e0fca07 100644
--- a/cd3_automation_toolkit/setUpOCI.py
+++ b/cd3_automation_toolkit/user-scripts/setUpOCI.py
@@ -5,9 +5,12 @@
import requests
import subprocess
import glob
+import sys,os
+sys.path.append(os.getcwd())
+sys.path.append(os.getcwd()+"/..")
from ocicloud.python import *
from common.python import *
-
+
def show_firewall_options(options, quit=False, menu=False, extra=None, index=0):
# Just add whitespace between number and option. It just makes it look better
number_offset = len(str(len(options))) + 1
@@ -238,7 +241,7 @@ def fetch_compartments(outdir, outdir_struct, ct):
ct.get_network_compartment_ids(config['tenancy'], "root", config, signer)
ct.all_regions.append('global')
print("\nWriting to variables files...")
- home_region_services = ['identity', 'tagging', 'budget', 'quota']
+ home_region_services = ['identity', 'compartment', 'policy', 'user', 'group', 'networksource', 'tagging', 'budget','quota']
for region in ct.all_regions:
# for global directory
if region == 'global':
@@ -374,12 +377,12 @@ def export_identityOptions(prim_options=[]):
options = show_options(options, quit=True, menu=True, index=1)
execute_options(options, inputfile, outdir, config, signer, ct)
# Update modified path list
- update_path_list(regions_path=[ct.home_region], service_dirs=[service_dir_identity])
+ update_path_list(regions_path=[ct.home_region], service_dirs=[service_dir_user,service_dir_compartment,service_dir_group,service_dir_networksource])
def export_compartments(inputfile, outdir,config, signer, ct):
resource = 'Compartments'
- export_identity(inputfile, outdir, service_dir_identity,resource, config, signer, ct)
+ export_identity(inputfile, outdir, service_dir_compartment,resource, config, signer, ct)
options = [Option(None, create_compartments, 'Processing Compartments Tab'), ]
execute_options(options)
print("\n\nExecute import_commands_compartments.sh script created under home region directory to synch TF with OCI Identity Compartments\n")
@@ -387,7 +390,7 @@ def export_compartments(inputfile, outdir,config, signer, ct):
def export_policies(inputfile, outdir,config, signer, ct):
resource = 'IAM Policies'
#compartments = ct.get_compartment_map(var_file, resource)
- export_identity(inputfile, outdir, service_dir_identity,resource, config, signer, ct, export_compartments=compartments)
+ export_identity(inputfile, outdir, service_dir_policy,resource, config, signer, ct, export_compartments=compartments)
options = [Option(None, create_policies, 'Processing Policies Tab'), ]
execute_options(options)
print("\n\nExecute import_commands_policies.sh script created under home region directory to synch TF with OCI " +resource +"\n")
@@ -395,7 +398,7 @@ def export_policies(inputfile, outdir,config, signer, ct):
def export_groups(inputfile, outdir,config, signer, ct):
resource = 'IAM Groups'
selected_domains_data = ct.get_identity_domain_data(config, signer, resource,var_file)
- export_identity(inputfile, outdir, service_dir_identity,resource, config, signer, ct, export_domains=selected_domains_data)
+ export_identity(inputfile, outdir, service_dir_group,resource, config, signer, ct, export_domains=selected_domains_data)
options = [Option(None, create_groups, 'Processing Groups Tab'), ]
execute_options(options)
print("\n\nExecute import_commands_groups.sh script created under home region directory to synch TF with OCI " +resource +"\n")
@@ -405,17 +408,17 @@ def export_users(inputfile, outdir,config,signer, ct):
resource = 'IAM Users'
# check if tenancy is identity_domain enabled
selected_domains_data = ct.get_identity_domain_data(config, signer, resource,var_file)
- identity.export_users(inputfile, outdir, service_dir_identity, config, signer, ct,export_domains=selected_domains_data)
+ identity.export_users(inputfile, outdir, service_dir_user, config, signer, ct,export_domains=selected_domains_data)
options = [Option(None, create_terraform_users, 'Processing Users Tab'), ]
- execute_options(options,inputfile, outdir,service_dir_identity, prefix, ct)
+ execute_options(options,inputfile, outdir,service_dir_user, prefix, ct)
print("\n\nExecute import_commands_users.sh script created under home region directory to synch TF with OCI " +resource +"\n")
def export_networkSources(inputfile, outdir, config, signer, ct):
resource = 'Network Sources'
- identity.export_networkSources(inputfile, outdir, service_dir_identity, config, signer, ct)
+ identity.export_networkSources(inputfile, outdir, service_dir_networksource, config, signer, ct)
options = [Option(None, create_terraform_networkSources, 'Processing NetworkSources Tab'), ]
- execute_options(options, inputfile, outdir, service_dir_identity, prefix, ct)
+ execute_options(options, inputfile, outdir, service_dir_networksource, prefix, ct)
print("\n\nExecute import_commands_networkSources.sh script created under home region directory to synch TF with OCI " +resource +"\n")
def export_governance(prim_options=[]):
@@ -1010,9 +1013,9 @@ def create_compartments():
choices = [Option("Validate Compartments", None, None)]
errors = cd3Validator.validate_cd3(choices, inputfile, var_file, prefix, outdir, ct)
if errors == False:
- create_terraform_compartments(inputfile, outdir, service_dir_identity, prefix, ct)
+ create_terraform_compartments(inputfile, outdir, service_dir_compartment, prefix, ct)
# Update modified path list
- update_path_list(regions_path=[ct.home_region], service_dirs=[service_dir_identity])
+ update_path_list(regions_path=[ct.home_region], service_dirs=[service_dir_compartment])
else:
print("Please correct the errors in CD3 Sheet and try again. Exiting!!!")
exit(1)
@@ -1023,9 +1026,9 @@ def create_groups():
choices = [Option("Validate Groups", None, None)]
errors = cd3Validator.validate_cd3(choices, inputfile, var_file, prefix, outdir, ct)
if errors == False:
- create_terraform_groups(inputfile, outdir, service_dir_identity, prefix, ct)
+ create_terraform_groups(inputfile, outdir, service_dir_group, prefix, ct)
# Update modified path list
- update_path_list(regions_path=[ct.home_region], service_dirs=[service_dir_identity])
+ update_path_list(regions_path=[ct.home_region], service_dirs=[service_dir_group])
else:
print("Please correct the errors in CD3 Sheet and try again. Exiting!!!")
exit(1)
@@ -1037,23 +1040,23 @@ def create_policies():
errors = cd3Validator.validate_cd3(choices, inputfile, var_file, prefix, outdir, ct)
if errors == False:
- create_terraform_policies(inputfile, outdir, service_dir_identity, prefix, ct)
+ create_terraform_policies(inputfile, outdir, service_dir_policy, prefix, ct)
# Update modified path list
- update_path_list(regions_path=[ct.home_region], service_dirs=[service_dir_identity])
+ update_path_list(regions_path=[ct.home_region], service_dirs=[service_dir_policy])
else:
print("Please correct the errors in CD3 Sheet and try again. Exiting!!!")
exit(1)
def create_users():
- create_terraform_users(inputfile, outdir, service_dir_identity, prefix, ct)
+ create_terraform_users(inputfile, outdir, service_dir_user, prefix, ct)
# Update modified path list
- update_path_list(regions_path=[ct.home_region], service_dirs=[service_dir_identity])
+ update_path_list(regions_path=[ct.home_region], service_dirs=[service_dir_user])
def create_networksources():
- create_terraform_networkSources(inputfile, outdir, service_dir_identity, prefix, ct)
+ create_terraform_networkSources(inputfile, outdir, service_dir_networksource, prefix, ct)
# Update modified path list
- update_path_list(regions_path=[ct.home_region], service_dirs=[service_dir_identity])
+ update_path_list(regions_path=[ct.home_region], service_dirs=[service_dir_networksource])
def create_identity(prim_options=[]):
@@ -1667,10 +1670,11 @@ def initiate_cis_scan(outdir, prefix, config_file,sub_options=[]):
def start_cis_download(outdir, prefix, config_file):
current_dir=os.path.dirname(os.path.abspath(__file__))
- print("Downloading the script file as 'cis_reports.py' at location "+current_dir+"/../othertools/")
+ file_path=current_dir+"/../../othertools/"
+ print("Downloading the script file as 'cis_reports.py' at location "+file_path)
resp = requests.get("https://raw.githubusercontent.com/oracle-quickstart/oci-cis-landingzone-quickstart/main/scripts/cis_reports.py")
resp_contents = resp.text
- with open(current_dir+"/../othertools/cis_reports.py", "w", encoding="utf-8") as fd:
+ with open(file_path+"/cis_reports.py", "w", encoding="utf-8") as fd:
fd.write(resp_contents)
#cmdpath = os.path.dirname(os.path.abspath(__file__)) + "/../"
@@ -1678,7 +1682,7 @@ def start_cis_download(outdir, prefix, config_file):
print("Download complete!!")
def start_cis_scan(outdir, prefix, config_file):
- cmdpath = os.path.dirname(os.path.abspath(__file__))+ "/../othertools/"
+ cmdpath = os.path.dirname(os.path.abspath(__file__))+ "/../../othertools/"
cmd = "python "+cmdpath+"cis_reports.py"
if auth_mechanism == "instance_principal":
@@ -1714,9 +1718,10 @@ def start_cis_scan(outdir, prefix, config_file):
def get_latest_showoci(outdir, prefix,config_file):
current_dir = os.path.dirname(os.path.abspath(__file__))
- print("Getting latest showoci report script at location "+current_dir+"/../othertools/")
+ file_path=current_dir+"/../../othertools/"
+ print("Getting latest showoci report script at location "+file_path)
- cmdpath=os.path.dirname(os.path.abspath(__file__)) + "/../othertools/"
+ cmdpath=file_path
tag= "oci-python-sdk"
if (os.path.isdir(cmdpath+tag)):
@@ -1729,7 +1734,7 @@ def get_latest_showoci(outdir, prefix,config_file):
def execute_showoci(outdir, prefix, config_file_path):
- cmdpath = os.path.dirname(os.path.abspath(__file__)) + "/../othertools/"
+ cmdpath = os.path.dirname(os.path.abspath(__file__)) + "/../../othertools/"
tag = "oci-python-sdk"
if not os.path.isfile(cmdpath+tag+"/examples/showoci/showoci.py"):
get_latest_showoci(outdir, prefix, config_file=config_file_path)
@@ -1776,7 +1781,7 @@ def run_showoci(outdir, prefix, config_file,sub_options=[]):
execute_options(options, outdir, prefix, config_file)
def run_vizoci(outdir, prefix, config_file,sub_options=[]):
- cmdpath = os.path.dirname(os.path.abspath(__file__)) + "/../othertools/"
+ cmdpath = os.path.dirname(os.path.abspath(__file__)) + "/../../othertools/"
tag = "vizoci"
cwd= os.getcwd()
os.chdir(cmdpath + tag)
@@ -1869,7 +1874,7 @@ def export_update_dr_plan(outdir, prefix, config_file_path,option=''):
input1 = "Please enter excel file name from where DR plan will be updated; Leave blank to read file with name '$prefix_" + tag + "-plan.xlsx': "
input2 = "Please enter sheet name in the excel from where DR plan will be updated(without spaces); Leave blank to read sheet with name 'FSDR-Plan': "
- cmdpath = os.path.dirname(os.path.abspath(__file__))+"/../othertools/"+tag+"/"+c+"_drplan.py"
+ cmdpath = os.path.dirname(os.path.abspath(__file__))+"/../../othertools/"+tag+"/"+c+"_drplan.py"
if not os.path.isfile(cmdpath):
print(cmdpath+" doesnt exist. Exiitng!!!")
@@ -2253,7 +2258,7 @@ def create_firewall(inputfile, outdir, service_dir, prefix, ct,sub_options=[]):
exec(varname + "= value",globals())
#print("Reading varfile for compartments....\n")
-var_file = (f'{outdir}/{home_region}/{service_dir_identity}/variables_{home_region}.tf').replace('//','/')
+var_file = (f'{outdir}/{home_region}/{service_dir_compartment}/variables_{home_region}.tf').replace('//','/')
try:
# read variables file
diff --git a/jenkins_install/setUpOCI.groovy b/jenkins_install/setUpOCI.groovy
index 4c9cdc6a0..4dda0247d 100644
--- a/jenkins_install/setUpOCI.groovy
+++ b/jenkins_install/setUpOCI.groovy
@@ -1,13 +1,14 @@
def buildstatus = ""
+
def git_status = 0
def prefix = "${env.JOB_NAME}".split('/')[0]
def exportNetworkRules(stage_name) {
return {
stage("${stage_name}") {
catchError(buildResult: 'FAILURE', stageResult: 'FAILURE') {
- labelledShell( label: 'Executing setUpOCI python script', script: """
+ labelledShell( label: 'Executing setUpCloud python script', script: """
cd /cd3user/oci_tools/cd3_automation_toolkit
- python setUpOCI.py --devops True --main_options "Network" --sub_options "Security Rules,Route Rules,DRG Route Rules" --sub_child_options "Export Security Rules (From OCI into SecRulesinOCI sheet),Add/Modify/Delete Security Rules (Reads SecRulesinOCI sheet),Export Route Rules (From OCI into RouteRulesinOCI sheet),Add/Modify/Delete Route Rules (Reads RouteRulesinOCI sheet),Export DRG Route Rules (From OCI into DRGRouteRulesinOCI sheet),Add/Modify/Delete DRG Route Rules (Reads DRGRouteRulesinOCI sheet)" --add_filter "comp_filter=,[],@," ${env.prop_file}
+ python setUpCloud.py oci --devops True --main_options "Network" --sub_options "Security Rules,Route Rules,DRG Route Rules" --sub_child_options "Export Security Rules (From OCI into SecRulesinOCI sheet),Add/Modify/Delete Security Rules (Reads SecRulesinOCI sheet),Export Route Rules (From OCI into RouteRulesinOCI sheet),Add/Modify/Delete Route Rules (Reads RouteRulesinOCI sheet),Export DRG Route Rules (From OCI into DRGRouteRulesinOCI sheet),Add/Modify/Delete DRG Route Rules (Reads DRGRouteRulesinOCI sheet)" --add_filter "comp_filter=,[],@," ${env.prop_file}
""")
script {
git_status = labelledShell( label: 'Check git status', script: 'cd ${prefix_dir}/terraform_files; git status --porcelain | wc -l', returnStdout: true).trim()
@@ -18,7 +19,7 @@ def exportNetworkRules(stage_name) {
cd ${prefix_dir}/terraform_files
echo "-----start timestamp-----"
time_stamp="$(date +%m-%d-%Y-%H-%M-%S)"
- commit_msg="commit for setUpOCI build ${BUILD_NUMBER}"
+ commit_msg="commit for setUpCloud build ${BUILD_NUMBER}"
git add -A .
git commit -m "${commit_msg}"
git push origin develop
@@ -166,7 +167,7 @@ pipeline {
parameters {
stashedFile (
name: 'Excel_Template',
- description: "Upload input Excel file.\nPreviously uploaded file will be used if left empty."
+ description: "Upload input Excel file.\nPreviously uploaded file/Blank template (for first run) will be used if left empty"
)
}
stages {
@@ -257,7 +258,7 @@ pipeline {
}
}
}
- stage('Execute setUpOCI') {
+ stage('Execute setUpCloud') {
when{
allOf {
expression {return ParametersValidation == "Passed" }
@@ -268,10 +269,10 @@ pipeline {
steps {
catchError(buildResult: 'FAILURE', stageResult: 'FAILURE') {
- labelledShell( label: 'Executing setUpOCI python script', script: '''
+ labelledShell( label: 'Executing setUpCloud python script', script: '''
set +x
cd /cd3user/oci_tools/cd3_automation_toolkit
- python setUpOCI.py --devops True --main_options "${MainOptions}" --sub_options "${SubOptions}" --sub_child_options "${SubChildOptions}" --add_filter "${AdditionalFilters}" $prop_file
+ python setUpCloud.py oci --devops True --main_options "${MainOptions}" --sub_options "${SubOptions}" --sub_child_options "${SubChildOptions}" --add_filter "${AdditionalFilters}" $prop_file
cd -
rm -rf *.*
# Check for cis_reports, show_oci and vizoci directories
@@ -397,7 +398,7 @@ pipeline {
cd ${prefix_dir}/terraform_files
echo "-----start timestamp-----"
time_stamp="$(date +%m-%d-%Y-%H-%M-%S)"
- commit_msg="commit for setUpOCI build ${BUILD_NUMBER}"
+ commit_msg="commit for setUpCloud build ${BUILD_NUMBER}"
git add -A .
git commit -m "${commit_msg}"
git push origin develop
diff --git a/othertools/cis_reports.py b/othertools/cis_reports.py
index b774798da..2639a2425 100644
--- a/othertools/cis_reports.py
+++ b/othertools/cis_reports.py
@@ -41,9 +41,11 @@
except Exception:
OUTPUT_DIAGRAMS = False
-RELEASE_VERSION = "3.1.2"
-PYTHON_SDK_VERSION = "2.165.1"
-UPDATED_DATE = "January 22, 2025"
+csv.field_size_limit(sys.maxsize)
+
+RELEASE_VERSION = "3.2.1"
+PYTHON_SDK_VERSION = "2.173.0"
+UPDATED_DATE = "May 1, 2026"
##########################################################################
@@ -154,11 +156,16 @@ class CIS_Report:
__days_to_expiry = 30
__days_used = 45
+ # Global Service Limit threshold
+ __service_utilization_limit = 80.0
+
# Time Format
__iso_time_format = "%Y-%m-%dT%H:%M:%S"
__oci_ocid_pattern = r'ocid1.[a-z0-9_]+.[a-z0-9]+.(?:[a-z0-9._-]+.|.)[a-z0-9]{20,}'
+ __simple_email = r'[a-zA-Z0-9.!#$%&\'*+/=?^_`{|}~-]+@[a-zA-Z0-9-]+(?:\.[a-zA-Z0-9-]+)*\.[a-zA-Z]{2,}'
+
# Start print time info
start_datetime = datetime.datetime.now().replace(tzinfo=pytz.UTC)
start_time_str = str(start_datetime.strftime(__iso_time_format))
@@ -175,12 +182,12 @@ class CIS_Report:
datetime.timedelta(days=__KMS_DAYS_OLD)
str_kms_key_time_max_datetime = kms_key_time_max_datetime.strftime(__iso_time_format)
kms_key_time_max_datetime = datetime.datetime.strptime(str_kms_key_time_max_datetime, __iso_time_format)
- # For Certificates Check
+ # For Certificates Check
cert_key_time_max_datetime = start_datetime + \
datetime.timedelta(days=__days_to_expiry)
str_cert_key_time_max_datetime = cert_key_time_max_datetime.strftime(__iso_time_format)
cert_key_time_max_datetime = datetime.datetime.strptime(str_cert_key_time_max_datetime, __iso_time_format)
- # For Unused Credentials Check
+ # For Unused Credentials Check
local_user_time_max_datetime = start_datetime - \
datetime.timedelta(days=__days_used)
str_local_user_time_max_datetime = local_user_time_max_datetime.strftime(__iso_time_format)
@@ -192,7 +199,7 @@ def __init__(self, config, signer, proxy, output_bucket, report_directory, repor
disable_api_keys=False):
# CIS Foundation benchmark 3.0.0
- self.cis_foundations_benchmark_3_0 = {
+ self.cis_foundations_benchmark_3_0 = {
'1.1' : {'id': 'IAM-1', 'section': 'Identity and Access Management', 'recommendation_#': '1.1', 'Title': 'Ensure service level admins are created to manage resources of particular service', 'Status': True, 'Level': 1, 'Total': [], 'Findings': [], 'Remediation': []},
'1.2' : {'id': 'IAM-2', 'section': 'Identity and Access Management', 'recommendation_#': '1.2', 'Title': 'Ensure permissions on all resources are given only to the tenancy administrator group', 'Status': True, 'Level': 1, 'Total': [], 'Findings': [], 'Remediation': []},
'1.3' : {'id': 'IAM-3', 'section': 'Identity and Access Management', 'recommendation_#': '1.3', 'Title': 'Ensure IAM administrators cannot update tenancy Administrators group', 'Status': True, 'Level': 1, 'Total': [], 'Findings': [], 'Remediation': []},
@@ -685,14 +692,20 @@ def __init__(self, config, signer, proxy, output_bucket, report_directory, repor
# OBP Checks
self.obp_foundations_checks = {
- 'SIEM_Audit_Log_All_Comps': {'id': 'OBP-SIEM-1', 'section': "SIEM Logging", 'Title': 'All compartment audit logs sent to SIEM', 'Status': True, 'Findings': [], 'OBP': [], "Documentation": "https://docs.oracle.com/en/solutions/oci-aggregate-logs-siem/index.html"}, # Assuming True
- 'SIEM_Audit_Incl_Sub_Comp': {'id': 'OBP-SIEM-2', 'section': "SIEM Logging", 'Title': 'Include all sub compartments', 'Status': True, 'Findings': [], 'OBP': [], "Documentation": "https://docs.oracle.com/en/solutions/oci-aggregate-logs-siem/index.html"}, # Assuming True
+ 'SIEM_Audit_Log_All_Comps': {'id': 'OBP-SIEM-1', 'section': "SIEM Logging", 'Title': 'All compartment audit logs sent to SIEM in all regions', 'Status': True, 'Findings': [], 'OBP': [], "Documentation": "https://docs.oracle.com/en/solutions/oci-aggregate-logs-siem/index.html"}, # Assuming True
+ 'SIEM_Audit_Incl_Sub_Comp': {'id': 'OBP-SIEM-2', 'section': "SIEM Logging", 'Title': 'Include all sub-compartments flag checked at the tenancy root compartment', 'Status': True, 'Findings': [], 'OBP': [], "Documentation": "https://docs.oracle.com/en/solutions/oci-aggregate-logs-siem/index.html"}, # Assuming True
'SIEM_VCN_Flow_Logging': {'id': 'OBP-SIEM-3', 'section': "SIEM Logging", 'Title': 'VCN Flow logs sent to SIEM', 'Status': None, 'Findings': [], 'OBP': [], "Documentation": "https://docs.oracle.com/en/solutions/oci-aggregate-logs-siem/index.html"},
'SIEM_Write_Bucket_Logs': {'id': 'OBP-SIEM-4', 'section': "SIEM Logging", 'Title': 'Bucket write logs sent to SIEM', 'Status': None, 'Findings': [], 'OBP': [], "Documentation": "https://docs.oracle.com/en/solutions/oci-aggregate-logs-siem/index.html"},
'SIEM_Read_Bucket_Logs': {'id': 'OBP-SIEM-5', 'section': "SIEM Logging", 'Title': 'Bucket read logs sent to SIEM', 'Status': None, 'Findings': [], 'OBP': [], "Documentation": "https://docs.oracle.com/en/solutions/oci-aggregate-logs-siem/index.html"},
- 'Networking_Connectivity': {'id': 'OBP-NTW-1', 'section': "Advanced Networking", 'Title': 'Scalable and secure topology in OCI', 'Status': True, 'Findings': [], 'OBP': [], "Documentation": "https://docs.oracle.com/en-us/iaas/Content/Network/Troubleshoot/drgredundancy.htm"},
- 'Cloud_Guard_Config': {'id': 'OBP-CSP-1', 'section': "CSPM", 'Title': 'Cloud Guard enabled and configured', 'Status': None, 'Findings': [], 'OBP': [], "Documentation": "https://www.ateam-oracle.com/post/tuning-oracle-cloud-guard"},
- 'Certificates_Near_Expiry': {'id': 'OBP-CRT-1', 'section': "Certificates", 'Title': 'Certificates to expire in 30 days', 'Status': None, 'Findings': [], 'OBP': [], "Documentation": "TBD"},
+ 'Log_Retention': {'id': 'LAM-19', 'section': "Logging", 'Title': 'Retain Audit Logs for 90 days', 'Status': None, 'Findings': [], 'OBP': [], "Documentation": "https://docs.oracle.com/en-us/iaas/Content/Logging/Task/update-logging-log.htm"},
+ 'Networking_Redudancy': {'id': 'OBP-NTW-1', 'section': "Advanced Networking", 'Title': 'Scalable and secure topology in OCI', 'Status': True, 'Findings': [], 'OBP': [], "Documentation": "https://docs.oracle.com/en-us/iaas/Content/Network/Troubleshoot/drgredundancy.htm"},
+ 'Networking_DRG_Upgraded': {'id': 'OBP-NTW-2', 'section': "Advanced Networking", 'Title': 'Dynamic Route Gateway (DRG) upgraded to version 2', 'Status': None, 'Findings': [], 'OBP': [], "Documentation": "https://docs.oracle.com/en-us/iaas/Content/Network/Tasks/drg-upgrade.htm"},
+ 'Networking_Hub_Spoke': {'id': 'OBP-NTW-3', 'section': "Advanced Networking", 'Title': 'Hub and Spoke Network Architecture', 'Status': None, 'Findings': [], 'OBP': [], "Documentation": "https://docs.oracle.com/en/solutions/hub-spoke-network/index.html"},
+ 'Networking_IPSec_connections': {'id': 'OBP-NTW-4', 'section': "Advanced Networking", 'Title': 'IPSec connections with two tunnels', 'Status': None, 'Findings': [], 'OBP': [], "Documentation": "https://docs.oracle.com/en-us/iaas/Content/Network/Tasks/overviewIPsec.htm"},
+ 'Networking_IPSec_bgp': {'id': 'OBP-NTW-5', 'section': "Advanced Networking", 'Title': 'IPSec connections with BGP Routing', 'Status': None, 'Findings': [], 'OBP': [], "Documentation": "https://docs.oracle.com/en-us/iaas/Content/Network/Tasks/overviewIPsec.htm"},
+ 'Cloud_Guard_Config': {'id': 'OBP-CSP-1', 'section': "CSPM", 'Title': 'Cloud Guard enabled and configured', 'Status': None, 'Findings': [], 'OBP': [], "Documentation": "https://docs.oracle.com/en-us/iaas/Content/cloud-guard/using/part-customize.htm"},
+ 'Cloud_Guard_Problems': {'id': 'OBP-CSP-2', 'section': "CSPM", 'Title': 'Cloud Guard problems with Risk Level Critical or High', 'Status': None, 'Findings': [], 'OBP': [], "Documentation": "https://docs.oracle.com/en-us/iaas/Content/cloud-guard/using/part-problems.htm"},
+ 'Certificates_Near_Expiry': {'id': 'OBP-CRT-1', 'section': "Certificates", 'Title': 'Certificates to expire in 30 days', 'Status': None, 'Findings': [], 'OBP': [], "Documentation": "https://docs.oracle.com/en-us/iaas/Content/certificates/renewing-certificate.htm"},
'Service_Limits': {'id': 'OBP-GOV-1', 'section': "Governance", 'Title': 'Visibility into OCI Limits', 'Status': None, 'Findings': [], 'OBP': [], "Documentation": "https://docs.oracle.com/en/solutions/oci-best-practices/manage-your-service-limits1.html#GUID-457D23F7-98C4-4F74-9E1B-A8F3BCA60C6E"},
'Cost_Tracking_Budgets': {'id': 'OBP-GOV-2', 'section': "Governance", 'Title': 'Alerting on unexpected spending', 'Status': False, 'Findings': [], 'OBP': [], "Documentation": "https://docs.oracle.com/en-us/iaas/Content/Billing/Concepts/budgetsoverview.htm#Budgets_Overview"},
'Quotas': {'id': 'OBP-GOV-3', 'section': "Governance", 'Title': 'Quota policies are used', 'Status': False, 'Findings': [], 'OBP': [], "Documentation": "https://docs.oracle.com/en-us/iaas/Content/Quotas/Concepts/resourcequotas.htm"},
@@ -701,6 +714,9 @@ def __init__(self, config, signer, proxy, output_bucket, report_directory, repor
'ADB_CMK': {'id': 'OBP-ADB-3', 'section': "Autonoumous Database", 'Title': 'ADB Database data is encrypted with a customer managed key', 'Status': None, 'Findings': [], 'OBP': [], "Documentation": "https://docs.oracle.com/en/cloud/paas/autonomous-database/serverless/adbsb/support-tls-mtls-authentication.html#GUID-3F3F1FA4-DD7D-4211-A1D3-A74ED35C0AF5"},
'ADB_Contacts': {'id': 'OBP-ADB-4', 'section': "Autonoumous Database", 'Title': 'ABD Databases have a contact listed', 'Status': None, 'Findings': [], 'OBP': [], "Documentation": "https://docs.oracle.com/en/cloud/paas/autonomous-database/serverless/adbsb/support-tls-mtls-authentication.html#GUID-3F3F1FA4-DD7D-4211-A1D3-A74ED35C0AF5"},
'ADB_Private_IP': {'id': 'OBP-ADB-5', 'section': "Autonoumous Database", 'Title': 'ADB Database are have private endpoints into a customer managed VCN', 'Status': None, 'Findings': [], 'OBP': [], "Documentation": "https://docs.oracle.com/en/cloud/paas/autonomous-database/serverless/adbsb/support-tls-mtls-authentication.html#GUID-3F3F1FA4-DD7D-4211-A1D3-A74ED35C0AF5"},
+ 'IAM_Stmt_Root_Count': {'id': 'IAM-18', 'section': "Identity and Access Management", 'Title': 'IAM Policies are created at appropriate ', 'Status': None, 'Findings': [], 'OBP': [], "Documentation": "https://docs.oracle.com/en-us/iaas/Content/Identity/policymgmt/policy-limits-compartment-hierarchy.htm"},
+ 'IAM_Stmt_Comp_Hierarchy_Count': {'id': 'IAM-19', 'section': "Identity and Access Management", 'Title': 'IAM Policy Statements Limit per Compartment Hierarchy', 'Status': None, 'Findings': [], 'OBP': [], "Documentation": "https://docs.oracle.com/en-us/iaas/Content/Identity/policymgmt/policy-limits-compartment-hierarchy.htm"},
+ 'IAM_Account_Lockout': {'id': 'IAM-20', 'section': "Identity and Access Management", 'Title': 'Account Lockout 5 or more', 'Status': None, 'Findings': [], 'OBP': [], "Documentation": "https://docs.oracle.com/en-us/iaas/Content/Identity/passwordpolicies/Managing-Password-Policies_set-password-policies-your-identity-domain.htm"},
}
# CIS and OBP Regional Data
# 4.6 is not regional because OCI IAM Policies only exist in the home region
@@ -840,11 +856,11 @@ def __init__(self, config, signer, proxy, output_bucket, report_directory, repor
"boot-volume-backups": ["request.permission=BOOT_VOLUME_BACKUP_DELETE"]}}
# CIS Network Filter Check
- self.all_traffic_rules = [{'ruleAction': 'INCLUDE', 'protocol': 'all', 'udpOptions': None, 'isEnabled': True, 'sourceCidr': None,
+ self.all_traffic_rules = [{'ruleAction': 'INCLUDE', 'protocol': 'all', 'udpOptions': None, 'isEnabled': True, 'sourceCidr': None,
'samplingRate': 1, 'flowLogType': 'ALL', 'destinationCidr': None, 'icmpOptions': None, 'priority': 0, 'tcpOptions': None},
{'ruleAction': 'INCLUDE','protocol': 'all','udpOptions': None,'sourceCidr': '0.0.0.0/0','isEnabled': True,'samplingRate': 1,
'flowLogType': 'ALL','icmpOptions': None,'destinationCidr': '0.0.0.0/0','priority': 0,'tcpOptions': None}]
-
+
# Tenancy Data
self.__tenancy = None
self.__cloud_guard_config = None
@@ -852,11 +868,11 @@ def __init__(self, config, signer, proxy, output_bucket, report_directory, repor
self.__os_namespace = None
self.regional_limits_dict = {}
self.__service_limits = []
-
+
# For IAM Checks
self.__tenancy_password_policy = None
self.__compartments = []
- self.__raw_compartment = []
+ self.__raw_compartment = {}
self.__policies = []
self.__users = []
self.__groups = {} # Indexed by GRP OCID
@@ -864,6 +880,7 @@ def __init__(self, config, signer, proxy, output_bucket, report_directory, repor
self.__tag_defaults = []
self.__dynamic_groups = []
self.__identity_domains = []
+ self.__compartment_hierarchy_info = []
# For Networking checks
self.__network_security_groups = []
@@ -903,6 +920,7 @@ def __init__(self, config, signer, proxy, output_bucket, report_directory, repor
# Cloud Guard checks
self.__cloud_guard_targets = {}
+ self.__cloud_guard_problems = []
# For Storage Checks
self.__buckets = []
@@ -928,7 +946,7 @@ def __init__(self, config, signer, proxy, output_bucket, report_directory, repor
# For Budgets
self.__budgets = []
- # Quotas
+ # Quotas
self.__quotas = []
# For Service Connector
@@ -999,7 +1017,7 @@ def __init__(self, config, signer, proxy, output_bucket, report_directory, repor
(obj.region_name for obj in regions if obj.is_home_region),None)
budget_config = self.__config.copy()
budget_config["region"] = budget_home_region
-
+
self.__budget_client = oci.budget.BudgetClient(
budget_config, signer=self.__signer)
if proxy:
@@ -1012,7 +1030,7 @@ def __init__(self, config, signer, proxy, output_bucket, report_directory, repor
if region.is_home_region:
self.__home_region = region.region_name
print("Home region for tenancy is " + self.__home_region)
- if self.__home_region != self.__config['region']:
+ if self.__home_region != self.__config['region']:
print_header("It is recommended to run the CIS Complaince script in your home region")
print_header("The current region is: " + self.__config['region'])
@@ -1055,15 +1073,15 @@ def __init__(self, config, signer, proxy, output_bucket, report_directory, repor
name="Default"
).data
self.__identity_domains_enabled=True
- print_header("Identity Domains Enabled in Tenancy")
+ print_header("Identity Domains Enabled in Tenancy")
except Exception as e:
if e.status == 404:
- print_header("Identity Domains Disabled in Tenancy")
+ print_header("Identity Domains Disabled in Tenancy")
self.__identity_domains_enabled = False
else:
raise RuntimeError(
"Failed to list identity domains." + str(e.args))
-
+
# Creating signers and config for all regions
self.__create_regional_signers(proxy)
@@ -1097,7 +1115,7 @@ def __init__(self, config, signer, proxy, output_bucket, report_directory, repor
self.__output_raw_data = True
# Determining if OCI API unused for 45 days check is disable or not
- self.__disable_api_keys = disable_api_keys
+ self.__disable_api_keys = disable_api_keys
# Determine if __oci_cloud_url will be override with a different realm ex. OC2 or sovreign region
self.__oci_cloud_url = "https://cloud.oracle.com"
@@ -1128,6 +1146,7 @@ def __init__(self, config, signer, proxy, output_bucket, report_directory, repor
self.__oci_vault_uri = self.__oci_cloud_url + "/security/kms/vaults/"
self.__oci_budget_uri = self.__oci_cloud_url + "/usage/budgets/"
self.__oci_cgtarget_uri = self.__oci_cloud_url + "/cloud-guard/targets/"
+ self.__oci_cgproblems_uri = self.__oci_cloud_url + "/cloud-guard/problems/"
self.__oci_onssub_uri = self.__oci_cloud_url + "/notification/subscriptions/"
self.__oci_serviceconnector_uri = self.__oci_cloud_url + "/connector-hub/service-connectors/"
self.__oci_fastconnect_uri = self.__oci_cloud_url + "/networking/fast-connect/virtual-circuit/"
@@ -1145,11 +1164,11 @@ def __map_compliance_to_script(self):
# Compliance Mappings
self.__compliance_mappings = ComplianceMappings()
current_mappings = self.__compliance_mappings.mappings
-
+
# Name of the other compliance framework
self.__primary_framework_name = list(current_mappings['IAM-1'].keys())[0]
self.__other_framework_name = list(current_mappings['IAM-1'].keys())[1]
-
+
for mapping in current_mappings:
for cis_recommendation in self.cis_foundations_benchmark_3_0:
if self.cis_foundations_benchmark_3_0[cis_recommendation]['id'] == mapping:
@@ -1184,7 +1203,7 @@ def __create_client(self, client, service_endpoint=None, key=None, proxy=None, c
return created_client
##########################################################################
-# Create regional config, signers and append them to self.__regions object
+# Create regional config, signers and append them to self.__regions object
##########################################################################
def __create_regional_signers(self, proxy):
print("Creating regional signers and configs...")
@@ -1215,7 +1234,7 @@ def __create_regional_signers(self, proxy):
region_values['fss_client'] = self.__create_client(oci.file_storage.FileStorageClient, key="fss", proxy=proxy)
region_values['sch_client'] = self.__create_client(oci.sch.ServiceConnectorClient, key="sch", proxy=proxy)
region_values['instance'] = self.__create_client(oci.core.ComputeClient, key="compute", proxy=proxy)
- region_values['limits_client'] = self.__create_client(oci.limits.LimitsClient, key="limits_client", proxy=proxy)
+ region_values['limits_client'] = self.__create_client(oci.limits.LimitsClient, key="limits_client", proxy=proxy)
region_values['certificate_client'] = self.__create_client(oci.certificates_management.CertificatesManagementClient, key="cert_mgmt", proxy=proxy)
region_values['logging_search_client'] = self.__create_client(oci.loggingsearch.LogSearchClient, key="logging_search", proxy=proxy)
@@ -1223,7 +1242,7 @@ def __create_regional_signers(self, proxy):
debug("__create_regional_signers: error reading " + str(self.__config))
self.__errors.append({"id": "__create_regional_signers", "error": str(e)})
raise RuntimeError("Failed to create regional clients for data collection: " + str(e))
-
+
##########################################################################
# Check for Managed PaaS Compartment
@@ -1279,9 +1298,10 @@ def __identity_read_compartments(self):
"is_accessible": compartment.is_accessible,
"lifecycle_state": compartment.lifecycle_state,
"time_created": compartment.time_created.strftime(self.__iso_time_format),
+ "statements_in_compartment" : 0,
"region": ""
}
- self.__raw_compartment.append(record)
+ self.__raw_compartment[compartment.id] = record
self.cis_foundations_benchmark_3_0['6.1']['Total'].append(compartment)
# Add root compartment which is not part of the list_compartments
@@ -1299,10 +1319,11 @@ def __identity_read_compartments(self):
"is_accessible": "",
"lifecycle_state": "",
"time_created": "",
+ "statements_in_compartment" : 0,
"region": ""
}
- self.__raw_compartment.append(root_compartment)
+ self.__raw_compartment[self.__tenancy.id] = root_compartment
self.__set_managed_paas_compartment()
@@ -1322,11 +1343,11 @@ def __identity_read_domain_info(self, domain_data ):
domain_dict = oci.util.to_dict(domain_data)
try:
debug("__identity_read_domain_info: Getting Identity Domain Password Policy for: " + domain_data.display_name)
- idcs_url = domain_data.url + "/admin/v1/PasswordPolicies/PasswordPolicy"
+ idcs_url = domain_data.url + "/admin/v1/PasswordPolicies/PasswordPolicy"
raw_pwd_policy_resp = requests.get(url=idcs_url, auth=self.__signer)
raw_pwd_policy_dict = json.loads(raw_pwd_policy_resp.content)
debug("__identity_read_domain_info: Recieved Identity Domain Password Policy for: " + domain_data.display_name)
-
+
# Creating Identity Domains Client and storing it
debug("__identity_read_domain_info: Creating Identity Domain Client for: " + domain_data.display_name)
domain_dict['IdentityDomainClient'] = oci.identity_domains.IdentityDomainsClient(\
@@ -1335,9 +1356,9 @@ def __identity_read_domain_info(self, domain_data ):
pwd_policy_dict = oci.util.to_dict(domain_dict['IdentityDomainClient'].get_password_policy(\
password_policy_id=raw_pwd_policy_dict['ocid']).data)
-
+
domain_dict['password_policy'] = pwd_policy_dict
- domain_dict['errors'] = None
+ domain_dict['errors'] = None
self.__identity_domains.append(domain_dict)
debug("-" * 100)
debug(f"__identity_read_domain_info: Domain Dict is: {domain_dict}")
@@ -1349,7 +1370,7 @@ def __identity_read_domain_info(self, domain_data ):
def __identity_read_domains(self):
if not(self.__identity_domains_enabled):
- return
+ return
print("Processing Identity Domains...")
raw_identity_domains = []
# Finding all Identity Domains in the tenancy
@@ -1366,23 +1387,23 @@ def __identity_read_domains(self):
except Exception as e:
debug("__identity_read_domains: Exception collecting Identity Domains\n" + str(e))
# If this fails the tenancy likely doesn't have identity domains or the permissions are off
- print("\tFound " + str(len(raw_identity_domains)) + " Identity Domains")
-
+ print("\tFound " + str(len(raw_identity_domains)) + " Identity Domains")
+
v_domain_reader_threads = []
for domain in raw_identity_domains:
debug("__identity_read_domains: Getting password policy for domain: " + domain.display_name)
thread = Thread(target=self.__identity_read_domain_info, args=([domain]))
v_domain_reader_threads.append(thread)
-
+
for thread in v_domain_reader_threads:
thread.start()
for thread in v_domain_reader_threads:
- thread.join()
+ thread.join()
+
+ print("\tProcessed " + str(len(self.__identity_domains)) + " Identity Domains")
+ return
- print("\tProcessed " + str(len(self.__identity_domains)) + " Identity Domains")
- return
-
##########################################################################
# Load Groups
##########################################################################
@@ -1395,7 +1416,7 @@ def __identity_read_groups(self):
debug("processing __identity_read_groups for Identity Domain: " + identity_domain['display_name'])
id_domain_deep_link = self.__oci_identity_domains_uri + identity_domain['id']
try:
- groups_data = self.__identity_domains_get_all_results(func=identity_domain['IdentityDomainClient'].list_groups,
+ groups_data = self.__identity_domains_get_all_results(func=identity_domain['IdentityDomainClient'].list_groups,
args={'attribute_sets' : ['default']})
print(f"\tRead {str(len(groups_data))} groups in Identity Domain: " + identity_domain['display_name'])
for grp in groups_data:
@@ -1414,8 +1435,8 @@ def __identity_read_groups(self):
}
# Adding one record per group with no membership info
self.__groups[grp.ocid] = group_record
-
-
+
+
except Exception as e:
self.__errors.append({"id" : "__identity_read_groups", "error" : identity_domain['display_name']+" : "+str(e)})
print("__identity_read_groups: error reading" + str(e))
@@ -1423,7 +1444,7 @@ def __identity_read_groups(self):
"Error in __identity_read_groups" + str(e.args))
return self.__groups
- else:
+ else:
try:
debug("processing __identity_read_groups_and_membership for Non Identity Domains Tenancy")
groups_data = oci.pagination.list_call_get_all_results(
@@ -1477,8 +1498,8 @@ def __identity_read_groups(self):
debug("__identity_read_groups: error reading" + str(e))
RuntimeError(
"Error in __identity_read_groups" + str(e.args))
-
-
+
+
##########################################################################
# Identity Domains Helper function for pagination
##########################################################################
@@ -1489,28 +1510,28 @@ def __identity_read_domains_group_members(self, domain_client, group_ocid):
try:
members += self.__identity_domains_get_all_results(func=domain_client.list_users,
args={'filter' : filter, 'attribute_sets' : ['default']})
- debug("__identity_read_domains_group_members: Collected total keys: " + str(len(members)))
-
+ debug("__identity_read_domains_group_members: Collected total keys: " + str(len(members)))
+
return members
except Exception as e:
self.__errors.append({"id" : f"__identity_read_groups_and_membership: {group_ocid}", "error" : str(e)})
print(f"__identity_read_groups_and_membership: {group_ocid}\n \t{str(e)}")
return members
-
+
##########################################################################
# Identity Domains Helper function for pagination
##########################################################################
def __identity_domains_get_all_results(self, func, args):
-
+
if "start_index" not in args:
args['start_index'] = 1
if "count" not in args:
- args["count"] = 500
+ args["count"] = 500
if "filter" not in args:
args["filter"] = ''
if "attributes" not in args:
- args["attributes"] = ''
+ args["attributes"] = ''
if "attribute_sets" not in args:
args["attribute_sets"] = ['all']
@@ -1528,22 +1549,22 @@ def __identity_domains_get_all_results(self, func, args):
next_index = result.start_index + result.items_per_page
if next_index > result.total_results:
break
-
+
args['start_index'] = next_index
- return resources
+ return resources
##########################################################################
# Load users
##########################################################################
def __identity_read_users_per_domain(self, identity_domain):
-
+
try:
- users_data = self.__identity_domains_get_all_results(func=identity_domain['IdentityDomainClient'].list_users,
+ users_data = self.__identity_domains_get_all_results(func=identity_domain['IdentityDomainClient'].list_users,
args={'attributes':'urn:ietf:params:scim:schemas:oracle:idcs:extension:user:User:isFederatedUser,urn:ietf:params:scim:schemas:oracle:idcs:extension:capabilities:User, groups,urn:ietf:params:scim:schemas:oracle:idcs:extension:userCredentials:User, urn:ietf:params:scim:schemas:oracle:idcs:extension:userState:User:lastSuccessfulLoginDate','attribute_sets':['default'],'count' : 400})
# Local scoped list to store users per domains
v_domain_users = []
# Adding record to the users
-
+
for user in users_data:
record = {}
deep_link = self.__oci_identity_domains_uri + identity_domain['id'] + "/users/" + user.ocid
@@ -1601,11 +1622,11 @@ def __identity_read_users_per_domain(self, identity_domain):
# Concat the list of users from this domain to the global __users list
print(f"\tRead {str(len(users_data))} users in: "+identity_domain['display_name'])
self.__users.extend(v_domain_users)
-
+
except Exception as e:
debug("__identity_read_users_per_domain: Identity Domains are : " + str(self.__identity_domains_enabled))
self.__errors.append({'id' : "__identity_read_users", 'error' : str(e)})
- raise RuntimeError(f"Error in __identity_read_users_per_domain: Identity Domain: {identity_domain['display_name']}, User: {user.user_name}, Error: {str(e)}")
+ raise RuntimeError(f"Error in __identity_read_users_per_domain: Identity Domain: {identity_domain['display_name']}, Error: {str(e)}")
def __identity_read_users(self):
@@ -1616,7 +1637,7 @@ def __identity_read_users(self):
for identity_domain in self.__identity_domains:
thread = Thread(target=self.__identity_read_users_per_domain, args=([identity_domain]))
v_domain_user_reader_threads.append(thread)
-
+
for thread in v_domain_user_reader_threads:
thread.start()
@@ -1647,7 +1668,7 @@ def __identity_read_users(self):
'email': user.email,
'email_verified': user.email_verified,
'external_identifier': user.external_identifier,
- 'is_federated': True if user.identity_provider_id is not None else False,
+ 'is_federated': True if user.identity_provider_id is not None else False,
'is_mfa_activated': user.is_mfa_activated,
'lifecycle_state': True if user.lifecycle_state == 'ACTIVE' else False,
'domain_name' : "",
@@ -1693,7 +1714,7 @@ def __identity_read_users(self):
def __identity_read_user_api_key(self, user_ocid, identity_domain=None):
api_keys = []
debug("__identity_read_user_api_key: Initiating API Keys collection for User ID: " + user_ocid)
- try:
+ try:
if self.__identity_domains_enabled:
debug("__identity_read_user_api_key: Initiating API Keys collection for Identity Domain User ID: " + user_ocid)
filter = f'user.ocid eq \"{user_ocid}\"'
@@ -1702,10 +1723,9 @@ def __identity_read_user_api_key(self, user_ocid, identity_domain=None):
debug("__identity_read_user_api_key: Collected total keys: " + str(len(user_api_keys_data)))
for api_key in user_api_keys_data:
- deep_link = self.__oci_users_uri + "/domains/" + identity_domain['id'] + "/users/" + user_ocid + "/api-keys"
- record = oci.util.to_dict(api_key)
- record['deep_link'] = self.__generate_csv_hyperlink(deep_link, api_key.fingerprint)
- record['time_created'] = self.get_date_iso_format(record['meta']['created'])
+ full_record = oci.util.to_dict(api_key)
+ record = {k: v for k, v in full_record.items() if k in ["id","ocid","fingerprint"]}
+ record['time_created'] = self.get_date_iso_format(full_record['meta']['created'])
apikey_used_in_45_days = self.__identity_check_logging_for_api_activity(user_ocid=user_ocid, api_key=api_key.fingerprint)
record['apikey_used_in_45_days'] = apikey_used_in_45_days
api_keys.append(record)
@@ -1718,20 +1738,19 @@ def __identity_read_user_api_key(self, user_ocid, identity_domain=None):
).data
for api_key in user_api_keys_data:
- deep_link = self.__oci_users_uri + user_ocid + "/api-keys"
- record = oci.util.to_dict(api_key)
- record['deep_link'] = self.__generate_csv_hyperlink(deep_link, api_key.fingerprint)
- record['id'] = record['key_id']
- record['time_created'] = self.get_date_iso_format(record['time_created'])
+ full_record = oci.util.to_dict(api_key)
+ record = {k: v for k, v in full_record.items() if k in ["id","ocid","fingerprint"]}
+ record['id'] = full_record['key_id']
+ record['time_created'] = self.get_date_iso_format(full_record['time_created'])
apikey_used_in_45_days = self.__identity_check_logging_for_api_activity(user_ocid=user_ocid, api_key=api_key.fingerprint)
record['apikey_used_in_45_days'] = apikey_used_in_45_days
api_keys.append(record)
-
+
return api_keys
except Exception as e:
- self.__errors.append({"id" : user_ocid, "error" : "Failed to API Keys for User ID"})
- debug("__identity_read_user_api_key: Failed to API Keys for User ID: " + user_ocid)
+ self.__errors.append({"id" : user_ocid, "error" : "Failed to read API Keys metadata for User ID. "})
+ debug("__identity_read_user_api_key: Failed to read API Keys for User ID: " + user_ocid)
debug("__identity_read_user_api_key: Error for API Keys: " + str(e))
return api_keys
raise RuntimeError(
@@ -1747,11 +1766,11 @@ def __identity_check_logging_for_api_activity(self, user_ocid, api_key):
def numOfDays(date1, date2):
#check which date is greater to avoid days output in -ve number
- if date2 > date1:
+ if date2 > date1:
return (date2-date1).days
else:
return (date1-date2).days
-
+
##########################################################################
# Inputs: start_date(date), end_date(date), data_ranges(list) max_days_between(int)
# Returns: List of dicts with {"start_date" : start_date, "end_date" : end_date}}
@@ -1766,10 +1785,10 @@ def get_date_ranges(start_date, end_date, date_ranges, max_days_between=9):
date_ranges.append({"start_date" : start_date, "end_date" : end_date})
return date_ranges
-
+
##########################################################################
# Inputs: search_query, start_date and end_date in datetime, results
- # Returns: Bool if the key was used in
+ # Returns: Bool if the key was used in
##########################################################################
def run_logging_search_query_api_usage(search_query, api_key_used, start_date: datetime, end_date: datetime):
if self.__disable_api_keys:
@@ -1779,7 +1798,7 @@ def run_logging_search_query_api_usage(search_query, api_key_used, start_date: d
print("Processing Audit Logs for API Key Usage...")
for region_key, region_values in self.__regions.items():
try:
-
+
response = region_values['logging_search_client'].search_logs(
search_logs_details=oci.loggingsearch.models.SearchLogsDetails(
search_query=search_query,
@@ -1793,16 +1812,16 @@ def run_logging_search_query_api_usage(search_query, api_key_used, start_date: d
if audit_logs.summary.result_count > 0:
for result in audit_logs.results:
userInfo = {
- "principalName" : result.data["data.identity.principalName"],
+ "principalName" : result.data["data.identity.principalName"],
"principalId" : result.data["data.identity.principalId"]
}
debug(f'run_logging_search_query_api_usage: Audit search results: {userInfo}')
api_key_used.append(userInfo)
break
-
+
else:
debug('run_logging_search_query_api_usage: No APIKey usage records found in the past 14 days in')
-
+
return api_key_used
except Exception as e:
self.__errors.append({"id" : "run_logging_search_query_api_usage", "error" : str(e)})
@@ -1817,15 +1836,15 @@ def run_logging_search_query_api_usage(search_query, api_key_used, start_date: d
tenancy_search_str = f'\"{self.__tenancy.id}/_Audit_Include_Subcompartment\"'
search_query = "search " + tenancy_search_str + """ | data.identity.credentials = '""" + principle_id + """' and data.identity.tenantId = '""" + self.__tenancy.id + """' | summarize count() by data.identity.principalId, data.identity.principalName"""
debug(f'__identity_check_logging_for_api_activity: Search Query is: {search_query}')
-
+
end_date = self.start_datetime
start_date = end_date - datetime.timedelta(days=self.__days_used)
-
+
search_date_range = get_date_ranges(start_date=start_date, \
end_date=end_date, \
date_ranges=[],
max_days_between=13)
-
+
debug(f'__identity_check_logging_for_api_activity: Initiated Threads for dates range : {str(search_date_range)}')
threads = []
@@ -1848,7 +1867,7 @@ def run_logging_search_query_api_usage(search_query, api_key_used, start_date: d
##########################################################################
- # Load user auth tokens
+ # Load user auth tokens metadata
##########################################################################
def __identity_read_user_auth_token(self, user_ocid, identity_domain=None):
auth_tokens = []
@@ -1860,10 +1879,9 @@ def __identity_read_user_auth_token(self, user_ocid, identity_domain=None):
debug("__identity_read_user_auth_token: Collected total keys: " + str(len(auth_tokens_data)))
for token in auth_tokens_data:
- record = oci.util.to_dict(token)
- deep_link = self.__oci_users_uri + "/domains/" + identity_domain['id'] + "/users/" + user_ocid + "/auth-tokens"
- record['deep_link'] = self.__generate_csv_hyperlink(deep_link, token.description)
- record['time_created'] = self.get_date_iso_format(record['meta']['created'])
+ full_record = oci.util.to_dict(token)
+ record = {k: v for k, v in full_record.items() if k in ["id","ocid","description"]}
+ record['time_created'] = self.get_date_iso_format(full_record['meta']['created'])
auth_tokens.append(record)
else:
@@ -1873,23 +1891,21 @@ def __identity_read_user_auth_token(self, user_ocid, identity_domain=None):
).data
for token in auth_tokens_data:
- deep_link = self.__oci_users_uri + user_ocid + "/swift-credentials"
record = oci.util.to_dict(token)
record['time_created'] = self.get_date_iso_format(record['time_created'])
- record['deep_link'] = self.__generate_csv_hyperlink(deep_link, token.description)
auth_tokens.append(record)
return auth_tokens
except Exception as e:
- self.__errors.append({"id" : user_ocid, "error" : "Failed to auth tokens for User ID"})
+ self.__errors.append({"id" : user_ocid, "error" : "Failed to read auth token metadata for User ID"})
debug("__identity_read_user_auth_token: Failed to auth tokens for User ID: " + user_ocid)
return auth_tokens
raise RuntimeError(
"Error in identity_read_user_auth_token: " + str(e.args))
##########################################################################
- # Load user customer secret key
+ # Load user customer secret keys metadata
##########################################################################
def __identity_read_user_customer_secret_key(self, user_ocid, identity_domain=None):
customer_secret_key = []
@@ -1901,15 +1917,14 @@ def __identity_read_user_customer_secret_key(self, user_ocid, identity_domain=No
debug("__identity_read_user_customer_secret_key: Collected total keys: " + str(len(customer_secret_key_data)))
for key in customer_secret_key_data:
- deep_link = self.__oci_users_uri + "/domains/" + identity_domain['id'] + "/users/" + user_ocid + "/secret-keys"
- record = oci.util.to_dict(key)
- record['deep_link'] = self.__generate_csv_hyperlink(deep_link, key.display_name)
- record['time_created'] = self.get_date_iso_format(record['meta']['created'])
- record['time_expires'] = record['expires_on']
+ full_record = oci.util.to_dict(key)
+ record = {k: v for k, v in full_record.items() if k in ["id","ocid","display_name"]}
+ record['time_created'] = self.get_date_iso_format(full_record['meta']['created'])
+ record['time_expires'] = full_record['expires_on']
customer_secret_key.append(record)
-
+
return customer_secret_key
-
+
else:
customer_secret_key_data = oci.pagination.list_call_get_all_results(
self.__regions[self.__home_region]['identity_client'].list_customer_secret_keys,
@@ -1918,8 +1933,6 @@ def __identity_read_user_customer_secret_key(self, user_ocid, identity_domain=No
for key in customer_secret_key_data:
record = oci.util.to_dict(key)
- deep_link = self.__oci_users_uri + user_ocid + "/secret-keys"
- record['deep_link'] = self.__generate_csv_hyperlink(deep_link, key.display_name)
record['time_created'] = self.get_date_iso_format(record['time_created'])
record['time_expires'] = record['time_expires']
customer_secret_key.append(record)
@@ -1927,14 +1940,14 @@ def __identity_read_user_customer_secret_key(self, user_ocid, identity_domain=No
return customer_secret_key
except Exception as e:
- self.__errors.append({"id" : user_ocid, "error" : "Failed to customer secrets for User ID"})
+ self.__errors.append({"id" : user_ocid, "error" : "Failed to read customer secrets metadata for User ID"})
debug("__identity_read_user_customer_secret_key: Failed to customer secrets for User ID: " + user_ocid)
return customer_secret_key
raise RuntimeError(
"Error in identity_read_user_customer_secret_key: " + str(e.args))
##########################################################################
- # Load Database Passwords
+ # Load Database Passwords metadata
##########################################################################
def __identity_read_user_database_password(self, user_ocid, identity_domain=None):
database_password = []
@@ -1948,16 +1961,15 @@ def __identity_read_user_database_password(self, user_ocid, identity_domain=None
for password in raw_database_password:
debug("__identity_read_user_database_password: Got Password")
- deep_link = self.__oci_users_uri + "/domains/" + identity_domain['id'] + "/users/" + user_ocid + "/db-passwords"
- record = oci.util.to_dict(password)
- record['deep_link'] = self.__generate_csv_hyperlink(deep_link, record['name'])
- record['time_created'] = self.get_date_iso_format(record['meta']['created'])
+ full_record = oci.util.to_dict(password)
+ record = {k: v for k, v in full_record.items() if k in ["id","ocid","description"]}
+ record['time_created'] = self.get_date_iso_format(full_record['meta']['created'])
database_password.append(record)
return database_password
except Exception as e:
- self.__errors.append({"id" : user_ocid, "error" : "Failed to get database passwords for User ID"})
+ self.__errors.append({"id" : user_ocid, "error" : "Failed to read database password metadata for User ID"})
debug("__identity_read_user_database_password: Failed to get database passwords for User ID: " + user_ocid)
debug("__identity_read_user_database_password: Error: " + str(e))
return database_password
@@ -1970,27 +1982,25 @@ def __identity_read_user_database_password(self, user_ocid, identity_domain=None
for password in raw_database_password:
debug("__identity_read_user_database_password: Got Password")
- deep_link = self.__oci_users_uri + user_ocid + "/db-password"
record = oci.util.to_dict(password)
record['ocid'] = record['id']
- record['deep_link'] = deep_link
record['time_created'] = self.get_date_iso_format(record['time_created'])
database_password.append(record)
return database_password
except Exception as e:
- self.__errors.append({"id" : user_ocid, "error" : "Failed to get database passwords for User ID"})
+ self.__errors.append({"id" : user_ocid, "error" : "Failed to read database passwords metadata for User ID"})
debug("__identity_read_user_database_password: Failed to get database passwords for User ID: " + user_ocid)
debug("__identity_read_user_database_password: Error: " + str(e))
return database_password
##########################################################################
- # OCI Helper function to search for OCI resource type
+ # OCI Helper function to search for OCI resource type
##########################################################################
def __search_resource_in_region(self, resource: str, region_values: dict):
-
+
query = (
f"query {resource} resources return allAdditionalFields "
f"where compartmentId != '{self.__managed_paas_compartment_id}'"
@@ -2004,7 +2014,7 @@ def __search_resource_in_region(self, resource: str, region_values: dict):
return getattr(resp, "data", []) or []
##########################################################################
- # Tenancy IAM Policies
+ # Tenancy IAM Policies
##########################################################################
def __identity_read_tenancy_policies(self):
try:
@@ -2022,8 +2032,10 @@ def __identity_read_tenancy_policies(self):
"compartment_id": policy.compartment_id,
"description": policy.additional_details['description'],
"lifecycle_state": policy.lifecycle_state,
- "statements": policy.additional_details['statements']
+ "statements": policy.additional_details['statements'],
+ "number_of_statements" : len(policy.additional_details['statements'])
}
+ self.__raw_compartment[policy.compartment_id]['statements_in_compartment'] += len(policy.additional_details['statements'])
self.__policies.append(record)
print("\tProcessed " + str(len(self.__policies)) + " IAM Policies")
return self.__policies
@@ -2042,7 +2054,8 @@ def __identity_read_dynamic_groups(self):
if self.__identity_domains_enabled:
for identity_domain in self.__identity_domains:
dynamic_groups_data = self.__identity_domains_get_all_results(func=identity_domain['IdentityDomainClient'].list_dynamic_resource_groups,
- args={})
+ args={'attributes' : 'ocid,displayName,description,compartmentOcid,domainOcid,'
+ 'tenancyOcid,matchingRule,idcsCreatedBy,idcsLastModifiedBy,meta,urn:ietf:params:scim:schemas:oracle:idcs:extension:OCITags'})
id_domain_deep_link = self.__oci_identity_domains_uri + identity_domain['id']
for dynamic_group in dynamic_groups_data:
debug("__identity_read_dynamic_groups: reading dynamic groups" + str(dynamic_group.display_name))
@@ -2064,7 +2077,7 @@ def __identity_read_dynamic_groups(self):
debug("__identity_read_dynamic_groups: reading dynamic groups" + str(dynamic_group.name))
record = oci.util.to_dict(dynamic_group)
record['deep_link'] = self.__generate_csv_hyperlink(deep_link, dynamic_group.name)
- record['domain_deeplink'] = None
+ record['domain_deeplink'] = None
record['domain_name'] = None,
record['domain_id'] = None,
self.__dynamic_groups.append(record)
@@ -2098,7 +2111,7 @@ def __identity_read_availability_domains(self):
"Error in __identity_read_availability_domains: " + str(e.args))
##########################################################################
- # Get Objects Store Buckets
+ # Get Objects Store Buckets
##########################################################################
def __os_read_buckets(self):
@@ -2164,7 +2177,7 @@ def __os_read_buckets(self):
raise RuntimeError("Error in __os_read_buckets " + str(e.args))
############################################
- # Load Block Volumes
+ # Load Block Volumes
############################################
def __block_volume_read_block_volumes(self):
try:
@@ -2218,7 +2231,7 @@ def __block_volume_read_block_volumes(self):
raise RuntimeError("Error in __block_volume_read_block_volumes " + str(e.args))
############################################
- # Load Boot Volumes
+ # Load Boot Volumes
############################################
def __boot_volume_read_boot_volumes(self):
try:
@@ -2235,7 +2248,7 @@ def __boot_volume_read_boot_volumes(self):
"deep_link": self.__generate_csv_hyperlink(deep_link, boot_volume.display_name),
"kms_key_id": boot_volume.additional_details['kmsKeyId'],
"lifecycle_state": boot_volume.lifecycle_state,
- "compartment_id": boot_volume.compartment_id,
+ "compartment_id": boot_volume.compartment_id,
"size_in_gbs": boot_volume.additional_details['sizeInGBs'],
"size_in_mbs": boot_volume.additional_details['sizeInMBs'],
"availability_domain": boot_volume.availability_domain,
@@ -2271,7 +2284,7 @@ def __boot_volume_read_boot_volumes(self):
raise RuntimeError("Error in __boot_volume_read_boot_volumes " + str(e.args))
############################################
- # Load FSS
+ # Load FSS
############################################
def __fss_read_fsss(self):
try:
@@ -2328,7 +2341,7 @@ def __fss_read_fsss(self):
raise RuntimeError("Error in __fss_read_fsss " + str(e.args))
##########################################################################
- # Network Security Groups
+ # Network Security Groups
##########################################################################
def __network_read_network_security_groups_rules(self):
self.__network_security_groups = []
@@ -2390,7 +2403,7 @@ def __network_read_network_security_groups_rules(self):
"Error in __network_read_network_security_groups_rules " + str(e.args))
##########################################################################
- # Network Security Lists
+ # Network Security Lists
##########################################################################
def __network_read_network_security_lists(self):
# Looping Through Compartments Except Managed
@@ -2455,7 +2468,7 @@ def __network_read_network_security_lists(self):
"Error in __network_read_network_security_lists " + str(e.args))
##########################################################################
- # Network Subnets Lists
+ # Network Subnets Lists
##########################################################################
def __network_read_network_subnets(self):
try:
@@ -2529,7 +2542,7 @@ def __network_read_network_subnets(self):
"Error in __network_read_network_subnets " + str(e.args))
##########################################################################
- # Network VCNs Lists
+ # Network VCNs Lists
##########################################################################
def __network_read_network_vcns(self):
try:
@@ -2541,7 +2554,7 @@ def __network_read_network_vcns(self):
deep_link = self.__oci_networking_uri + vcn.identifier + '?region=' + region_key
record = oci.util.to_dict(vcn)
record['deep_link'] = self.__generate_csv_hyperlink(deep_link, record['display_name'])
- record['subnets'] = {}
+ record['subnets'] = {}
record['network_security_groups'] = {}
record['security_lists'] = {}
# Adding VCN to VCN list
@@ -2554,7 +2567,7 @@ def __network_read_network_vcns(self):
"Error in __network_read_network_vcns " + str(e.args))
##########################################################################
- # Network Capture Filters Dictionary
+ # Network Capture Filters Dictionary
##########################################################################
def __network_read_network_capturefilters(self):
try:
@@ -2567,7 +2580,7 @@ def __network_read_network_capturefilters(self):
record = oci.util.to_dict(filter)
record['deep_link'] = self.__generate_csv_hyperlink(deep_link, record['display_name'])
- # Adding CaptureFilter to CaptureFilter Dict
+ # Adding CaptureFilter to CaptureFilter Dict
self.__network_capturefilters[filter.identifier] = record
print("\tProcessed " + str(len(self.__network_capturefilters)) + " Network Capture Filters ")
@@ -2578,7 +2591,7 @@ def __network_read_network_capturefilters(self):
"Error in __network_read_network_capturefilters " + str(e.args))
##########################################################################
- # Load DRG Attachments
+ # Load DRG Attachments
##########################################################################
def __network_read_drg_attachments(self):
count_of_drg_attachments = 0
@@ -2661,7 +2674,7 @@ def __network_read_drg_attachments(self):
"Error in __network_read_drg_attachments " + str(e.args))
##########################################################################
- # Load DRGs
+ # Load DRGs
##########################################################################
def __network_read_drgs(self):
try:
@@ -2702,7 +2715,7 @@ def __network_read_drgs(self):
"default_export_drg_route_distribution_id": drg.default_export_drg_route_distribution_id,
"compartment_id": drg.compartment_id,
"lifecycle_state": drg.lifecycle_state,
- "upgrade_status": upgrade_status,
+ "upgrade_status": upgrade_status.upper(),
"time_created": drg.time_created.strftime(self.__iso_time_format),
"freeform_tags": drg.freeform_tags,
"define_tags": drg.defined_tags,
@@ -2742,7 +2755,7 @@ def __network_read_drgs(self):
"Error in __network_read_drgs " + str(e.args))
##########################################################################
- # Load Network FastConnect
+ # Load Network FastConnect
##########################################################################
def __network_read_fastonnects(self):
try:
@@ -2853,7 +2866,7 @@ def __network_read_fastonnects(self):
"Error in __network_read_fastonnects " + str(e.args))
##########################################################################
- # Load IP Sec Connections
+ # Load IP Sec Connections
##########################################################################
def __network_read_ip_sec_connections(self):
try:
@@ -2880,6 +2893,8 @@ def __network_read_ip_sec_connections(self):
"tunnels": [],
"number_tunnels_up": 0,
"tunnels_up": True, # It is true unless I find out otherwise
+ "number_tunnels_bgp": 0,
+ "tunnels_bgp": True, # It is true unless I find out otherwise
"notes": ""
}
# Getting Tunnel Data
@@ -2915,6 +2930,10 @@ def __network_read_ip_sec_connections(self):
record['number_tunnels_up'] += 1
else:
record['tunnels_up'] = False
+ if tunnel_record['routing'].upper() == "BGP":
+ record['number_tunnels_bgp'] += 1
+ else:
+ record['tunnels_bgp'] = False
record["tunnels"].append(tunnel_record)
except Exception:
print("\t Unable to tunnels for ip_sec_connection: " + ip_sec.display_name + " id: " + ip_sec.identifier)
@@ -2954,7 +2973,7 @@ def __network_read_ip_sec_connections(self):
############################################
- # Load Autonomous Databases
+ # Load Autonomous Databases
############################################
def __adb_read_adbs(self):
try:
@@ -3005,7 +3024,7 @@ def __adb_read_adbs(self):
self.__errors.append({'id' : '__adb_read_adbs', 'error' : str(e)})
############################################
- # Load Oracle Integration Cloud
+ # Load Oracle Integration Cloud
############################################
def __oic_read_oics(self):
try:
@@ -3079,7 +3098,7 @@ def __oic_read_oics(self):
raise RuntimeError("Error in __oic_read_oics " + str(e.args))
############################################
- # Load Oracle Analytics Cloud
+ # Load Oracle Analytics Cloud
############################################
def __oac_read_oacs(self):
try:
@@ -3145,7 +3164,7 @@ def __oac_read_oacs(self):
raise RuntimeError("Error in __oac_read_oacs " + str(e.args))
##########################################################################
- # Events
+ # Events
##########################################################################
def __events_read_event_rules(self):
@@ -3177,7 +3196,7 @@ def __events_read_event_rules(self):
raise RuntimeError("Error in events_read_rules " + str(e.args))
##########################################################################
- # Logging - Log Groups and Logs
+ # Logging - Log Groups and Logs
##########################################################################
def __logging_read_log_groups_and_logs(self):
@@ -3204,7 +3223,7 @@ def __logging_read_log_groups_and_logs(self):
"notes" : ""
}
- try:
+ try:
logs = oci.pagination.list_call_get_all_results(
region_values['logging_client'].list_logs,
log_group_id=log_group.identifier
@@ -3246,7 +3265,7 @@ def __logging_read_log_groups_and_logs(self):
log_record["capture_filter"] = log.configuration.source.parameters["capture_filter"]
else:
log_record["capture_filter"] = None
-
+
elif log_record["lifecycle_state"] == "ACTIVE":
log_record["source_category"] = log.log_type
log_record["source_service"] = log.log_type
@@ -3255,10 +3274,10 @@ def __logging_read_log_groups_and_logs(self):
except Exception as e:
print(log)
print(e)
-
+
#### TESTING SOMETHING NEW ####
- try:
+ try:
## Active means your logging
if log_record['lifecycle_state'] == 'ACTIVE':
if self.__all_logs:
@@ -3268,24 +3287,24 @@ def __logging_read_log_groups_and_logs(self):
if log_record["source_category"] in self.__all_logs[log_record["source_service"]]:
debug("\t__logging_read_log_groups_and_logs: Adding log for existing service and category ")
self.__all_logs[log_record["source_service"]][log_record["source_category"]][log_record["source_resource"]] = log_record
-
+
else:
debug(f'\t__logging_read_log_groups_and_logs: Adding category {log_record["source_category"]}')
self.__all_logs[log_record["source_service"]][log_record["source_category"]] = {}
self.__all_logs[log_record["source_service"]][log_record["source_category"]][log_record["source_resource"]] = log_record
-
+
else:
debug(f'\t__logging_read_log_groups_and_logs: Adding Service {log_record["source_service"]}, and category {log_record["source_category"]}')
self.__all_logs[log_record["source_service"]] = {}
self.__all_logs[log_record["source_service"]][log_record["source_category"]] = {}
self.__all_logs[log_record["source_service"]][log_record["source_category"]][log_record["source_resource"]] = log_record
-
+
else:
debug(f'\t__logging_read_log_groups_and_logs: Starting Dict: Adding Service {log_record["source_service"]}, and category {log_record["source_category"]}' )
self.__all_logs[log_record["source_service"]] = {}
self.__all_logs[log_record["source_service"]][log_record["source_category"]] = {}
self.__all_logs[log_record["source_service"]][log_record["source_category"]][log_record["source_resource"]] = log_record
-
+
except Exception as e:
print(f'\tFailed to parse log: {log_record["id"]}')
@@ -3296,29 +3315,6 @@ def __logging_read_log_groups_and_logs(self):
print(e)
print("*" * 80)
- # if log.configuration.source.service == 'flowlogs':
- # self.__subnet_logs[log.configuration.source.resource] = {"log_group_id": log.log_group_id, "log_id": log.id}
-
- # elif log.configuration.source.service == 'objectstorage' and 'write' in log.configuration.source.category:
- # # Only write logs
- # self.__write_bucket_logs[log.configuration.source.resource] = {"log_group_id": log.log_group_id, "log_id": log.id, "region": region_key}
-
- # elif log.configuration.source.service == 'objectstorage' and 'read' in log.configuration.source.category:
- # # Only read logs
- # self.__read_bucket_logs[log.configuration.source.resource] = {"log_group_id": log.log_group_id, "log_id": log.id, "region": region_key}
-
- # elif log.configuration.source.service == 'loadbalancer' and 'error' in log.configuration.source.category:
- # self.__load_balancer_error_logs.append(
- # log.configuration.source.resource)
- # elif log.configuration.source.service == 'loadbalancer' and 'access' in log.configuration.source.category:
- # self.__load_balancer_access_logs.append(
- # log.configuration.source.resource)
- # elif log.configuration.source.service == 'apigateway' and 'access' in log.configuration.source.category:
- # self.__api_gateway_access_logs.append(
- # log.configuration.source.resource)
- # elif log.configuration.source.service == 'apigateway' and 'error' in log.configuration.source.category:
- # self.__api_gateway_error_logs.append(
- # log.configuration.source.resource)
except Exception as e:
self.__errors.append({"id" : log.id, "error" : str(e)})
# Append Log to log List
@@ -3337,7 +3333,7 @@ def __logging_read_log_groups_and_logs(self):
"Error in __logging_read_log_groups_and_logs " + str(e.args))
##########################################################################
- # Vault Keys
+ # Vault Keys
##########################################################################
def __kms_read_keys(self):
debug("__kms_read_keys: Initiating")
@@ -3350,13 +3346,17 @@ def __kms_read_keys(self):
vaults_set = set()
for key in keys_data:
vaults_set.add(key.additional_details['vaultId'])
-
+
for vault in vaults_set:
try:
debug("\t__kms_read_keys: Getting Vault details for Vault: " + str(vault))
vault_details = region_values['vault_client'].get_vault(vault_id=vault).data
debug("\t__kms_read_keys: Succeeded getting Vault details for: " + str(vault_details))
vault_dict = oci.util.to_dict(vault_details)
+ vault_dict['region'] = region_key
+ vault_deep_link = self.__oci_vault_uri + vault_dict['id'] + '?region=' + region_key
+ vault_dict['deep_link'] = self.__generate_csv_hyperlink(name=vault_dict['display_name'],
+ url=vault_deep_link)
vault_dict['keys'] = []
self.__vaults[vault] = vault_dict
debug("\t__kms_read_keys: Building KMS Client: " + str(vault))
@@ -3364,7 +3364,7 @@ def __kms_read_keys(self):
region_signer.region_name = region_key
region_config = self.__config
region_config['region'] = region_key
- self.__vaults[vault]['kms_client'] = oci.key_management.KmsManagementClient(config=region_config,
+ self.__vaults[vault]['kms_client'] = oci.key_management.KmsManagementClient(config=region_config,
signer=region_signer,
service_endpoint=vault_dict['management_endpoint'])
if self.__proxy:
@@ -3376,7 +3376,7 @@ def __kms_read_keys(self):
print(e)
self.__vaults[vault] = {"id" : vault, "keys" : [], "kms_client" : None}
self.__errors.append({"id" : vault, "error" : str(e) })
-
+
### Getting Wrapping Key
try:
wrapping_key_id = self.__vaults[vault]['kms_client'].get_wrapping_key().data.id
@@ -3390,6 +3390,9 @@ def __kms_read_keys(self):
if key.identifier != self.__vaults[key.additional_details['vaultId']]['wrapping_key_id']:
deep_link = self.__oci_vault_uri + key.additional_details['vaultId'] + "/vaults/" + key.identifier + '?region=' + region_key
key_record = oci.util.to_dict(key)
+ key_record['region'] = region_key
+ key_record['is_primary'] = self.__vaults[key.additional_details['vaultId']].get('is_primary')
+ key_record['is_vault_replicable'] = self.__vaults[key.additional_details['vaultId']].get('is_vault_replicable')
key_record['deep_link'] = self.__generate_csv_hyperlink(deep_link, key_record['display_name'])
try:
if self.__vaults[key.additional_details['vaultId']]['kms_client']:
@@ -3410,13 +3413,13 @@ def __kms_read_keys(self):
print(e)
key_record['currentKeyVersion_time_created'] = None
self.__errors.append({"id" : key.identifier, "error" : str(e) })
-
+
self.__vaults[key.additional_details['vaultId']]["keys"].append(key_record)
self.__kms_keys.append(key_record)
else:
debug("\t__kms_read_keys: Ignoring wrapping key: " + key.display_name)
- print("\tProcessed " + str(len(self.__kms_keys)) + " Keys")
+ print(f"\tProcessed {str(len(self.__kms_keys))} Keys in {str(len(self.__vaults))} Vaults")
return self.__vaults
except Exception as e:
raise RuntimeError(
@@ -3485,9 +3488,9 @@ def __quota_read(self):
# QUotas are only in the home region
quota_data = self.__search_resource_in_region("quota", self.__regions[self.__home_region] )
debug("\t__quota_read: Recieved " + str(len(quota_data)) + " quotas " + str(self.__regions[self.__home_region]['region_name']))
- try:
+ try:
for quota in quota_data:
- deep_link = self.__oci_quota_uri + quota.identifier + '?region=' + self.__regions[self.__home_region]['region_name']
+ deep_link = self.__oci_quota_uri + quota.identifier + '?region=' + self.__regions[self.__home_region]['region_name']
record = {
"id": quota.identifier,
"deep_link": self.__generate_csv_hyperlink(deep_link, quota.identifier),
@@ -3507,7 +3510,37 @@ def __quota_read(self):
raise RuntimeError(
"Error in __quota_read " + str(e.args))
-
+ ##########################################################################
+ # Cloud Guard Problems
+ ##########################################################################
+ def __read_cloud_guard_problems(self):
+ """
+ Retrieves every Cloud Guard problem for the reporting region,
+ converts each OCI model to a plain ``dict`` (via ``oci.util.to_dict``)
+ and stores the result in ``self.__cloud_guard_problems``.
+ """
+ cg_client = self.__regions[self.__cloud_guard_config.reporting_region]['cloud_guard_client']
+
+ try:
+ response = oci.pagination.list_call_get_all_results(
+ cg_client.list_problems,
+ compartment_id=self.__tenancy.id,
+ lifecycle_state='ACTIVE',
+ compartment_id_in_subtree=True,
+ access_level="ACCESSIBLE"
+ )
+ for problem in response.data:
+ record = oci.util.to_dict(problem)
+ deep_link = self.__oci_cgproblems_uri + record['id'] + "?region=" + record['region']
+ record['deep_link'] = self.__generate_csv_hyperlink(deep_link, record['resource_name']),
+ self.__cloud_guard_problems.append(record)
+ except Exception as e:
+ self.__errors.append({'id' : '__read_cloud_guard_problems', 'error' : str(e)})
+ debug("__read_cloud_guard_problems failed to process: " + str(e))
+
+ print("\tProcessed " + str(len(self.__cloud_guard_problems)) + " Cloud Guard Problems")
+
+
##########################################################################
# Cloud Guard Configuration
##########################################################################
@@ -3517,7 +3550,7 @@ def __cloud_guard_read_cloud_guard_configuration(self):
self.__tenancy.id).data
debug("__cloud_guard_read_cloud_guard_configuration Cloud Guard Configuration is: " + str(self.__cloud_guard_config))
self.__cloud_guard_config_status = self.__cloud_guard_config.status
-
+
print("\tProcessed Cloud Guard Configuration.")
return self.__cloud_guard_config_status
@@ -3533,54 +3566,52 @@ def __cloud_guard_read_cloud_guard_targets(self):
if self.__cloud_guard_config_status == "ENABLED":
cloud_guard_targets = 0
try:
- for compartment in self.__compartments:
- if self.__if_not_managed_paas_compartment(compartment.name):
- # Getting a compartments target
- cg_targets = self.__regions[self.__cloud_guard_config.reporting_region]['cloud_guard_client'].list_targets(
- compartment_id=compartment.id).data.items
- debug("__cloud_guard_read_cloud_guard_targets: " + str(cg_targets) )
- # Looping throufh targets to get target data
- for target in cg_targets:
- try:
- # Getting Target data like recipes
- try:
- target_data = self.__regions[self.__cloud_guard_config.reporting_region]['cloud_guard_client'].get_target(
- target_id=target.id
- ).data
+ # Getting a compartments target
+ cg_targets = self.__regions[self.__cloud_guard_config.reporting_region]['cloud_guard_client'].list_targets(
+ compartment_id=self.__tenancy.id, compartment_id_in_subtree=True, access_level="RESTRICTED",).data.items
+ debug("__cloud_guard_read_cloud_guard_targets: " + str(cg_targets) )
+ # Looping throufh targets to get target data
+ for target in cg_targets:
+ try:
+ # Getting Target data like recipes
+ try:
+ target_data = self.__regions[self.__cloud_guard_config.reporting_region]['cloud_guard_client'].get_target(
+ target_id=target.id
+ ).data
- except Exception:
- target_data = None
- deep_link = self.__oci_cgtarget_uri + target.id
- record = {
- "compartment_id": target.compartment_id,
- "defined_tags": target.defined_tags,
- "display_name": target.display_name,
- "deep_link": self.__generate_csv_hyperlink(deep_link, target.display_name),
- "freeform_tags": target.freeform_tags,
- "id": target.id,
- "lifecycle_state": target.lifecycle_state,
- "lifecyle_details": target.lifecyle_details,
- "system_tags": target.system_tags,
- "recipe_count": target.recipe_count,
- "target_resource_id": target.target_resource_id,
- "target_resource_type": target.target_resource_type,
- "time_created": target.time_created.strftime(self.__iso_time_format),
- "time_updated": str(target.time_updated),
- "inherited_by_compartments": target_data.inherited_by_compartments if target_data else "",
- "description": target_data.description if target_data else "",
- "target_details": target_data.target_details if target_data else "",
- "target_detector_recipes": target_data.target_detector_recipes if target_data else "",
- "target_responder_recipes": target_data.target_responder_recipes if target_data else ""
- }
- # Indexing by compartment_id
+ except Exception:
+ target_data = None
+ deep_link = self.__oci_cgtarget_uri + target.id
+ record = {
+ "compartment_id": target.compartment_id,
+ "defined_tags": target.defined_tags,
+ "display_name": target.display_name,
+ "deep_link": self.__generate_csv_hyperlink(deep_link, target.display_name),
+ "freeform_tags": target.freeform_tags,
+ "id": target.id,
+ "lifecycle_state": target.lifecycle_state,
+ "lifecyle_details": target.lifecyle_details,
+ "system_tags": target.system_tags,
+ "recipe_count": target.recipe_count,
+ "target_resource_id": target.target_resource_id,
+ "target_resource_type": target.target_resource_type,
+ "time_created": target.time_created.strftime(self.__iso_time_format),
+ "time_updated": str(target.time_updated),
+ "inherited_by_compartments": target_data.inherited_by_compartments if target_data else "",
+ "description": target_data.description if target_data else "",
+ "target_details": target_data.target_details if target_data else "",
+ "target_detector_recipes": target_data.target_detector_recipes if target_data else "",
+ "target_responder_recipes": target_data.target_responder_recipes if target_data else ""
+ }
+ # Indexing by compartment_id
- self.__cloud_guard_targets[compartment.id] = record
+ self.__cloud_guard_targets[target.compartment_id] = record
- cloud_guard_targets += 1
+ cloud_guard_targets += 1
- except Exception:
- print("\t Failed to Cloud Guard Target Data for: " + target.display_name + " id: " + target.id)
- self.__errors.append({"id" : target.id, "error" : "Failed to Cloud Guard Target Data for: " + target.display_name + " id: " + target.id })
+ except Exception:
+ print("\t Failed to read Cloud Guard Target Data for: " + target.display_name + " id: " + target.id)
+ self.__errors.append({"id" : target.id, "error" : "Failed to read Cloud Guard Target Data for: " + target.display_name + " id: " + target.id })
print("\tProcessed " + str(cloud_guard_targets) + " Cloud Guard Targets")
return self.__cloud_guard_targets
@@ -3593,13 +3624,17 @@ def __cloud_guard_read_cloud_guard_targets(self):
# Identity Password Policy
##########################################################################
def __identity_read_tenancy_password_policy(self):
+ if self.__identity_domains_enabled:
+ self.__tenancy_password_policy = None
+ return self.__tenancy_password_policy
+
+ print("\tProcessed Tenancy Password Policy...")
+
try:
self.__tenancy_password_policy = self.__regions[self.__home_region]['identity_client'].get_authentication_policy(
self.__tenancy.id
).data
- print("\tProcessed Tenancy Password Policy...")
- return self.__tenancy_password_policy
except Exception as e:
if "NotAuthorizedOrNotFound" in str(e):
self.__tenancy_password_policy = None
@@ -3607,9 +3642,10 @@ def __identity_read_tenancy_password_policy(self):
self.__errors.append({"id" : self.__tenancy.id, "error" : "*** Access to password policies in this tenancy requires elevated permissions. ***"})
else:
raise RuntimeError("Error in __identity_read_tenancy_password_policy " + str(e.args))
+ return self.__tenancy_password_policy
##########################################################################
- # Oracle Notifications Services for Subscriptions
+ # Oracle Notifications Services for Subscriptions
##########################################################################
def __ons_read_subscriptions(self):
debug("__ons_read_subscriptions: Starting: ")
@@ -3677,7 +3713,7 @@ def __identity_read_tag_defaults(self):
print("Error in __identity_read_tag_defaults " + str(e.args))
self.__errors.append({'id' : '__identity_read_tag_defaults', 'error' : str(e)})
##########################################################################
- # Get Service Connectors
+ # Get Service Connectors
##########################################################################
def __sch_read_service_connectors(self):
@@ -3803,7 +3839,7 @@ def __search_resources_in_root_compartment(self):
print("\tProcessed " + str(len(self.__resources_in_root_compartment)) + " resources in the root compartment")
return self.__resources_in_root_compartment
-
+
##########################################################################
# Gets a resource and it's additional fields via Search Servuce
##########################################################################
@@ -3816,17 +3852,17 @@ def __search_query_resource_type(self, resource_type, search_client):
query=query),
tenant_id=self.__tenancy.id
).data
-
+
return oci.util.to_dict(results)
except Exception:
debug("__search_query_resource_type: failed to get type: " + str(resource_type))
return []
-
+
##########################################################################
# All Resources in Tenancy
##########################################################################
def __search_resources_all_resources_in_tenancy(self):
-
+
for region_key, region_values in self.__regions.items():
self.__all_resources_json[region_key] = {}
try:
@@ -3839,18 +3875,18 @@ def __search_resources_all_resources_in_tenancy(self):
for type in self.__all_resources_json[region_key]:
self.__all_resources_json[region_key][type] += self.__search_query_resource_type(type, region_values['search_client'])
-
+
except Exception as e:
raise RuntimeError(
"Error in __search_resources_all_resources_in_tenancy " + str(e.args))
-
+
print("\tProcessed " + str(len(self.__all_resources_json)) + " resources in the tenancy")
- # print(self.__all_resources_json)
+ # print(self.__all_resources_json)
return self.__all_resources_json
-
+
##########################################################################
# Get Compute Instances
- ##########################################################################
+ ##########################################################################
def __core_instance_read_compute(self):
try:
# looping through regions
@@ -3867,7 +3903,10 @@ def __core_instance_read_compute(self):
instance_data = region_values['instance'].get_instance(
instance_id=instance['identifier']
).data
- record = oci.util.to_dict(instance_data)
+ full_record = oci.util.to_dict(instance_data)
+ #Remove extended_metadata and metadata fields
+ record = {k: v for k, v in full_record.items() if k not in ["metadata","extended_metadata"]}
+
record['deep_link'] = self.__generate_csv_hyperlink(deep_link, instance['display_name'])
record['error'] = ""
# self.__all_resources_json[region_key]['Instance'].append(record)
@@ -3895,9 +3934,9 @@ def __core_instance_read_compute(self):
def __get_region_name_from_key(self,region_key):
debug("__get_region_name_from_key")
for key, region_values in self.__regions.items():
- if region_values['region_key'].upper() == region_key.upper() or region_values['region_name'].upper() == region_key.upper():
+ if region_values['region_key'].upper() == region_key.upper() or region_values['region_name'].upper() == region_key.upper():
return region_values['region_name']
-
+
##########################################################################
# Query All certificates in the tenancy
##########################################################################
@@ -3939,21 +3978,21 @@ def __certificates_read_certificates(self):
self.__errors.append({'id' : '__certificates_read_certificates', 'error' : str(e)})
print("\tProcessed " + str(len(self.__raw_oci_certificates)) + " Certificates")
-
+
##########################################################################
# Query Services Limits
- ##########################################################################
+ ##########################################################################
def __service_limits_utilization(self):
debug("__service_limits_utilization: Starting")
-
+
service_limit_name_limit_value_mapping = {}
-
+
def regional_service_limits(oci_region):
thread_pool = concurrent.futures.ThreadPoolExecutor(max_workers=10)
-
+
for oci_service in oci_services:
-
+
thread_pool.submit(service_limit_values, oci_service.name, oci_region)
thread_pool.shutdown(wait=True)
@@ -3963,7 +4002,7 @@ def service_limit_values(service_name, oci_region):
self.__regions[oci_region]['limits_client'].list_limit_values,
self.__tenancy.id, service_name).data
service_limit_name_limit_value_mapping[oci_region][service_name] = oci_service_limit_values
-
+
def utilization_function(oci_region, service_name, limit_name, availability_domain):
try:
@@ -4014,26 +4053,26 @@ def service_limit_function(region_name, service_names):
oci_services = oci.pagination.list_call_get_all_results(
self.__regions[self.__home_region]['limits_client'].list_services,
compartment_id=self.__tenancy.id).data
-
+
except Exception as e:
self.__errors.append({"id": "__service_limits_utilization", \
"error" : str(e)})
raise RuntimeError(
"Error in __service_limits_utilization " + str(e))
-
+
for region_key in self.__regions.keys():
regional_services={}
for oci_service in oci_services:
-
+
regional_services[oci_service.name]=None
-
+
service_limit_name_limit_value_mapping[region_key] = regional_services
regional_limit_threadpool = concurrent.futures.ThreadPoolExecutor(max_workers=len(self.__regions))
service_limit_threadpool = concurrent.futures.ThreadPoolExecutor(max_workers=len(self.__regions))
-
+
for region_key in self.__regions.keys():
# Get all OCI services
@@ -4081,7 +4120,7 @@ def __report_cis_analyze_tenancy_data(self):
def __cis_check_iam_policies(self):
-
+
# 1.1 Check - Checking for policy statements that are not restricted to a service
for policy in self.__policies:
for statement in policy['statements']:
@@ -4133,6 +4172,7 @@ def __cis_check_iam_policies(self):
# Iterating through all policies
for policy in self.__policies:
if policy['name'].lower() not in ['tenant admin policy', 'psm-root-policy']:
+ insert_policy = None
for statement in policy['statements']:
for resource in self.cis_iam_checks['1.15']:
if "allow group".upper() in statement.upper() and "to manage ".upper() in statement.upper() and resource.upper() in statement.upper():
@@ -4143,17 +4183,19 @@ def __cis_check_iam_policies(self):
not(all(permission.upper() in clean_where_clause for permission in self.cis_iam_checks['1.15-storage-admin'][resource])):
debug("__report_cis_analyze_tenancy_data CIS 1.15 no permissions to delete storage: " + str(policy['name']))
pass
- # Checking if this is the Storage admin with allowed
+ # Checking if this is the Storage admin with allowed
elif all(permission.upper() in clean_where_clause for permission in self.cis_iam_checks['1.15-storage-admin'][resource]) and \
not(all(permission.upper() in clean_where_clause for permission in self.cis_iam_checks['1.15'][resource])):
debug("__report_cis_analyze_tenancy_data CIS 1.15 storage admin policy is: " + str(policy['name']))
pass
else:
- self.cis_foundations_benchmark_3_0['1.15']['Findings'].append(policy)
+ insert_policy = True
debug("__report_cis_analyze_tenancy_data CIS 1.15 else policy is\n: " + str(policy['name']))
else:
- self.cis_foundations_benchmark_3_0['1.15']['Findings'].append(policy)
+ insert_policy = True
+ if insert_policy:
+ self.cis_foundations_benchmark_3_0['1.15']['Findings'].append(policy)
if self.cis_foundations_benchmark_3_0['1.15']['Findings']:
self.cis_foundations_benchmark_3_0['1.15']['Status'] = False
@@ -4173,7 +4215,7 @@ def __cis_check_password_policies(self):
else:
self.cis_foundations_benchmark_3_0['1.4']['Status'] = None
- # 1.4, 1.5 and 1.6 Checking Identity Domains Password Policy for expiry less than 365 and
+ # 1.4, 1.5 and 1.6 Checking Identity Domains Password Policy for expiry less than 365 and
debug("__report_cis_analyze_tenancy_data: Identity Domains Enabled is: " + str(self.__identity_domains_enabled))
if self.__identity_domains_enabled:
for domain in self.__identity_domains:
@@ -4222,7 +4264,7 @@ def __cis_check_password_policies(self):
self.cis_foundations_benchmark_3_0['1.6']['Status'] = False
else:
self.cis_foundations_benchmark_3_0['1.6']['Status'] = True
-
+
# Adding all identity domains to Total
self.cis_foundations_benchmark_3_0['1.4']['Total'] = self.__identity_domains
self.cis_foundations_benchmark_3_0['1.5']['Total'] = self.__identity_domains
@@ -4249,7 +4291,7 @@ def __cis_check_users(self):
"user_name": user['name'],
"user_id": user['id'],
"key_id": key['id'],
- "domain_deeplink": user['domain_deeplink'],
+ "domain_deeplink": user['domain_deeplink'],
'fingerprint': key['fingerprint'],
# 'inactive_status': key['inactive_status'],
# 'lifecycle_state': key['lifecycle_state'],
@@ -4318,7 +4360,7 @@ def __cis_check_users(self):
for key in user['database_passwords']:
if self.api_key_time_max_datetime >= datetime.datetime.strptime(key['time_created'], self.__iso_time_format):
self.cis_foundations_benchmark_3_0['1.11']['Status'] = False
-
+
finding = {
"user_name": user['name'],
"user_id": user['id'],
@@ -4333,7 +4375,7 @@ def __cis_check_users(self):
# CIS Total 1.11 Adding - Keys to CIS Total
self.cis_foundations_benchmark_3_0['1.11']['Total'].append(key)
-
+
# CIS 1.12 Active Admins with API keys
# Iterating through all users to see if they have API Keys and if they are active users
@@ -4408,7 +4450,7 @@ def __cis_check_users(self):
- # CIS 1.17 Check - Ensure there is only one active API Key for any single OCI IAM user
+ # CIS 1.17 Check - Ensure there is only one active API Key for any single OCI IAM user
for user in self.__users:
if user['api_keys'] and not(len(user['api_keys']) < 2):
self.cis_foundations_benchmark_3_0['1.17']['Findings'].append(user)
@@ -4419,7 +4461,7 @@ def __cis_check_users(self):
self.cis_foundations_benchmark_3_0['1.17']['Status'] = True
# CIS Total 1.17 Adding - All IAM Policies for to CIS Total
self.cis_foundations_benchmark_3_0['1.17']['Total'] = self.__users
-
+
def __cis_check_dynamic_groups(self):
# CIS 1.14 Check - Ensure Dynamic Groups are used for OCI instances, OCI Cloud Databases and OCI Function to access OCI resources
# Iterating through all dynamic groups ensure there are some for fnfunc, instance or autonomous. Using reverse logic so starts as a false
@@ -4562,10 +4604,16 @@ def __cis_check_network_security(self):
self.cis_foundations_benchmark_3_0['2.7']['Total'] = self.__analytics_instances
# CIS 2.8 Check - Ensure Oracle Autonomous Shared Databases (ADB) access is restricted to allowed sources or deployed within a VCN
- # Iterating through ADB Checking for null NSGs, whitelisted ip or allowed IPs 0.0.0.0/0
- # Issue 295 fixed
+ # Control applies to Serverless only, so dedicated ADBs are excluded.
for autonomous_database in self.__autonomous_databases:
- if autonomous_database['lifecycle_state'] not in [ oci.database.models.AutonomousDatabaseSummary.LIFECYCLE_STATE_TERMINATED, oci.database.models.AutonomousDatabaseSummary.LIFECYCLE_STATE_TERMINATING, oci.database.models.AutonomousDatabaseSummary.LIFECYCLE_STATE_UNAVAILABLE ]:
+ if (
+ autonomous_database['lifecycle_state'] not in [
+ oci.database.models.AutonomousDatabaseSummary.LIFECYCLE_STATE_TERMINATED,
+ oci.database.models.AutonomousDatabaseSummary.LIFECYCLE_STATE_TERMINATING,
+ oci.database.models.AutonomousDatabaseSummary.LIFECYCLE_STATE_UNAVAILABLE,
+ ]
+ and not autonomous_database.get('is_dedicated', False)
+ ):
if not (autonomous_database['whitelisted_ips']) and not (autonomous_database['subnet_id']):
self.cis_foundations_benchmark_3_0['2.8']['Status'] = False
self.cis_foundations_benchmark_3_0['2.8']['Findings'].append(autonomous_database)
@@ -4575,7 +4623,6 @@ def __cis_check_network_security(self):
self.cis_foundations_benchmark_3_0['2.8']['Status'] = False
self.cis_foundations_benchmark_3_0['2.8']['Findings'].append(autonomous_database)
- # CIS Total 2.8 Adding - All ADBs to CIS Total
self.cis_foundations_benchmark_3_0['2.8']['Total'] = self.__autonomous_databases
def __cis_check_compute_instances(self):
@@ -4586,13 +4633,13 @@ def __cis_check_compute_instances(self):
debug(f"__report_cis_analyze_tenancy_data {instance['display_name']} doesn't disable IMDSv1")
self.cis_foundations_benchmark_3_0['3.1']['Status'] = False
self.cis_foundations_benchmark_3_0['3.1']['Findings'].append(instance)
-
+
# CIS Check 3.2 Secure Boot enabled
if instance['platform_config'] is None or not(instance['platform_config']['is_secure_boot_enabled']):
debug(f"__report_cis_analyze_tenancy_data {instance['display_name']} doesn't enable secure boot")
self.cis_foundations_benchmark_3_0['3.2']['Status'] = False
self.cis_foundations_benchmark_3_0['3.2']['Findings'].append(instance)
-
+
# CIS Check 3.3 Encryption in Transit enabled
if instance['launch_options'] is None or not(instance['launch_options']['is_pv_encryption_in_transit_enabled']):
debug(f"__report_cis_analyze_tenancy_data {instance['display_name']} doesn't enable encryption in transit")
@@ -4622,7 +4669,7 @@ def __cis_check_tagging_and_monitoring(self):
self.cis_foundations_benchmark_3_0['4.2']['Status'] = True
else:
self.cis_foundations_benchmark_3_0['4.2']['Findings'].append(sub)
-
+
# CIS Check 4.2 Total - All Subscriptions to CIS Total
@@ -4630,6 +4677,40 @@ def __cis_check_tagging_and_monitoring(self):
# CIS Checks 4.3 - 4.12 and 4.15 and 4.18
# Iterate through all event rules
+ def __event_with_topic_has_active_sub(event_actions):
+ """
+ Checks event actions
+ returns True if it finds either a non-ONS action or an ONS action whose topicId has at least one ACTIVE subscription.
+ Returns False when no qualifying action exists.
+ """
+ for action in event_actions or []:
+ if action.get('actionType') == 'ONS' and action.get('topicId'):
+ topic_id = action.get('topicId')
+ if topic_id and __has_active_subscription_for_topic(topic_id=topic_id):
+ debug("*** Active Subscription found for event: " + event['display_name'] + " ***")
+ return True
+ elif not action.get('actionType') == 'ONS':
+ # None ONS based notifications are assumed to be ok
+ return True
+ return False
+ def __has_active_subscription_for_topic(topic_id: str) -> bool:
+ """
+ Return True if any subscription in self.__subscriptions has:
+ - subscription["topic_id"] == topic_id and subscription["lifecycle_state"] == "ACTIVE"
+ Otherwise return False.
+ """
+ if not topic_id:
+ return False
+
+ for subscription in self.__subscriptions:
+ if (
+ subscription.get("topic_id") == topic_id
+ and subscription.get("lifecycle_state") == "ACTIVE"
+ ):
+ return True
+
+ return False
+
for event in self.__event_rules:
if event['lifecycle_state'] == "ACTIVE" and event['compartment_id'] == self.__tenancy.id:
# Convert Event Condition to dict
@@ -4647,18 +4728,22 @@ def __cis_check_tagging_and_monitoring(self):
try:
# Checking if each region has the required events
if (all(x in eventtype_dict['eventtype'] for x in changes)) and key in self.__cis_regional_checks:
- self.__cis_regional_findings_data[key][event['region']] = True
-
+ if __event_with_topic_has_active_sub(event.get('actions')):
+ self.__cis_regional_findings_data[key][event['region']] = True
+
# Cloud Guard Check is only required in the Cloud Guard Reporting Region
elif self.__cloud_guard_config and key == "4.15" and \
event['region'] == self.__cloud_guard_config.reporting_region and \
(all(x in eventtype_dict['eventtype'] for x in changes)):
- self.cis_foundations_benchmark_3_0[key]['Status'] = True
-
+ if __event_with_topic_has_active_sub(event.get('actions')):
+ self.cis_foundations_benchmark_3_0[key]['Status'] = True
+
# For Checks that are home region based checking those
elif (all(x in eventtype_dict['eventtype'] for x in changes)) and \
key not in self.__cis_regional_checks and event['region'] == self.__home_region:
- self.cis_foundations_benchmark_3_0[key]['Status'] = True
+ if __event_with_topic_has_active_sub(event.get('actions')):
+ self.cis_foundations_benchmark_3_0[key]['Status'] = True
+
except Exception as e:
print(e)
print("*** Invalid Event Data for event: " + event['display_name'] + " ***")
@@ -4674,7 +4759,7 @@ def __cis_check_tagging_and_monitoring(self):
else:
for region in non_compliant:
self.cis_foundations_benchmark_3_0[key]['Findings'].append({'region' : region})
-
+
### Testing ###
# CIS Check 4.13 - VCN FlowLog enable
# Generate list of subnets IDs
@@ -4684,7 +4769,7 @@ def __cis_check_tagging_and_monitoring(self):
try:
if self.__all_logs and 'flowlogs' in self.__all_logs and \
'vcn' in self.__all_logs['flowlogs'] and vcn_id in self.__all_logs['flowlogs']['vcn']:
-
+
debug(f"__report_cis_analyze_tenancy_data: Flowlogs checking VCN {vcn_id} for Subnet: {subnet['id']} ")
if self.__all_logs['flowlogs']['vcn'][vcn_id]['capture_filter']:
capture_filter_id = self.__all_logs['flowlogs']['vcn'][vcn_id]['capture_filter']
@@ -4699,14 +4784,14 @@ def __cis_check_tagging_and_monitoring(self):
self.cis_foundations_benchmark_3_0['4.13']['Findings'].append(subnet)
elif self.__all_logs and 'flowlogs' in self.__all_logs and \
- 'subnet' in self.__all_logs['flowlogs'] and subnet['id'] in self.__all_logs['flowlogs']['subnet']:
-
+ 'subnet' in self.__all_logs['flowlogs'] and subnet['id'] in self.__all_logs['flowlogs']['subnet']:
+
debug(f"__report_cis_analyze_tenancy_data: Flowlogs checking Subnet {subnet['id']} in subnet")
debug(self.__all_logs['flowlogs']['subnet'][subnet['id']]['capture_filter'])
if self.__all_logs['flowlogs']['subnet'][subnet['id']]['capture_filter']:
debug(f"__report_cis_analyze_tenancy_data: Flowlogs checking Subnet {subnet['id']} capture filter in subnet")
capture_filter_id = self.__all_logs['flowlogs']['subnet'][subnet['id']]['capture_filter']
- capture_filter = self.__network_capturefilters[capture_filter_id]
+ capture_filter = self.__network_capturefilters[capture_filter_id]
if not(self.all_traffic_rules[0] in capture_filter['additional_details']['flowLogCaptureFilterRules'] or \
self.all_traffic_rules[1] in capture_filter['additional_details']['flowLogCaptureFilterRules']):
# VCN is being logging but it is has a capture filter we need to check
@@ -4716,14 +4801,14 @@ def __cis_check_tagging_and_monitoring(self):
elif self.__all_logs and self.__all_logs['flowlogs'] and \
'all' in self.__all_logs['flowlogs'] and subnet['id'] in self.__all_logs['flowlogs']['all']:
-
+
debug(f"__report_cis_analyze_tenancy_data: Flowlogs checking Subnet {subnet['id']} in all")
debug(self.__all_logs['flowlogs']['all'][subnet['id']]['capture_filter'])
if self.__all_logs['flowlogs']['all'][subnet['id']]['capture_filter']:
debug(f"__report_cis_analyze_tenancy_data: Flowlogs checking Subnet {subnet['id']} capture filter in all")
capture_filter_id = self.__all_logs['flowlogs']['all'][subnet['id']]['capture_filter']
- capture_filter = self.__network_capturefilters[capture_filter_id]
+ capture_filter = self.__network_capturefilters[capture_filter_id]
if not(self.all_traffic_rules[0] in capture_filter['additional_details']['flowLogCaptureFilterRules'] or \
self.all_traffic_rules[1] in capture_filter['additional_details']['flowLogCaptureFilterRules']):
# VCN is being logging but it is has a capture filter we need to check
@@ -4737,7 +4822,7 @@ def __cis_check_tagging_and_monitoring(self):
self.cis_foundations_benchmark_3_0['4.13']['Findings'].append(subnet)
except Exception as e:
- self.cis_foundations_benchmark_3_0['4.13']['Status'] = False
+ self.cis_foundations_benchmark_3_0['4.13']['Status'] = False
if ".capturefilter." in str(e):
print(f"Unable to read capturefilter rules for: {str(e)}.\n*** Please ensure your auditor has permissions: 'to read capture-filters in tenancy. ***")
self.__errors.append({"id" : str(e), "error" : "Unable to read capturefilter rules *** Please ensure your auditor has permissions: 'to read capture-filters in tenancy'."})
@@ -4747,7 +4832,7 @@ def __cis_check_tagging_and_monitoring(self):
msg = f'Unable to process all logs and capture filter rules: {str(e)}'
print(msg)
self.__errors.append({"id": "__network_subnet_logs", "error": msg})
-
+
# CIS Check 4.13 Total - Adding All Subnets to total
self.cis_foundations_benchmark_3_0['4.13']['Total'] = self.__network_subnets
@@ -4761,21 +4846,21 @@ def __cis_check_tagging_and_monitoring(self):
# CIS Check 4.16 - Encryption keys over 365
# Generating list of keys
for key in self.__kms_keys:
+ if key.get('is_primary'):
+ try:
+ if self.kms_key_time_max_datetime and self.kms_key_time_max_datetime >= datetime.datetime.strptime(key['currentKeyVersion_time_created'], self.__iso_time_format):
+ self.cis_foundations_benchmark_3_0['4.16']['Status'] = False
+ self.cis_foundations_benchmark_3_0['4.16']['Findings'].append(
+ key)
+ if self.kms_key_time_max_datetime is None:
+ self.cis_foundations_benchmark_3_0['4.16']['Status'] = False
+ self.cis_foundations_benchmark_3_0['4.16']['Findings'].append(
+ key)
+ except Exception:
+ self.cis_foundations_benchmark_3_0['4.16']['Status'] = False
+ self.cis_foundations_benchmark_3_0['4.16']['Findings'].append(
+ key)
- try:
- if self.kms_key_time_max_datetime and self.kms_key_time_max_datetime >= datetime.datetime.strptime(key['currentKeyVersion_time_created'], self.__iso_time_format):
- self.cis_foundations_benchmark_3_0['4.16']['Status'] = False
- self.cis_foundations_benchmark_3_0['4.16']['Findings'].append(
- key)
- if self.kms_key_time_max_datetime is None:
- self.cis_foundations_benchmark_3_0['4.16']['Status'] = False
- self.cis_foundations_benchmark_3_0['4.16']['Findings'].append(
- key)
- except Exception:
- self.cis_foundations_benchmark_3_0['4.16']['Status'] = False
- self.cis_foundations_benchmark_3_0['4.16']['Findings'].append(
- key)
-
# CIS Check 4.16 Total - Adding Key to total
self.cis_foundations_benchmark_3_0['4.16']['Total'].append(key)
@@ -4783,7 +4868,7 @@ def __cis_check_tagging_and_monitoring(self):
# Generating list of buckets names and need to make sure they have write level bucekt logs
if self.__all_logs and 'objectstorage' in self.__all_logs and\
'write' in self.__all_logs['objectstorage']:
-
+
for bucket in self.__buckets:
if not (bucket['name'] + "-" + bucket['region'] in self.__all_logs['objectstorage']['write']):
self.cis_foundations_benchmark_3_0['4.17']['Status'] = False
@@ -4917,10 +5002,10 @@ def __obp_init_regional_checks(self):
"status": False
}
}
-
-
+
+
##########################################################################
- # OBP Budgets Check
+ # OBP Budgets Check
##########################################################################
def __obp_check_budget(self):
if len(self.__budgets) > 0:
@@ -4933,23 +5018,23 @@ def __obp_check_budget(self):
if alert.type == "FORECAST":
self.obp_foundations_checks["Cost_Tracking_Budgets"]["Status"] = True
self.obp_foundations_checks["Cost_Tracking_Budgets"]["OBP"].append(budget)
- break
- else:
+ break
+ else:
self.obp_foundations_checks["Cost_Tracking_Budgets"]["Findings"].append(budget)
else:
self.obp_foundations_checks["Cost_Tracking_Budgets"]["Findings"].append(budget)
#######################################
# OBP Quotas Checks
- #######################################
+ #######################################
def __obp_check_quotas(self):
if self.__quotas:
self.obp_foundations_checks['Quotas']['Status'] = True
self.obp_foundations_checks['Quotas']['OBP'] = self.__quotas
-
+
#######################################
# OBP Audit Logs to SIEM check
- #######################################
+ #######################################
def __obp_check_audit_log_compartments(self):
# Building a Hash Table of Parent Child Hierarchy for Audit
dict_of_compartments = {}
@@ -4960,7 +5045,6 @@ def __obp_check_audit_log_compartments(self):
except Exception:
dict_of_compartments[compartment.compartment_id] = []
dict_of_compartments[compartment.compartment_id].append(compartment.id)
-
# Collecting Service Connectors Logs related to compartments
for sch_id, sch_values in self.__service_connectors.items():
# Only Active SCH with a target that is configured
@@ -4971,7 +5055,7 @@ def __obp_check_audit_log_compartments(self):
if source['compartment_id'] == self.__tenancy.id and source['log_group_id'].upper() == "_Audit_Include_Subcompartment".upper():
self.__obp_regional_checks[sch_values['region']]['Audit']['tenancy_level_audit'] = True
self.__obp_regional_checks[sch_values['region']]['Audit']['tenancy_level_include_sub_comps'] = True
-
+
# Since it is not the Tenancy we should add the compartment to the list and check if sub compartment are included
elif source['log_group_id'].upper() == "_Audit_Include_Subcompartment".upper():
self.__obp_regional_checks[sch_values['region']]['Audit']['compartments'] += self.__get_children(source['compartment_id'], dict_of_compartments)
@@ -4980,7 +5064,7 @@ def __obp_check_audit_log_compartments(self):
except Exception:
# There can be empty log groups
pass
-
+
# Analyzing Service Connector Audit Logs to see if each region has all compartments
for region_key, region_values in self.__obp_regional_checks.items():
# Checking if I already found the tenancy ocid with all child compartments included
@@ -4994,24 +5078,24 @@ def __obp_check_audit_log_compartments(self):
else:
region_values['Audit']['tenancy_level_audit'] = True
region_values['Audit']['findings'] = []
-
+
# Consolidating Audit findings into the OBP Checks
for region_key, region_values in self.__obp_regional_checks.items():
# If this flag is set all compartments are not logged in region
if not region_values['Audit']['tenancy_level_audit']:
self.obp_foundations_checks['SIEM_Audit_Log_All_Comps']['Status'] = False
-
+
# If this flag is set the region has the tenancy logging and all sub compartments flag checked
if not region_values['Audit']['tenancy_level_include_sub_comps']:
self.obp_foundations_checks['SIEM_Audit_Incl_Sub_Comp']['Status'] = False
self.obp_foundations_checks['SIEM_Audit_Incl_Sub_Comp']['Findings'].append({"region_name": region_key})
else:
self.obp_foundations_checks['SIEM_Audit_Incl_Sub_Comp']['OBP'].append({"region_name": region_key})
-
+
# Compartment Logs that are missed in the region
for compartment in region_values['Audit']['findings']:
try:
- finding = list(filter(lambda source: source['id'] == compartment, self.__raw_compartment))[0]
+ finding = list(filter(lambda source: source['id'] == compartment, self.__raw_compartment.values()))[0]
record = {
"id": finding['id'],
"name": finding['name'],
@@ -5045,11 +5129,11 @@ def __obp_check_audit_log_compartments(self):
exists_already = list(filter(lambda source: source['id'] == record['id'] and source['region'] == record['region'], self.obp_foundations_checks['SIEM_Audit_Log_All_Comps']['Findings']))
if not exists_already:
self.obp_foundations_checks['SIEM_Audit_Log_All_Comps']['Findings'].append(record)
-
+
# Compartment logs that are not missed in the region
for compartment in region_values['Audit']['compartments']:
try:
- finding = list(filter(lambda source: source['id'] == compartment, self.__raw_compartment))[0]
+ finding = list(filter(lambda source: source['id'] == compartment, self.__raw_compartment.values()))[0]
record = {
"id": finding['id'],
"name": finding['name'],
@@ -5083,10 +5167,10 @@ def __obp_check_audit_log_compartments(self):
exists_already = list(filter(lambda source: source['id'] == record['id'] and source['region'] == record['region'], self.obp_foundations_checks['SIEM_Audit_Log_All_Comps']['OBP']))
if not exists_already:
self.obp_foundations_checks['SIEM_Audit_Log_All_Comps']['OBP'].append(record)
-
+
#######################################
# OBP Cloud Guard Check
- #######################################
+ #######################################
def __obp_check_cloud_guard(self):
#######################################
# Cloud Guard Checks
@@ -5108,7 +5192,6 @@ def __obp_check_cloud_guard(self):
try:
# Cloud Guard Target attached to the root compartment with activity, config, and threat detector plus a responder
if self.__cloud_guard_targets[self.__tenancy.id]:
-
cloud_guard_record['target_at_root'] = True
if self.__cloud_guard_targets[self.__tenancy.id]:
@@ -5146,7 +5229,7 @@ def __obp_check_cloud_guard(self):
pass
all_cloud_guard_checks = True
- for key, value in cloud_guard_record.items():
+ for value in cloud_guard_record.values():
if not (value):
all_cloud_guard_checks = False
@@ -5156,6 +5239,20 @@ def __obp_check_cloud_guard(self):
else:
self.obp_foundations_checks['Cloud_Guard_Config']['Findings'].append(cloud_guard_record)
+ for problem in self.__cloud_guard_problems:
+ if problem['risk_level'] in ['HIGH','CRITICAL']:
+ self.obp_foundations_checks['Cloud_Guard_Problems']['Findings'].append(problem)
+ else:
+ self.obp_foundations_checks['Cloud_Guard_Problems']['OBP'].append(problem)
+
+ for key in self.obp_foundations_checks.keys():
+ if key.startswith("Cloud_Guard_"):
+ if self.obp_foundations_checks[key]['Findings']:
+ self.obp_foundations_checks[key]['Status'] = False
+ else:
+ self.obp_foundations_checks[key]['Status'] = True
+
+
#######################################
# OBP Networking Checks
#######################################
@@ -5166,9 +5263,11 @@ def __obp_check_networking(self):
number_of_valid_connected_vcns = 0
number_of_valid_fast_connect_circuits = 0
number_of_valid_site_to_site_connection = 0
+ number_of_valid_bgp_site_to_site_connection = 0
fast_connect_providers = set()
customer_premises_equipment = set()
+ drg_upgrade_status = self.__network_drgs[drg_id]['upgrade_status']
for attachment in drg_values:
if attachment['network_type'].upper() == 'VCN':
@@ -5180,8 +5279,10 @@ def __obp_check_networking(self):
for ipsec_connection in self.__network_ipsec_connections[drg_id]:
if ipsec_connection['tunnels_up']:
# Good IP Sec Connection increment valid site to site and track CPEs
- customer_premises_equipment.add(ipsec_connection['cpe_id'])
number_of_valid_site_to_site_connection += 1
+ customer_premises_equipment.add(ipsec_connection['cpe_id'])
+ if ipsec_connection['tunnels_bgp']:
+ number_of_valid_bgp_site_to_site_connection += 1
elif attachment['network_type'].upper() == 'VIRTUAL_CIRCUIT':
@@ -5200,10 +5301,12 @@ def __obp_check_networking(self):
record = {
"drg_id": drg_id,
"drg_display_name": self.__network_drgs[drg_id]['display_name'],
+ "upgrade_status": drg_upgrade_status,
"region": self.__network_drgs[drg_id]['region'],
"number_of_connected_vcns": number_of_valid_connected_vcns,
"number_of_customer_premises_equipment": len(customer_premises_equipment),
"number_of_connected_ipsec_connections": number_of_valid_site_to_site_connection,
+ "number_of_valid_bgp_site_to_site_connection": number_of_valid_bgp_site_to_site_connection,
"number_of_fastconnects_cicruits": number_of_valid_fast_connect_circuits,
"number_of_fastconnect_providers": len(fast_connect_providers),
}
@@ -5211,48 +5314,81 @@ def __obp_check_networking(self):
record = {
"drg_id": drg_id,
"drg_display_name": "Deleted with an active attachement",
+ "upgrade_status" : None,
"region": attachment['region'],
"number_of_connected_vcns": 0,
"number_of_customer_premises_equipment": 0,
"number_of_connected_ipsec_connections": 0,
+ "number_of_valid_bgp_site_to_site_connection": 0,
"number_of_fastconnects_cicruits": 0,
"number_of_fastconnect_providers": 0,
}
print(f"This DRG: {drg_id} is deleted with an active attachement: {attachment['display_name']}")
+ self.__errors.append({"id" : str(drg_id), "error" : f"This DRG: {drg_id} is deleted with an active attachement: {attachment['display_name']}"})
+
# Checking if the DRG and connected resourcs are aligned with best practices
# One attached VCN, One VPN connection and one fast connect
- if number_of_valid_connected_vcns and number_of_valid_site_to_site_connection and number_of_valid_fast_connect_circuits:
+ if number_of_valid_connected_vcns >= 2 and number_of_valid_site_to_site_connection and number_of_valid_fast_connect_circuits:
self.__obp_regional_checks[record['region']]["Network_Connectivity"]["drgs"].append(record)
self.__obp_regional_checks[record['region']]["Network_Connectivity"]["status"] = True
# Two VPN site to site connections to seperate CPEs
- elif number_of_valid_connected_vcns and number_of_valid_site_to_site_connection and len(customer_premises_equipment) >= 2:
+ elif number_of_valid_connected_vcns >= 2 and number_of_valid_site_to_site_connection and len(customer_premises_equipment) >= 2:
self.__obp_regional_checks[record['region']]["Network_Connectivity"]["drgs"].append(record)
self.__obp_regional_checks[record['region']]["Network_Connectivity"]["status"] = True
# Two FastConnects from Different providers
- elif number_of_valid_connected_vcns and number_of_valid_fast_connect_circuits and len(fast_connect_providers) >= 2:
+ elif number_of_valid_connected_vcns >= 2 and number_of_valid_fast_connect_circuits and len(fast_connect_providers) >= 2:
self.__obp_regional_checks[record['region']]["Network_Connectivity"]["drgs"].append(record)
self.__obp_regional_checks[record['region']]["Network_Connectivity"]["status"] = True
else:
self.__obp_regional_checks[record['region']]["Network_Connectivity"]["findings"].append(record)
+ # Checking if this there is a hub and spoke basically two VCNs to one DRG
+ if not number_of_valid_connected_vcns >= 2:
+ self.obp_foundations_checks['Networking_Hub_Spoke']['Findings'].append(record)
+ else:
+ self.obp_foundations_checks['Networking_Hub_Spoke']['OBP'].append(record)
+
+ # Checking if this DRG has been upgraded to Version 2
+ if not drg_upgrade_status == "UPGRADED":
+ self.obp_foundations_checks["Networking_DRG_Upgraded"]["Findings"].append(record)
+ else:
+ self.obp_foundations_checks["Networking_DRG_Upgraded"]["OBP"].append(record)
+
+ # Checking if this DRG IPSec connection has two connetions
+ if not number_of_valid_site_to_site_connection >= 2:
+ self.obp_foundations_checks["Networking_IPSec_connections"]["Findings"].append(record)
+ else:
+ self.obp_foundations_checks["Networking_IPSec_connections"]["OBP"].append(record)
+
+ # Checking if this DRG IPSec connection has two connetions
+ if not number_of_valid_bgp_site_to_site_connection >= 2:
+ self.obp_foundations_checks["Networking_IPSec_bgp"]["Findings"].append(record)
+ else:
+ self.obp_foundations_checks["Networking_IPSec_bgp"]["OBP"].append(record)
+
+
# Consolidating Regional
- for region_key, region_values in self.__obp_regional_checks.items():
+ for region_values in self.__obp_regional_checks.values():
# I assume you are well connected in all regions if find one region that is not it fails
- if not region_values["Network_Connectivity"]["status"]:
- self.obp_foundations_checks['Networking_Connectivity']['Status'] = False
+ self.obp_foundations_checks["Networking_Redudancy"]["Findings"] += region_values["Network_Connectivity"]["findings"]
+ self.obp_foundations_checks["Networking_Redudancy"]["OBP"] += region_values["Network_Connectivity"]["drgs"]
- self.obp_foundations_checks["Networking_Connectivity"]["Findings"] += region_values["Network_Connectivity"]["findings"]
- self.obp_foundations_checks["Networking_Connectivity"]["OBP"] += region_values["Network_Connectivity"]["drgs"]
+ for key in self.obp_foundations_checks.keys():
+ if key.startswith("Networking_"):
+ if self.obp_foundations_checks[key]['Findings']:
+ self.obp_foundations_checks[key]['Status'] = False
+ else:
+ self.obp_foundations_checks[key]['Status'] = True
#######################################
# OBP Certificate Expiry Check
- #######################################
+ #######################################
def __obp_check_certificates(self):
for cert in self.__raw_oci_certificates:
debug("\t__obp_analyze_tenancy_data: Iterating through certificates")
-
+
try:
if cert['current_version_summary']['validity'] and \
datetime.datetime.strptime(self.get_date_iso_format(cert['current_version_summary']['validity']['time_of_validity_not_after']), self.__iso_time_format) >= self.cert_key_time_max_datetime:
@@ -5278,7 +5414,7 @@ def __obp_check_subnet_logs(self):
cis_logged_subnets.add(subnet['id'])
for subnet in self.cis_foundations_benchmark_3_0['4.13']['Total']:
all_subnet_nets.add(subnet['id'])
-
+
list_of_properly_logged_subnets = all_subnet_nets - cis_logged_subnets
# need to check for no logs
for sch_id, sch_values in self.__service_connectors.items():
@@ -5294,19 +5430,19 @@ def __obp_check_subnet_logs(self):
for vcn_id, vcn_values in self.__network_vcns.items():
if subnet_id in vcn_values['subnets']:
log_values = self.__all_logs['flowlogs']['vcn'][vcn_id]
-
+
log_id = log_values['id']
log_group_id = log_values['log_group_id']
log_record = {"sch_id": sch_id, "sch_name": sch_values['display_name'], "id": subnet_id}
-
+
subnet_log_group_in_sch = list(filter(lambda source: source['log_group_id'] == log_group_id, sch_values['log_sources']))
subnet_log_in_sch = list(filter(lambda source: source['log_id'] == log_id, sch_values['log_sources']))
-
+
if subnet_log_group_in_sch and not (subnet_log_in_sch):
self.__obp_regional_checks[sch_values['region']]['VCN']['subnets'].append(log_record)
elif subnet_log_in_sch:
self.__obp_regional_checks[sch_values['region']]['VCN']['subnets'].append(log_record)
-
+
for region_values in self.__obp_regional_checks.values():
for finding in region_values['VCN']['subnets']:
logged_subnet = list(filter(lambda subnet: subnet['id'] == finding['id'], self.__network_subnets))
@@ -5317,12 +5453,12 @@ def __obp_check_subnet_logs(self):
record['sch_name'] = finding['sch_name']
if logged_subnet and not (existing_finding):
self.obp_foundations_checks['SIEM_VCN_Flow_Logging']['OBP'].append(record)
-
+
for finding in self.__network_subnets:
logged_subnet = list(filter(lambda subnet: subnet['id'] == finding['id'], self.obp_foundations_checks['SIEM_VCN_Flow_Logging']['OBP']))
if not (logged_subnet):
self.obp_foundations_checks['SIEM_VCN_Flow_Logging']['Findings'].append(finding)
-
+
if self.obp_foundations_checks['SIEM_VCN_Flow_Logging']['Findings']:
self.obp_foundations_checks['SIEM_VCN_Flow_Logging']['Status'] = False
else:
@@ -5347,7 +5483,7 @@ def __obp_check_bucket_logs(self):
bucket_log_group_in_sch = any(source['log_group_id'] == log_group_id and sch_values['region'] == log_region for source in sch_values['log_sources'])
bucket_log_in_sch = any(source['log_id'] == log_id and sch_values['region'] == log_region for source in sch_values['log_sources'])
-
+
# Checking if the Bucket's log group in is in SCH's log sources & the log_id is empty so it covers everything in the log group
if bucket_log_group_in_sch and not (bucket_log_in_sch):
self.__obp_regional_checks[sch_values['region']]['Write_Bucket']['buckets'].append(log_record)
@@ -5432,27 +5568,85 @@ def __obp_check_bucket_logs(self):
self.obp_foundations_checks['SIEM_Read_Bucket_Logs']['Findings'] += self.__buckets
else:
self.obp_foundations_checks['SIEM_Read_Bucket_Logs']['Status'] = True
-
+
+ #######################################
+ # OBP Subnet and Bucket Log Checks
+ #######################################
+ def __obp_check_log_retention(self):
+ for log_group in self.__logging_list:
+ for log in log_group['logs']:
+ required_keys = ['id', 'display_name', 'deep_link', 'compartment_id', 'is_enabled',
+ 'lifecycle_state', 'log_group_id', 'log_type', 'retention_duration',
+ 'time_created', 'time_last_modified', 'defined_tags', 'freeform_tags', 'region']
+ record = {key: log[key] for key in required_keys if key in log}
+
+ if log.get('retention_duration') < 90:
+ self.obp_foundations_checks['Log_Retention']['Findings'].append(record)
+ else:
+ self.obp_foundations_checks['Log_Retention']['OBP'].append(record)
+
+ if self.obp_foundations_checks['Log_Retention']['Findings']:
+ self.obp_foundations_checks['Log_Retention']['Status'] = False
+ elif self.obp_foundations_checks['Log_Retention']['OBP']:
+ self.obp_foundations_checks['Log_Retention']['Status'] = True
+
+ #######################################
+ # OBP Account Lockout Duration 5 or more
+ #######################################
+ def __obp_check_iam_account_lockout(self):
+ for domain in self.__identity_domains:
+ required_keys = ['id', 'display_name', 'url', 'errors']
+ record = {key: domain[key] for key in required_keys if key in domain}
+ pwd_policy = domain.get('password_policy') or {}
+ record['max_incorrect_attempts'] = pwd_policy.get('max_incorrect_attempts')
+ record['lockout_duration'] = pwd_policy.get('lockout_duration')
+ try:
+ record['automatic_account_unlock_enabled'] = (
+ record['lockout_duration'] is not None and int(record['lockout_duration']) > 0
+ )
+ except Exception:
+ record['automatic_account_unlock_enabled'] = None
+ try:
+ # Missing/invalid max_incorrect_attempts is non-compliant.
+ # Require lockout to trigger at 5 failed attempts or fewer.
+ if record['max_incorrect_attempts'] is None or str(record['max_incorrect_attempts']).strip() == "":
+ max_incorrect_attempts_compliant = False
+ else:
+ max_incorrect_attempts_compliant = (int(record['max_incorrect_attempts']) <= 5)
+ except Exception:
+ max_incorrect_attempts_compliant = False
+ # automatic_account_unlock_enabled is captured for visibility only and
+ # is not used as a compliance gate unless policy requires it.
+ if max_incorrect_attempts_compliant:
+ self.obp_foundations_checks['IAM_Account_Lockout']['OBP'].append(record)
+ else:
+ self.obp_foundations_checks['IAM_Account_Lockout']['Findings'].append(record)
+
+ if self.obp_foundations_checks['IAM_Account_Lockout']['Findings']:
+ self.obp_foundations_checks['IAM_Account_Lockout']['Status'] = False
+ elif self.obp_foundations_checks['IAM_Account_Lockout']['OBP']:
+ self.obp_foundations_checks['IAM_Account_Lockout']['Status'] = True
+
#######################################
# OBP Service Limit Check
- #######################################
+ #######################################
def __obp_check_close_service_limits(self):
if True:
for limit in self.__service_limits:
# If the limit is greater than 80% we should note it for an OBP
- if limit['service_limit_availability'] and limit['service_limit_availability'] >= 80.0:
+ if limit['service_limit_availability'] and limit['service_limit_availability'] >= self.__service_utilization_limit:
self.obp_foundations_checks['Service_Limits']['Findings'].append(limit)
else:
self.obp_foundations_checks['Service_Limits']['OBP'].append(limit)
-
+
if self.obp_foundations_checks['Service_Limits']['Findings']:
self.obp_foundations_checks['Service_Limits']['Status'] = False
elif self.obp_foundations_checks['Service_Limits']['OBP']:
self.obp_foundations_checks['Service_Limits']['Status'] = True
#######################################
# OBP ADB Checks
- #######################################
+ #######################################
def __obp_check_adbs(self):
for adb in self.__autonomous_databases:
if not adb['is_mtls_connection_required']:
@@ -5467,13 +5661,13 @@ def __obp_check_adbs(self):
if not adb['private_endpoint_ip']:
self.obp_foundations_checks['ADB_Private_IP']['Findings'].append(adb)
else:
- self.obp_foundations_checks['ADB_Private_IP']['OBP'].append(adb)
+ self.obp_foundations_checks['ADB_Private_IP']['OBP'].append(adb)
if not adb['data_safe_status'] == "REGISTERED":
self.obp_foundations_checks['ADB_DataSafe']['Findings'].append(adb)
else:
self.obp_foundations_checks['ADB_DataSafe']['OBP'].append(adb)
-
+
if not adb['customer_contacts']:
self.obp_foundations_checks['ADB_Contacts']['Findings'].append(adb)
else:
@@ -5484,7 +5678,62 @@ def __obp_check_adbs(self):
if self.obp_foundations_checks[key]['Findings']:
self.obp_foundations_checks[key]['Status'] = False
else:
- self.obp_foundations_checks[key]['Status'] = True
+ self.obp_foundations_checks[key]['Status'] = True
+
+ ##########################################################################
+ # OBP Check Aggregated Policy statements in all Compartment Chains
+ ##########################################################################
+ def __obp_check_policy_statements_in_comp_chains(self):
+ policy_stmt_limit = 500.0 #Should be returned by the Service Limit API
+ parent_ids = {v.get("compartment_id") for v in self.__raw_compartment.values() if v.get("compartment_id") }
+ leaf_nodes = [cid for cid in self.__raw_compartment.keys() if cid not in parent_ids]
+
+ for leaf in leaf_nodes:
+ path = []
+ cid = leaf
+ total = 0
+ while cid is not None:
+ path.append(cid)
+ if cid == self.__tenancy.id:
+ break
+ cid = self.__raw_compartment[cid]['compartment_id']
+ parts = [f"{self.__raw_compartment[x]['name']}({self.__raw_compartment[x]['statements_in_compartment']})" for x in reversed(path)]
+ total = sum(self.__raw_compartment[x]['statements_in_compartment'] for x in path)
+
+ record = {
+ "total":total,
+ "compartment_hierarchy": ' --> '.join(parts)
+ }
+
+ #Check compartment hierarchy total statements against the statement threshold.
+ policy_stmt_utilization_per_hierarchy = round((total*100.0)/policy_stmt_limit)
+ if policy_stmt_utilization_per_hierarchy >= self.__service_utilization_limit or total >= policy_stmt_limit:
+ self.obp_foundations_checks['IAM_Stmt_Comp_Hierarchy_Count']['Findings'].append(record)
+ else:
+ self.obp_foundations_checks['IAM_Stmt_Comp_Hierarchy_Count']['OBP'].append(record)
+
+ if self.obp_foundations_checks['IAM_Stmt_Comp_Hierarchy_Count']['Findings']:
+ self.obp_foundations_checks['IAM_Stmt_Comp_Hierarchy_Count']['Status'] = False
+ elif self.obp_foundations_checks['IAM_Stmt_Comp_Hierarchy_Count']['OBP']:
+ self.obp_foundations_checks['IAM_Stmt_Comp_Hierarchy_Count']['Status'] = True
+
+ self.__compartment_hierarchy_info.append(record)
+
+ #Check root compartment total statements against the statement threshold.
+ policy_stmt_utilization_at_root = round((self.__raw_compartment[self.__tenancy.id]["statements_in_compartment"]*100.0)/policy_stmt_limit)
+ record = {"Statements_at_root":self.__raw_compartment[self.__tenancy.id]["statements_in_compartment"]}
+ if policy_stmt_utilization_at_root >= self.__service_utilization_limit or self.__raw_compartment[self.__tenancy.id]["statements_in_compartment"] >= policy_stmt_limit:
+ self.obp_foundations_checks['IAM_Stmt_Root_Count']['Findings'].append(record)
+ else:
+ self.obp_foundations_checks['IAM_Stmt_Root_Count']['OBP'].append(record)
+
+ if self.obp_foundations_checks['IAM_Stmt_Root_Count']['Findings']:
+ self.obp_foundations_checks['IAM_Stmt_Root_Count']['Status'] = False
+ elif self.obp_foundations_checks['IAM_Stmt_Root_Count']['OBP']:
+ self.obp_foundations_checks['IAM_Stmt_Root_Count']['Status'] = True
+
+
+
##########################################################################
# Analyzes Tenancy Data for Oracle Best Practices Report
@@ -5493,14 +5742,17 @@ def __obp_analyze_tenancy_data(self):
self.__obp_init_regional_checks()
self.__obp_check_budget()
self.__obp_check_audit_log_compartments()
- self.__obp_check_cloud_guard()
+ self.__obp_check_cloud_guard()
self.__obp_check_networking()
self.__obp_check_certificates()
self.__obp_check_bucket_logs()
self.__obp_check_subnet_logs()
+ self.__obp_check_log_retention()
self.__obp_check_close_service_limits()
self.__obp_check_adbs()
self.__obp_check_quotas()
+ self.__obp_check_policy_statements_in_comp_chains()
+ self.__obp_check_iam_account_lockout()
##########################################################################
# Orchestrates data collection and CIS report generation
@@ -5528,7 +5780,7 @@ def __report_generate_cis_report(self, level):
/ len(recommendation.get('Total') or []) * 100
)) if len(recommendation.get('Total') or []) > 0 else 0) + "%")
record = {
- "Recommendation #": f"{key}",
+ "Recommendation #": f"{key}",
"Section": recommendation['section'],
"Level": str(recommendation['Level']),
"Compliant": compliant_output if compliant_output != "Not Applicable" else "N/A",
@@ -5725,6 +5977,8 @@ def __report_generate_html_summary_report(self, header, file_subject, data):
for key in item.keys():
str_item = str(item[key])
items_to_redact = re.findall(self.__oci_ocid_pattern, str_item)
+ emails_to_redact = re.findall(self.__simple_email, str_item)
+ items_to_redact += emails_to_redact
for redact_me in items_to_redact:
str_item = str_item.replace(redact_me, hashlib.sha256(str.encode(redact_me)).hexdigest())
@@ -6000,9 +6254,12 @@ def __report_generate_obp_report(self):
print(padding + "\t\t" + compliant + "\t" + "\t" + str(len(recommendation['Findings'])) + "\t" + "\t" + str(len(recommendation['OBP'])))
record = {
"Recommendation": str(key),
+ "ID" : recommendation['id'],
+ "Section": recommendation['section'],
"Compliant": compliant,
"OBP": (str(len(recommendation['OBP'])) if len(recommendation['OBP']) > 0 else " "),
"Findings": (str(len(recommendation['Findings'])) if len(recommendation['Findings']) > 0 else " "),
+ "Title": recommendation['Title'],
"Documentation": recommendation['Documentation']
}
obp_summary_report.append(record)
@@ -6042,7 +6299,7 @@ def __collect_tenancy_data(self):
thread_compartments.join()
thread_cloud_guard_config.join()
-
+
thread_identity_domains = Thread(target=self.__identity_read_domains)
thread_identity_domains.start()
thread_identity_domains.join()
@@ -6052,7 +6309,7 @@ def __collect_tenancy_data(self):
thread_identity_groups.join()
print("\nProcessing Home Region resources...")
-
+
cis_home_region_functions = [
self.__identity_read_users,
self.__identity_read_tenancy_password_policy,
@@ -6060,6 +6317,7 @@ def __collect_tenancy_data(self):
self.__identity_read_availability_domains,
self.__identity_read_tag_defaults,
self.__identity_read_tenancy_policies,
+ self.__read_cloud_guard_problems
]
# Budgets is global construct
@@ -6165,7 +6423,7 @@ def __report_generate_raw_data_output(self):
"identity_policies": self.__policies,
"identity_dynamic_groups": self.__dynamic_groups,
"identity_tags": self.__tag_defaults,
- "identity_compartments": self.__raw_compartment,
+ "identity_compartments": list(self.__raw_compartment.values()),
"network_security_groups": self.__network_security_groups,
"network_security_lists": self.__network_security_lists,
"network_subnets": self.__network_subnets,
@@ -6180,7 +6438,8 @@ def __report_generate_raw_data_output(self):
"boot_volumes": self.__boot_volumes,
"block_volumes": self.__block_volumes,
"file_storage_system": self.__file_storage_system,
- "keys_and_vaults": self.__kms_keys,
+ "kms_keys": self.__kms_keys,
+ "kms_vaults": list(self.__vaults.values()),
"ons_subscriptions": self.__subscriptions,
"budgets": self.__budgets,
"quotas" : self.__quotas,
@@ -6189,11 +6448,13 @@ def __report_generate_raw_data_output(self):
"network_ipsec_connections": list(itertools.chain.from_iterable(self.__network_ipsec_connections.values())),
"network_drgs": self.__raw_network_drgs,
"cloud_guard_target": list(self.__cloud_guard_targets.values()),
+ "cloud_guard_problems" : self.__cloud_guard_problems,
"regions": self.__raw_regions,
"network_drg_attachments": list(itertools.chain.from_iterable(self.__network_drg_attachments.values())),
"instances": self.__Instance,
"certificates" : self.__raw_oci_certificates,
- "service_limits" : self.__service_limits
+ "service_limits" : self.__service_limits,
+ "compartment_hierarchy_policy_count" : self.__compartment_hierarchy_info
}
for key in raw_csv_files:
rfn = self.__print_to_csv_file('raw_data', key, raw_csv_files[key])
@@ -6216,7 +6477,7 @@ def __report_generate_raw_data_output(self):
##########################################################################
def __os_copy_report_to_object_storage(self, bucketname, filename):
object_name = filename
- # print(self.__os_namespace)
+ debug(f"__os_copy_report_to_object_storage: Writing to {filename} to namespace: {self.__os_namespace} bucket: {bucketname}")
try:
with open(filename, "rb") as f:
try:
@@ -6271,6 +6532,8 @@ def __print_to_csv_file(self, header, file_subject, data):
for key in item.keys():
str_item = str(item[key])
items_to_redact = re.findall(self.__oci_ocid_pattern, str_item)
+ emails_to_redact = re.findall(self.__simple_email, str_item)
+ items_to_redact += emails_to_redact
for redact_me in items_to_redact:
str_item = str_item.replace(redact_me, hashlib.sha256(str.encode(redact_me)).hexdigest())
@@ -6312,16 +6575,18 @@ def __print_to_json_file(self, header, file_subject, data):
# if no data
if len(data) == 0:
return None
-
+
# get the file name of the JSON
file_path = self.__get_output_file_path(header, file_subject, '.json')
# Serializing JSON to string
json_object = json.dumps(data, indent=4)
-
+
# If this flag is set all OCIDs are Hashed to redact them
if self.__redact_output:
items_to_redact = re.findall(self.__oci_ocid_pattern,json_object)
+ emails_to_redact = re.findall(self.__simple_email, json_object)
+ items_to_redact += emails_to_redact
for redact_me in items_to_redact:
json_object = json_object.replace(redact_me,hashlib.sha256(str.encode(redact_me)).hexdigest() )
@@ -6329,16 +6594,16 @@ def __print_to_json_file(self, header, file_subject, data):
# Writing to json file
with open(file_path, mode='w', newline='') as json_file:
json_file.write(json_object)
-
+
print("JSON: " + file_subject.ljust(22) + " --> " + file_path)
-
+
# Used by Upload
return file_path
-
+
except Exception as e:
raise Exception("Error in print_to_json_file: " + str(e.args))
-
-
+
+
##########################################################################
# Orchestrates Data collection and reports
##########################################################################
@@ -6361,13 +6626,11 @@ def generate_reports(self, level=2):
if self.__output_raw_data:
self.__report_generate_raw_data_output()
- if self.__errors:
- error_report = self.__print_to_csv_file("error", "report", self.__errors)
+ error_report_file_name = self.__print_to_csv_file("error", "report", self.__errors)
- if self.__output_bucket:
- if error_report:
- self.__os_copy_report_to_object_storage(
- self.__output_bucket, error_report)
+ if error_report_file_name and self.__output_bucket:
+ self.__os_copy_report_to_object_storage(
+ self.__output_bucket, error_report_file_name)
end_datetime = datetime.datetime.now().replace(tzinfo=pytz.UTC)
end_time_str = str(end_datetime.strftime("%Y-%m-%dT%H:%M:%S"))
@@ -6541,6 +6804,56 @@ def set_parser_arguments():
##########################################################################
# execute_report
##########################################################################
+def _build_worksheet_name(csv_path, report_prefix, seen_names):
+ """Return a workbook-safe worksheet name or None if the sheet should be skipped."""
+ base_name = os.path.basename(csv_path)
+ if report_prefix:
+ base_name = base_name.replace(report_prefix, "")
+
+ name = (base_name
+ .replace(".csv", "")
+ .replace("raw_data_", "raw_")
+ .replace("Findings", "fds")
+ .replace("Best_Practices", "OBP"))
+
+ if "raw_cloud_guard_target" in name:
+ return None
+
+ replacement_rules = (
+ ("Identity_and_Access_Management", "IAM"),
+ ("Storage_Object_Storage", "Object_Storage"),
+ ("raw_identity_groups_and_membership", "raw_iam_groups_and_membership"),
+ ("Cost_Tracking_Budgets_Best_Practices", "Budgets_Best_Practices"),
+ ("Cost_Tracking", "Cost"),
+ ("Storage_File_Storage_Service", "FSS"),
+ ("Networking_IPSec_connections", "Networking_IPSec"),
+ ("IAM_Stmt_Comp_Hierarchy_Count", "IAM_Stmt_Comp_Count"),
+ ("compartment_hierarchy_policy_count", "compartment_policy_cnt"),
+ )
+
+ for pattern, replacement in replacement_rules:
+ if pattern in name:
+ name = name.replace(pattern, replacement)
+ break
+
+ name = re.sub(r"_+", "_", name).strip("_") or "worksheet"
+
+ if len(name) > 31:
+ name = name.replace("_", "")
+ if len(name) > 31:
+ name = name[:28]
+
+ candidate = name
+ counter = 1
+ while candidate in seen_names or len(candidate) > 31:
+ suffix = f"_{counter}"
+ candidate = f"{name[:31 - len(suffix)]}{suffix}" if len(name) + len(suffix) > 31 else f"{name}{suffix}"
+ counter += 1
+
+ seen_names.add(candidate)
+ return candidate
+
+
def execute_report():
# Get Command Line Parser
@@ -6581,12 +6894,12 @@ def execute_report():
dest='is_instance_principals', help='Use Instance Principals for Authentication.')
parser.add_argument('-dt', action='store_true', default=False,
dest='is_delegation_token', help='Use Delegation Token for Authentication in Cloud Shell.')
- parser.add_argument('-st', action='store_true', default=False,
+ parser.add_argument('-st', action='store_true', default=False,
dest='is_security_token', help='Authenticate using Security Token.')
parser.add_argument('-v', action='store_true', default=False,
dest='version', help='Show the version of the script and exit.')
parser.add_argument('--debug', action='store_true', default=False,
- dest='debug', help='Enables debugging messages printed to screen.')
+ dest='debug', help='Enables debugging messages printed to screen.')
cmd = parser.parse_args()
if cmd.version:
@@ -6612,40 +6925,38 @@ def execute_report():
pass
csvfiles = glob.glob(f'{csv_report_directory}/{report_prefix}*.csv')
csvfiles.sort()
+ seen_worksheet_names = set()
for csvfile in csvfiles:
+ worksheet_name = _build_worksheet_name(csvfile, report_prefix, seen_worksheet_names)
+ if not worksheet_name:
+ continue
- worksheet_name = csvfile.split(os.path.sep)[-1].replace(report_prefix, "").replace(".csv", "").replace("raw_data_", "raw_").replace("Findings", "fds").replace("Best_Practices", "bps")
-
- if "Identity_and_Access_Management" in worksheet_name:
- worksheet_name = worksheet_name.replace("Identity_and_Access_Management", "IAM")
- elif "Storage_Object_Storage" in worksheet_name:
- worksheet_name = worksheet_name.replace("Storage_Object_Storage", "Object_Storage")
- elif "raw_identity_groups_and_membership" in worksheet_name:
- worksheet_name = worksheet_name.replace("raw_identity", "raw_iam")
- elif "Cost_Tracking_Budgets_Best_Practices" in worksheet_name:
- worksheet_name = worksheet_name.replace("Cost_Tracking_", "")
- elif "Storage_File_Storage_Service" in worksheet_name:
- worksheet_name = worksheet_name.replace("Storage_File_Storage_Service", "FSS")
- elif "raw_cloud_guard_target" in worksheet_name:
- # cloud guard targets are too large for a cell
+ try:
+ worksheet = workbook.add_worksheet(worksheet_name)
+ with open(csvfile, 'rt', encoding='unicode_escape') as f:
+ reader = csv.reader(f)
+ last_row = 0
+ last_col = 0
+ has_data = False
+ for r, row in enumerate(reader):
+ has_data = True
+ last_row = r
+ for c, col in enumerate(row):
+ last_col = c
+ # Format URL only if the column starts with "=HYPERLINK"
+ if col.startswith("=HYPERLINK"):
+ url_info = re.findall(r'"(.*?)"', col)
+ if url_info and len(url_info[0]) < 2079: # Excel Link limit
+ worksheet.write_url(r, c, url_info[0], string=url_info[1])
+ else:
+ worksheet.write(r, c, col)
+ if has_data:
+ worksheet.autofilter(0, 0, last_row, last_col)
+ worksheet.autofit()
+ except Exception as e:
+ print(f"** Failed to output to Excel worksheet {worksheet_name} **")
+ print(e)
continue
- elif len(worksheet_name) > 31:
- worksheet_name = worksheet_name.replace("_", "")
-
- worksheet = workbook.add_worksheet(worksheet_name)
- with open(csvfile, 'rt', encoding='unicode_escape') as f:
- reader = csv.reader(f)
- for r, row in enumerate(reader):
- for c, col in enumerate(row):
- # Format URL only if the column starts with "=HYPERLINK"
- if col.startswith("=HYPERLINK"):
- url_info = re.findall(r'"(.*?)"', col)
- if url_info and len(url_info[0]) < 2079: # Excel Link limit
- worksheet.write_url(r, c, url_info[0], string=url_info[1])
- else:
- worksheet.write(r, c, col)
- worksheet.autofilter(0, 0, r - 1, c - 1)
- worksheet.autofit()
workbook.close()
except Exception as e: