APIC DB Size check (#266)

* APIC DB size check + pytests

* run_cmd function + pytests

---------

Co-authored-by: Gabriel <gmonroy@cisco.com>

Author: welkinhe

aci-preupgrade-validation-script.py

@@ -1260,6 +1260,28 @@ def icurl(apitype, query, page_size=100000):
     return total_imdata
 
 
+def run_cmd(cmd, splitlines=True):
+    """
+    Run a shell command.
+    :param cmd: Command to run, can be a string or a list.
+    :param splitlines: If True, splits the output into a list of lines. 
+                       If False, returns the raw text output as a single string.
+    Returns the output of the command.
+    """
+    if isinstance(cmd, list):
+        cmd = ' '.join(cmd)
+    try:
+        log.info('run_cmd = ' + cmd)
+        response = subprocess.check_output(cmd, shell=True).decode('utf-8')
+        log.debug('response: ' + str(response))
+        if splitlines:
+            return response.splitlines()
+        return response
+    except subprocess.CalledProcessError as e:
+        log.error("Command '%s' failed with error: %s", cmd, str(e))
+        raise e
+
+
 def get_credentials():
     prints('To use a non-default Login Domain, enter apic#DOMAIN\\\\USERNAME')
     while True:
@@ -3166,6 +3188,7 @@ def cimc_compatibilty_check(tversion, **kwargs):
     return Result(result=result, headers=headers, data=data, recommended_action=recommended_action, doc_url=doc_url)
 
 
+# Subprocess Check - icurl
 @check_wrapper(check_title="Intersight Device Connector upgrade status")
 def intersight_upgrade_status_check(**kwargs):
     result = FAIL_UF
@@ -3518,6 +3541,7 @@ def internal_vlanpool_check(tversion, **kwargs):
     return Result(result=result, headers=headers, data=data, recommended_action=recommended_action, doc_url=doc_url)
 
 
+# Subprocess check - openssl
 @check_wrapper(check_title="APIC CA Cert Validation")
 def apic_ca_cert_validation(**kwargs):
     result = FAIL_O
@@ -3549,16 +3573,12 @@ def apic_ca_cert_validation(**kwargs):
             '''
             # Re-run cleanup for Issue #120
             if os.path.exists(cert_gen_filename):
-                log.debug('CA CHECK file found and removed: ' + ''.join(cert_gen_filename))
                 os.remove(cert_gen_filename)
             if os.path.exists(key_pem):
-                log.debug('CA CHECK file found and removed: ' + ''.join(key_pem))
                 os.remove(key_pem)
             if os.path.exists(csr_pem):
-                log.debug('CA CHECK file found and removed: ' + ''.join(csr_pem))
                 os.remove(csr_pem)
             if os.path.exists(sign):
-                log.debug('CA CHECK file found and removed: ' + ''.join(sign))
                 os.remove(sign)
 
             with open(cert_gen_filename, 'w') as f:
@@ -5209,6 +5229,74 @@ def isis_database_byte_check(tversion, **kwargs):
     return Result(result=result, headers=headers, data=data, recommended_action=recommended_action, doc_url=doc_url)
 
 
+# Subprocess check - cat + acidiag
+@check_wrapper(check_title='APIC Database Size')
+def apic_database_size_check(cversion, **kwargs):
+    result = PASS
+    headers = ["APIC ID", "DME", "Class Name", "Object Count"]
+    data = []
+    recommended_action = 'Contact Cisco TAC to investigate all flagged high object counts'
+    doc_url = 'https://datacenter.github.io/ACI-Pre-Upgrade-Validation-Script/validations/#apic-database-size'
+
+    dme_svc_list = ['vmmmgr', 'policymgr', 'eventmgr', 'policydist']
+    unique_list = {}
+    apic_id_to_name = {}
+    apic_node_mo = icurl('class', 'infraWiNode.json')
+    for apic in apic_node_mo:
+        if apic['infraWiNode']['attributes']['operSt'] == 'available':
+            apic_id = apic['infraWiNode']['attributes']['id']
+            apic_name = apic['infraWiNode']['attributes']['nodeName']
+            if apic_id not in apic_id_to_name:
+                apic_id_to_name[apic_id] = apic_name
+
+    # For 3 APIC cluster, only check APIC Id 2 due to static local shards (R0)
+    if len(apic_id_to_name) == 3:
+        apic_id_to_name = {"2": apic_id_to_name["2"]}
+
+    if cversion.older_than("6.1(3a)"):
+        for dme in dme_svc_list:
+            for id in apic_id_to_name:
+                apic_hostname = apic_id_to_name[id]
+                collect_stats_cmd = 'cat /debug/'+apic_hostname+'/'+dme+'/mitmocounters/mo | grep -v ALL | sort -rn -k3'
+                top_class_stats = run_cmd(collect_stats_cmd, splitlines=True)
+
+                for svc_stats in top_class_stats[:4]:
+                    if ":" in svc_stats:
+                        class_name = svc_stats.split(":")[0].strip()
+                        mo_count = svc_stats.split(":")[1].strip()
+                        if int(mo_count) > 1000*1000*1.5:
+                            unique_list[class_name] = {"id": id, "dme": dme, "checked_val": mo_count}
+    else:
+        headers = ["APIC ID", "DME", "Shard", "Size"]
+        recommended_action = 'Contact Cisco TAC to investigate all flagged large DB sizes'
+        for id in apic_id_to_name:
+            collect_stats_cmd = "acidiag dbsize --topshard --apic " + id + " -f json"
+            try:
+                collect_shard_stats_data = run_cmd(collect_stats_cmd, splitlines=False)
+            except subprocess.CalledProcessError:
+                return Result(result=MANUAL, msg="acidiag command not available to current user")
+            top_db_stats = json.loads(collect_shard_stats_data)
+
+            for db_stats in top_db_stats['dbs']:
+                if int(db_stats['size_b']) >= 1073741824 * 5:
+                    apic_id = db_stats['apic']
+                    dme = db_stats['dme']
+                    shard = db_stats['shard_replica']
+                    size = db_stats['size_h']
+                    unique_list[shard] = {"id": id, "dme": dme, "checked_val": size}
+
+    # dedup based on unique_key
+    if unique_list:
+        for unique_key, details in unique_list.items():
+            apic_id = details['id']
+            dme = details['dme']
+            checked_val = details['checked_val']
+            data.append([apic_id, dme, unique_key, checked_val])
+
+    if data:
+        result = FAIL_UF
+    return Result(result=result, headers=headers, data=data, recommended_action=recommended_action, doc_url=doc_url)
+
 # ---- Script Execution ----
 
 def parse_args(args):
@@ -5345,7 +5433,6 @@ def get_checks(api_only, debug_function):
         telemetryStatsServerP_object_check,
         llfc_susceptibility_check,
         internal_vlanpool_check,
-        apic_ca_cert_validation,
         fabricdomain_name_check,
         sup_hwrev_check,
         sup_a_high_memory_check,
@@ -5370,13 +5457,15 @@ def get_checks(api_only, debug_function):
     conn_checks = [
         # General
         apic_version_md5_check,
+        apic_database_size_check,
 
         # Faults
         standby_apic_disk_space_check,
         apic_ssd_check,
 
         # Bugs
         observer_db_size_check,
+        apic_ca_cert_validation,
 
     ]
     if debug_function:

docs/docs/validations.md

@@ -35,6 +35,7 @@ Items                                                        | This Script
 [Post Upgrade CallBack Integrity][g15]                       | :white_check_mark: | :no_entry_sign:           | :no_entry_sign:
 [6.0(2)+ requires 32 and 64 bit switch images][g16]          | :white_check_mark: | :no_entry_sign:           | :no_entry_sign:
 [Fabric Link Redundancy][g17]                                | :white_check_mark: | :no_entry_sign:           | :no_entry_sign:
+[APIC Database Size][g18]                                    | :white_check_mark: | :no_entry_sign:           | :no_entry_sign:
 
 [g1]: #compatibility-target-aci-version
 [g2]: #compatibility-cimc-version
@@ -53,6 +54,7 @@ Items                                                        | This Script
 [g15]: #post-upgrade-callback-integrity
 [g16]: #602-requires-32-and-64-bit-switch-images
 [g17]: #fabric-link-redundancy
+[g18]: #apic-database-size
 
 ### Fault Checks
 Items                                         | Faults         | This Script       | APIC built-in                 | Pre-Upgrade Validator (App)
@@ -471,6 +473,24 @@ When upgrading the switches, traffic traversing a Leaf Switch that is connected
 
 To prevent this scenario, ensure that every leaf is connected to at least two Spine Switches (or tier-1 Leaf Switches). This check will alert if any Leaf Switches are found to only be connected to a single Spine Switch (or tier-1 Leaf Switch).
 
+### APIC Database Size
+
+APIC Database Shard sizing is generally expected to remain below 5G in steady-state conditions, even in the case of high scale setups. Database shard sizing directly influences the Upgrade Workflow timing, as the database conversion phase directly involves parsing through the entire DB contents. Large shards lead to longer upgrade timings, which in some cases have been seen to lead to upgrade failures. In most cases, a large shard size has been mapped to an underlying condition that needs to be addressed.
+
+The script performs 2 different checks depending on the version you are running.
+
+For current versions below 6.1(3):
+
+- The script checks all APICs' class's object count for a subset of services (DMEs) via a file scan. 
+- If the count is found to be above `150*1000*1000`, then that class will be flagged for further investigation.
+
+For current version is 6.1(3f):
+
+- 6.1(3f) introduces a new `acidiag dbsize` command which displays the top largest DB sizes. 
+- The script will utilize the new command and flag any DB shard which have surpassed 5G.
+
+In either scenario, contact TAC to collect a database dump of the flagged DME(s) and shard(s) for further analysis.
+
 
 ## Fault Check Details
 

tests/apic_database_size_check/infraWiNode_3.json

@@ -0,0 +1,97 @@
+{
+    "totalCount": "9",
+    "imdata": [
+      {
+        "infraWiNode": {
+          "attributes": {
+            "dn": "topology/pod-1/node-1/av/node-1",
+            "id": "1",
+            "nodeName": "apic1",
+            "operSt": "available"
+          }
+        }
+      },
+      {
+        "infraWiNode": {
+          "attributes": {
+
+            "dn": "topology/pod-1/node-1/av/node-2",
+            "id": "2",
+            "nodeName": "apic2",
+            "operSt": "available"
+          }
+        }
+      },
+      {
+        "infraWiNode": {
+          "attributes": {
+            "dn": "topology/pod-1/node-1/av/node-3",
+            "id": "3",
+            "nodeName": "apic3",
+            "operSt": "available"
+          }
+        }
+      },
+      {
+        "infraWiNode": {
+          "attributes": {
+            "dn": "topology/pod-1/node-2/av/node-1",
+            "id": "1",
+            "nodeName": "apic1",
+            "operSt": "available"
+          }
+        }
+      },
+      {
+        "infraWiNode": {
+          "attributes": {
+            "dn": "topology/pod-1/node-2/av/node-2",
+            "id": "2",
+            "nodeName": "apic2",
+            "operSt": "available"
+          }
+        }
+      },
+      {
+        "infraWiNode": {
+          "attributes": {
+            "dn": "topology/pod-1/node-2/av/node-3",
+            "id": "3",
+            "nodeName": "apic3",
+            "operSt": "available"
+
+          }
+        }
+      },
+      {
+        "infraWiNode": {
+          "attributes": {
+            "dn": "topology/pod-1/node-3/av/node-3",
+            "id": "3",
+            "nodeName": "apic3",
+            "operSt": "available"
+          }
+        }
+      },
+      {
+        "infraWiNode": {
+          "attributes": {
+            "dn": "topology/pod-1/node-3/av/node-1",
+            "id": "1",
+            "nodeName": "apic1",
+            "operSt": "available"
+          }
+        }
+      },
+      {
+        "infraWiNode": {
+          "attributes": {
+            "dn": "topology/pod-1/node-3/av/node-2",
+            "id": "2",
+            "nodeName": "apic2",
+            "operSt": "available"
+          }
+        }
+      }
+    ]
+  }