large-db-test-adjust

Author: welkinhe

aci-preupgrade-validation-script.py

@@ -5205,6 +5205,74 @@ def isis_database_byte_check(tversion, **kwargs):
         return Result(result=NA, msg=VER_NOT_AFFECTED)
     return Result(result=result, headers=headers, data=data, recommended_action=recommended_action, doc_url=doc_url)
 
+@check_wrapper(check_title='Large APIC Database')
+def large_apic_database_check(cversion, **kwargs):
+    title = 'Large APIC Database'
+    result = PASS
+    msg = ''
+    headers = ["APIC Id", "DME", "Class", "Counters"]
+
+    data = []
+    recommended_action = 'Contact Cisco TAC to investigate the large than usual DB size'
+    doc_url = 'https://datacenter.github.io/ACI-Pre-Upgrade-Validation-Script/validations/#large-apic-database'
+
+    top_class_stats = kwargs.get("top_class_stats")
+    top_db_stats = kwargs.get("top_db_stats")
+
+    dme_svc_list = ['vmmmgr', 'policymgr', 'eventmgr', 'policydist']
+    apic_svr_dict = {}
+    apic_node_api = 'infraWiNode.json'
+    apic_node_mo = icurl('class', apic_node_api)
+    for apic in apic_node_mo:
+        if apic['infraWiNode']['attributes']['operSt'] == 'available':
+            apic_id = apic['infraWiNode']['attributes']['id']
+            apic_name = apic['infraWiNode']['attributes']['nodeName']
+            if apic_id not in apic_svr_dict:
+                apic_svr_dict[apic_id] = apic_name
+
+    if cversion.older_than("6.1(3a)"):
+        for dme in dme_svc_list:
+            for id in apic_svr_dict:
+                if len(apic_svr_dict) == 3 and int(id) != 2:
+                    continue
+                apic_hostname = apic_svr_dict[id]
+
+                if not top_class_stats:
+                    collect_stats_cmd = 'cat /debug/'+apic_hostname+'/'+dme+'/mitmocounters/mo | grep -v ALL | sort -rn -k3 | head -3'
+                    collect_shard_stats = subprocess.Popen(collect_stats_cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, shell=True)
+                    top_class_stats = collect_shard_stats.communicate()[0].strip().decode("utf-8")
+
+                for svc_stats in top_class_stats.splitlines():
+                    logging.debug("APIC Id: " + id + " + DME name: " + dme + " + MoCounter " + str(svc_stats))
+                    if ":" in svc_stats:
+                        class_name = svc_stats.split(":")[0].strip()
+                        mo_count = svc_stats.split(":")[1].strip()
+                        if int(mo_count) > 1000*1000*1.5:
+                            data.append([id, dme, class_name, mo_count])
+    else:
+        headers = ["APIC Id", "DME", "Shard ", "Size"]
+        for id in apic_svr_dict:
+            if len(apic_svr_dict) == 3 and int(id) != 2:
+                continue
+            collect_stats_cmd = "acidiag dbsize --topshard --apic " + id + " -f json"
+            logging.debug(collect_stats_cmd)
+
+            if not top_db_stats:
+                collect_shard_stats = subprocess.Popen(collect_stats_cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, shell=True)
+                collect_shard_stats_data = collect_shard_stats.communicate()[0].strip()
+                top_db_stats = json.loads(collect_shard_stats_data)
+
+            for db_stats in top_db_stats['dbs']:
+                logging.debug(db_stats)
+                if int(db_stats['size_b'])>=1073741824 * 5:
+                    apic_id = db_stats['apic']
+                    dme = db_stats['dme']
+                    shard = db_stats['shard_replica']
+                    size = db_stats['size_h']
+                    data.append([id, dme, shard, size])
+    if data:
+        result = FAIL_O
+    return Result(result=result, headers=headers, data=data, recommended_action=recommended_action, doc_url=doc_url)
 
 # ---- Script Execution ----
 
@@ -5289,6 +5357,7 @@ def get_checks(api_only, debug_function):
         post_upgrade_cb_check,
         validate_32_64_bit_image_check,
         fabric_link_redundancy_check,
+        large_apic_database_check,
 
         # Faults
         apic_disk_space_faults_check,

docs/docs/validations.md

@@ -35,6 +35,7 @@ Items                                                        | This Script
 [Post Upgrade CallBack Integrity][g15]                       | :white_check_mark: | :no_entry_sign:           | :no_entry_sign:
 [6.0(2)+ requires 32 and 64 bit switch images][g16]          | :white_check_mark: | :no_entry_sign:           | :no_entry_sign:
 [Fabric Link Redundancy][g17]                                | :white_check_mark: | :no_entry_sign:           | :no_entry_sign:
+[Large APIC Database][g18]                                   | :white_check_mark: | :no_entry_sign:           | :no_entry_sign:
 
 [g1]: #compatibility-target-aci-version
 [g2]: #compatibility-cimc-version
@@ -53,6 +54,7 @@ Items                                                        | This Script
 [g15]: #post-upgrade-callback-integrity
 [g16]: #602-requires-32-and-64-bit-switch-images
 [g17]: #fabric-link-redundancy
+[g18]: #large-apic-database
 
 ### Fault Checks
 Items                                         | Faults         | This Script       | APIC built-in                 | Pre-Upgrade Validator (App)
@@ -471,6 +473,18 @@ When upgrading the switches, traffic traversing a Leaf Switch that is connected
 
 To prevent this scenario, ensure that every leaf is connected to at least two Spine Switches (or tier-1 Leaf Switches). This check will alert if any Leaf Switches are found to only be connected to a single Spine Switch (or tier-1 Leaf Switch).
 
+### Large APIC Database
+
+It is generally expected that individual APIC DB Shard sizes remain below 5G in steady state, even in high scale configurations. If a single shard goes above this size, it can lead to an elongated upgrade timing, or even an upgrade failure, and in most cases is indicative of an underlying condition.
+
+The script performs 2 different checks depending on the version you are running.
+
+If current version is below 6.1(3), the script checks all APICs' class's object count. If the count is found to be above 150*1000*1000 then it will flag the DME for further investigation.
+
+If the current version is 6.1(3f) or above, the script will utilize the newly added `acidiag dbsize` command and check if any of the top large DB files have exceeded 5G.
+
+When alerted to this specific warning, Contact TAC to collect a database dump of the DME and shards that are in question for further analysis.
+
 
 ## Fault Check Details