Welkin issue 245 (#263)

welkinhe · monrog2 · jeestr4d · commit 1098b6c8ebbf · 2025-07-22T14:36:38.000-06:00
* Introduced CSCwp22212 check for stale pconsRA objects

---------

Co-authored-by: Gabriel &lt;gmonroy@cisco.com&gt;
diff --git a/aci-preupgrade-validation-script.py b/aci-preupgrade-validation-script.py
@@ -5101,33 +5101,62 @@ def ave_eol_check(index, total_checks, tversion, **kwargs):
     return result
 
 
-def configpush_shard_check(index, total_checks, tversion, **kwargs):
-    title = 'configpushShardCont headTx'
-    result = NA
+def stale_pcons_ra_mo_check(index, total_checks, cversion, tversion, **kwargs):
+    title = 'Stale pconsRA Objects'
+    result = PASS
     msg = ''
-    headers = ["dn", "headTx",  "tailTx"]
+    headers = ["Stale pconsRA DN", "Non-Existing DN"]
+
     data = []
-    recommended_action = 'Contact Cisco TAC for Support before upgrade'
-    doc_url = 'https://datacenter.github.io/ACI-Pre-Upgrade-Validation-Script/validations/#policydist-configpushshardcont-defect'
+    recommended_action = 'Contact Cisco TAC to delete stale pconsRA before upgrading'
+    doc_url = 'https://datacenter.github.io/ACI-Pre-Upgrade-Validation-Script/validations/#stale-pconsra-object'
     print_title(title, index, total_checks)
-    configpushShardCont = 'configpushShardCont.json'
-    if tversion.older_than("6.1(4a)"):
-        configpush_sh_cont = icurl('class', configpushShardCont)
-        if configpush_sh_cont:
-            result = PASS
-            for sh_cont in configpush_sh_cont:
-                if sh_cont['configpushShardCont']['attributes']['headTx'] != '0' and sh_cont['configpushShardCont']['attributes']['tailTx'] == '0':
-                    sh_cont_dn = sh_cont['configpushShardCont']['attributes']['dn']
-                    headtx = sh_cont['configpushShardCont']['attributes']['headTx']
-                    tailtx = sh_cont['configpushShardCont']['attributes']['tailTx']
-                    data.append([sh_cont_dn, headtx, tailtx])
-    
+
+    if not tversion:
+        print_result(title, MANUAL, "Target version not supplied. Skipping.")
+        return MANUAL
+
+    if cversion.older_than("6.0(3d)") and tversion.newer_than("6.0(3c)") and tversion.older_than("6.1(4a)"):
+        pcons_rssubtreedep_api = 'pconsRsSubtreeDep.json?query-target-filter=wcard(pconsRsSubtreeDep.tDn,"/instdn-")'
+        pcons_rssubtreedep_mo = icurl('class', pcons_rssubtreedep_api)
+        pcons_inst_dn_reg = r'registry/class-\d+/instdn-\[(?P<policy_dn>.+?)\]/ra'
+        pcons_ra_dn_reg = r'(?P<pcons_ra_dn>.+?)/p...-\['
+
+        pcons_ra_set = set()
+        policy_dn_set = set()
+
+        for mo in pcons_rssubtreedep_mo:
+            pcons_rssubtreedep_tdn = mo['pconsRsSubtreeDep']['attributes']['tDn']
+            instdn_found = re.search(pcons_inst_dn_reg, pcons_rssubtreedep_tdn)
+            radn_found = re.search(pcons_ra_dn_reg, pcons_rssubtreedep_tdn)
+            if instdn_found and radn_found:
+                pcons_ra_dn = radn_found.group('pcons_ra_dn')
+                policy_dn = instdn_found.group('policy_dn')
+                if pcons_ra_dn not in pcons_ra_set:
+                    pcons_ra_set.add(pcons_ra_dn)
+                if policy_dn not in policy_dn_set:
+                    policy_dn_set.add(policy_dn)
+
+        for policy_dn in policy_dn_set:
+            policy_dn_api = policy_dn + '.json'
+            policy_dn_mo = icurl('mo', policy_dn_api)
+            if not policy_dn_mo:
+                for pcons_ra_dn in pcons_ra_set:
+                    if policy_dn in pcons_ra_dn:
+                        pcons_ra_api = pcons_ra_dn + '.json'
+                        pcons_ra_dn_mo = icurl('mo', pcons_ra_api)
+                        if pcons_ra_dn_mo:
+                            data.append([pcons_ra_dn, policy_dn])
+    else:
+        print_result(title, NA, "Target version not supplied or not applicable. Skipping.")
+        return NA
+
     if data:
         result = FAIL_O
-
-    print_result(title, result, msg, headers, data, recommended_action=recommended_action, doc_url=doc_url)    
+    print_result(title, result, msg, headers, data, recommended_action=recommended_action, doc_url=doc_url)
     return result
 
+
 if __name__ == "__main__":
     prints('    ==== %s%s, Script Version %s  ====\n' % (ts, tz, SCRIPT_VERSION))
     prints('!!!! Check https://github.com/datacenter/ACI-Pre-Upgrade-Validation-Script for Latest Release !!!!\n')
@@ -5243,7 +5272,7 @@ def configpush_shard_check(index, total_checks, tversion, **kwargs):
         pbr_high_scale_check,
         standby_sup_sync_check,
         observer_db_size_check,
-        configpush_shard_check,
+        stale_pcons_ra_mo_check,
 
     ]
     summary = {PASS: 0, FAIL_O: 0, FAIL_UF: 0, ERROR: 0, MANUAL: 0, POST: 0, NA: 0, 'TOTAL': len(checks)}
diff --git a/docs/docs/validations.md b/docs/docs/validations.md
@@ -184,8 +184,7 @@ Items                                           | Defect       | This Script
 [PBR High Scale][d23]                           | CSCwi66348   | :white_check_mark: | :no_entry_sign:           |:no_entry_sign:
 [Standby Sup Image Sync][d24]                   | CSCwi66348   | :white_check_mark: | :no_entry_sign:           |:no_entry_sign:
 [Observer Database Size][d25]                   | CSCvw45531   | :white_check_mark: | :no_entry_sign:           |:no_entry_sign:
-[Policydist configpushShardCont defect][d26]    | CSCwp95515   | :white_check_mark: | :no_entry_sign:           |:no_entry_sign:
-
+[Stale pconsRA Object][d26]                     | CSCwp22212   | :white_check_mark: | :no_entry_sign:           |:no_entry_sign:
 
 [d1]: #ep-announce-compatibility
 [d2]: #eventmgr-db-size-defect-susceptibility
@@ -212,7 +211,7 @@ Items                                           | Defect       | This Script
 [d23]: #pbr-high-scale
 [d24]: #standby-sup-image-sync
 [d25]: #observer-database-size
-[d26]: #policydist-configpushshardcont-defect
+[d26]: #stale-pconsra-object
 
 
 ## General Check Details
@@ -2543,13 +2542,11 @@ This check logs in to each APIC, checks the contents of the `/data2/dbstats/` di
 !!! tip
     Certain high churn logging configurations have been found to grow this DB exceptionally large while on a non-fixed version. 'Contract Permit Logging' is one such configuration.
 
-### Policydist configpushShardCont defect
-
-Due to [CSCwp95515][57], a configpushShardCont MO in policydist has a non-zero headTx while tailTx is zero. if APIC cluster is upgraded or if config is pushed to a PM shard corresponding to the DN that has the bad properties policydist can crash.
+### Stale pconsRA Object
 
-The Policydist component is responsible for policy enforcement and replicating policy actions and maintaining consistency of policy state across all APICs in the cluster.
+Due to [CSCwp22212][57], the existence of stale pconsRA objects within an ACI fabric can cause the APIC Policymanager process to crash after an upgrade to 6.0(3d) and above. This script looks for instances of stale pconsRA objects and flags them for cleanup when found. 
 
-The script scans for any instance of configpushShardCont that can lead to this defect, contact Cisco TAC to resolve the issue prior to the upgrade.
+TAC must be engaged to cleanup these objects, as they require root access.
 
 
 [0]: https://github.com/datacenter/ACI-Pre-Upgrade-Validation-Script
@@ -2609,4 +2606,4 @@ The script scans for any instance of configpushShardCont that can lead to this d
 [54]: https://bst.cloudapps.cisco.com/bugsearch/bug/CSCvt47850
 [55]: https://www.cisco.com/c/en/us/products/collateral/cloud-systems-management/application-policy-infrastructure-controller-apic/eol-apic-virtual-edge-pod-pb.html
 [56]: https://www.cisco.com/c/en/us/td/docs/dcn/whitepapers/cisco-aci-virtual-edge-migration.html
-[57]: https://bst.cloudapps.cisco.com/bugsearch/bug/CSCwp95515
+[57]: https://bst.cloudapps.cisco.com/bugsearch/bug/CSCwp22212
diff --git a/tests/stale_pcons_ra_mo_check/pconsRA.json b/tests/stale_pcons_ra_mo_check/pconsRA.json
@@ -0,0 +1,33 @@
+{
+    "totalCount": "1",
+    "imdata": [
+        {
+            "pconsRA": {
+                "attributes": {
+                    "applyFilter": "yes",
+                    "childAction": "",
+                    "ctx": "uni/infra/attentp-PHY-DOM_AttEntityP",
+                    "deployStatus": "deployed",
+                    "distAll": "no",
+                    "dn": "registry/class-1809/instdn-[uni/phys-PHY-DOM]/ra-[uni/infra/attentp-PHY-DOM_AttEntityP]-6-0-0-0-SubtreeWithRels-mo",
+                    "incrDeploy": "no",
+                    "lcOwn": "local",
+                    "markedForDelete": "no",
+                    "modOnly": "no",
+                    "modTs": "2025-01-23T11:42:00.631+10:00",
+                    "nonDeployable": "no",
+                    "onBehalfOf": "no",
+                    "pendingRelease": "no",
+                    "resolverType": "mo",
+                    "rn": "ra-[uni/infra/attentp-PHY-DOM_AttEntityP]-6-0-0-0-SubtreeWithRels-mo",
+                    "slotId": "0",
+                    "status": "",
+                    "svc": "6",
+                    "systemId": "0",
+                    "systemType": "0",
+                    "updateType": "SubtreeWithRels"
+                }
+            }
+        }
+    ]
+}
diff --git a/tests/stale_pcons_ra_mo_check/pconsRsSubtreeDep.json b/tests/stale_pcons_ra_mo_check/pconsRsSubtreeDep.json
@@ -0,0 +1,24 @@
+{
+    "totalCount": "3",
+    "imdata": [
+      {
+        "pconsRsSubtreeDep": {
+          "attributes": {
+            "childAction": "",
+            "dn": "depRegistry/sdep-physDomP/rssubtreeDep-[registry/class-1809/instdn-[uni/phys-PHY-DOM]/ra-[uni/infra/attentp-PHY-DOM_AttEntityP]-6-0-0-0-SubtreeWithRels-mo/pown-[uni/phys-PHY-DOM]]",
+            "forceResolve": "yes",
+            "lcOwn": "local",
+            "modTs": "2025-01-23T11:41:55.730+10:00",
+            "rType": "mo",
+            "rn": "rssubtreeDep-[registry/class-1809/instdn-[uni/phys-PHY-DOM]/ra-[uni/infra/attentp-PHY-DOM_AttEntityP]-6-0-0-0-SubtreeWithRels-mo/pown-[uni/phys-PHY-DOM]]",
+            "state": "unformed",
+            "stateQual": "none",
+            "status": "",
+            "tCl": "pconsAPolDep",
+            "tDn": "registry/class-1809/instdn-[uni/phys-PHY-DOM]/ra-[uni/infra/attentp-PHY-DOM_AttEntityP]-6-0-0-0-SubtreeWithRels-mo/pown-[uni/phys-PHY-DOM]",
+            "tType": "mo"
+          }
+        }
+      }    
+    ]
+  }  
diff --git a/tests/stale_pcons_ra_mo_check/policyDn.json b/tests/stale_pcons_ra_mo_check/policyDn.json
@@ -0,0 +1,28 @@
+{
+    "totalCount": "1",
+    "imdata": [
+      {
+        "physDomP": {
+          "attributes": {
+            "annotation": "",
+            "childAction": "",
+            "configIssues": "",
+            "dn": "uni/phys-PHY-DOM",
+            "extMngdBy": "",
+            "lcOwn": "local",
+            "modTs": "2025-01-23T11:41:59.855+10:00",
+            "monPolDn": "uni/fabric/monfab-default",
+            "name": "PHY-DOM",
+            "nameAlias": "",
+            "ownerKey": "",
+            "ownerTag": "",
+            "rn": "phys-PHY-DOM",
+            "status": "",
+            "uid": "15374",
+            "userdom": "all"
+          }
+        }
+      }
+    ]
+  }
+  
diff --git a/tests/stale_pcons_ra_mo_check/policyDn_empty.json b/tests/stale_pcons_ra_mo_check/policyDn_empty.json
@@ -0,0 +1,5 @@
+{
+    "totalCount": "0",
+    "imdata": [
+    ]
+}
diff --git a/tests/stale_pcons_ra_mo_check/test_stale_pcons_ra_mo_check.py b/tests/stale_pcons_ra_mo_check/test_stale_pcons_ra_mo_check.py
@@ -0,0 +1,103 @@
+import os
+import pytest
+import logging
+import importlib
+import json
+from helpers.utils import read_data
+
+script = importlib.import_module("aci-preupgrade-validation-script")
+
+log = logging.getLogger(__name__)
+dir = os.path.dirname(os.path.abspath(__file__))
+pcons_rs_subtree_dep_api = 'pconsRsSubtreeDep.json?query-target-filter=wcard(pconsRsSubtreeDep.tDn,"/instdn-")'
+pcons_ra_api = 'registry/class-1809/instdn-[uni/phys-PHY-DOM]/ra-[uni/infra/attentp-PHY-DOM_AttEntityP]-6-0-0-0-SubtreeWithRels-mo.json'
+policy_dn_api = 'uni/phys-PHY-DOM.json'
+
+@pytest.mark.parametrize(
+    "icurl_outputs, cversion, tversion, expected_result",
+    [
+        # MANUAL when tversion not given
+        (
+            {
+                pcons_rs_subtree_dep_api: read_data(dir, 'pconsRsSubtreeDep.json'),
+                pcons_ra_api: read_data(dir, 'pconsRA.json'),
+                policy_dn_api: read_data(dir, 'policyDn.json'),
+            },
+            "5.3(2a)",
+            "",
+            script.MANUAL,
+        ),
+        # NA when tversion is older than 6.0(3d)
+        (
+            {
+                pcons_rs_subtree_dep_api: read_data(dir, 'pconsRsSubtreeDep.json'),
+                pcons_ra_api: read_data(dir, 'pconsRA.json'),
+                policy_dn_api: read_data(dir, 'policyDn.json'),
+            },
+            "5.3(2a)",
+            "6.0(3c)",
+            script.NA,
+        ),
+        # NA when tversion is older than 6.0(3d)
+        (
+            {
+                pcons_rs_subtree_dep_api: read_data(dir, 'pconsRsSubtreeDep.json'),
+                pcons_ra_api: read_data(dir, 'pconsRA.json'),
+                policy_dn_api: read_data(dir, 'policyDn.json'),
+            },
+            "6.1(2a)",
+            "6.1(3c)",
+            script.NA,
+        ),
+        # pass when tversion is 6.0(3d)+ and policy_dn is found.
+        (
+            {
+                pcons_rs_subtree_dep_api: read_data(dir, 'pconsRsSubtreeDep.json'),
+                pcons_ra_api: read_data(dir, 'pconsRA.json'),
+                policy_dn_api: read_data(dir, 'policyDn.json')
+            },
+            "5.3(2a)",
+            "6.0(3d)",
+            script.PASS,
+        ),
+        # pass when tversion is newer than 6.1(4a).
+        (
+            {
+                pcons_rs_subtree_dep_api: read_data(dir, 'pconsRsSubtreeDep.json'),
+                pcons_ra_api: read_data(dir, 'pconsRA.json'),
+                policy_dn_api: read_data(dir, 'policyDn.json')
+            },
+            "5.3(2a)",
+            "6.1(4a)",
+            script.NA,
+        ),
+        # FAIL_O when version tversion is older than 6.0(3d) and policy_dn is NOT found (pcons is stale).
+        (
+            {
+                pcons_rs_subtree_dep_api: read_data(dir, 'pconsRsSubtreeDep.json'),
+                pcons_ra_api: read_data(dir, 'pconsRA.json'),
+                policy_dn_api: read_data(dir, 'policyDn_empty.json'),
+            },
+            "5.3(2a)",
+            "6.0(3d)",
+            script.FAIL_O,
+        ),
+        # FAIL_O when version tversion is older than 6.0(3d) and policy_dn is NOT found (pcons is stale).
+        (
+            {
+                pcons_rs_subtree_dep_api: read_data(dir, 'pconsRsSubtreeDep.json'),
+                pcons_ra_api: read_data(dir, 'pconsRA.json'),
+                policy_dn_api: read_data(dir, 'policyDn_empty.json'),
+            },
+            "5.3(2a)",
+            "6.1(3d)",
+            script.FAIL_O,
+        ),
+    ],
+)
+def test_logic(mock_icurl, cversion, tversion, expected_result):
+    cver = script.AciVersion(cversion) if cversion else None
+    tver = script.AciVersion(tversion) if tversion else None
+
+    result = script.stale_pcons_ra_mo_check(1, 1, cver, tver,)
+    assert result == expected_result

-Original file line number
+Diff line change
@@ @@ -0,0 +1,5 @@ @@
 +{
 +    "totalCount": "0",
 +    "imdata": [
 +    ]
 +}