fix for issue NewValidation: CSCwp95515 - Policydist crash-loop on upgrade if configpushShardCont headTx is non-zero while tailTx is zero #253

aci-preupgrade-validation-script.py

@@ -5302,6 +5302,35 @@ def apic_database_size_check(cversion, **kwargs):
         result = FAIL_UF
     return Result(result=result, headers=headers, data=data, recommended_action=recommended_action, doc_url=doc_url)
 
+
+@check_wrapper(check_title='Policydist configpushShardCont crash')
+def configpush_shard_check(tversion, **kwargs):
+    result = NA
+    headers = ["dn", "headTx",  "tailTx"]
+    data = []
+    recommended_action = 'Contact Cisco TAC for Support before upgrade'
+    doc_url = 'https://datacenter.github.io/ACI-Pre-Upgrade-Validation-Script/validations/#policydist-configpushshardcont-crash'
+
+    if not tversion:
+        return Result(result=MANUAL, msg=TVER_MISSING) 
+
+    if tversion.older_than("6.1(4a)"):
+        result = PASS
+        configpushShardCont_api = 'configpushShardCont.json'
+        configpushShardCont_api += '?query-target-filter=and(eq(configpushShardCont.tailTx,"0"),ne(configpushShardCont.headTx,"0"))'
+        configpush_sh_cont = icurl('class', configpushShardCont_api)
+        if configpush_sh_cont:
+            for sh_cont in configpush_sh_cont:
+                headtx = sh_cont['configpushShardCont']['attributes']['headTx']
+                tailtx = sh_cont['configpushShardCont']['attributes']['tailTx']
+                sh_cont_dn = sh_cont['configpushShardCont']['attributes']['dn']
+                data.append([sh_cont_dn, headtx, tailtx])
+
+    if data:
+        result = FAIL_O
+
+    return Result(result=result, headers=headers, data=data, recommended_action=recommended_action, doc_url=doc_url)
+
 # ---- Script Execution ----
 
 
@@ -5461,6 +5490,7 @@ def get_checks(api_only, debug_function):
         pbr_high_scale_check,
         standby_sup_sync_check,
         isis_database_byte_check,
+        configpush_shard_check,
 
     ]
     conn_checks = [

docs/docs/validations.md

@@ -188,7 +188,7 @@ Items                                           | Defect       | This Script
 [Observer Database Size][d25]                   | CSCvw45531   | :white_check_mark: | :no_entry_sign:
 [Stale pconsRA Object][d26]                     | CSCwp22212   | :warning:{title="Deprecated"} | :no_entry_sign:
 [ISIS DTEPs Byte Size][d27]                     | CSCwp15375   | :white_check_mark: | :no_entry_sign:
-
+[Policydist configpushShardCont Crash][d28]     | CSCwp95515   | :white_check_mark: | 
 
 [d1]: #ep-announce-compatibility
 [d2]: #eventmgr-db-size-defect-susceptibility
@@ -217,6 +217,7 @@ Items                                           | Defect       | This Script
 [d25]: #observer-database-size
 [d26]: #stale-pconsra-object
 [d27]: #isis-dteps-byte-size
+[d28]: #policydist-configpushshardcont-crash
 
 
 ## General Check Details
@@ -2579,6 +2580,15 @@ Do not upgrade to any affected ACI software release if this check fails.
     Nexus Dashboard Insights (NDI) integration can cause ACI tech support generation to happen automatically as part of the bug scan feature.
 
 
+### Policydist configpushShardCont crash
+
+In ACI, there are internal objects which track the underlying transactions which occur as policies are handled by the Policydist process. One such object is `configpushShardCont` which populates the `headTx` and `tailTx` parameters to mark any potentially stuck transactions.
+
+Due to [CSCwp95515][59], upgrading to an affected version while having any `configpushShardCont` objects with a non-zero `headTx` and `tailTx: 0` can result in the Policydist process crashing if config is pushed to a PM shard matching the `dn` of the identified `configpushShardCont`.
+
+If any instances of `configpushShardCont` are flagged by this script, Cisco TAC must be contacted to identify and resolve the underlying issue before performing the upgrade.
+
+
 [0]: https://github.com/datacenter/ACI-Pre-Upgrade-Validation-Script
 [1]: https://www.cisco.com/c/dam/en/us/td/docs/Website/datacenter/apicmatrix/index.html
 [2]: https://www.cisco.com/c/en/us/support/switches/nexus-9000-series-switches/products-release-notes-list.html
@@ -2638,3 +2648,4 @@ Do not upgrade to any affected ACI software release if this check fails.
 [56]: https://www.cisco.com/c/en/us/td/docs/dcn/whitepapers/cisco-aci-virtual-edge-migration.html
 [57]: https://bst.cloudapps.cisco.com/bugsearch/bug/CSCwp22212
 [58]: https://bst.cloudapps.cisco.com/bugsearch/bug/CSCwp15375
+[59]: https://bst.cloudapps.cisco.com/bugsearch/bug/CSCwp95515

tests/configpush_shard_check/configpushShardCont_pos.json

@@ -0,0 +1,30 @@
+[
+    {
+        "configpushShardCont": {
+            "attributes": {
+                "childAction": "",
+                "dn": "configpush/sh-2",
+                "headTx": "576460752318322171",
+                "lcOwn": "local",
+                "modTs": "2025-06-28T21:32:58.754+00:00",
+                "shard": "2",
+                "status": "",
+                "tailTx": "0"
+            }
+        }
+    },
+    {
+        "configpushShardCont": {
+            "attributes": {
+                "childAction": "",
+                "dn": "configpush/sh-3",
+                "headTx": "516460756318314171",
+                "lcOwn": "local",
+                "modTs": "2025-06-23T18:12:06.867+00:00",
+                "shard": "3",
+                "status": "",
+                "tailTx": "0"
+            }
+        }
+    }
+]

tests/configpush_shard_check/test_configpush_shard_check.py

@@ -0,0 +1,56 @@
+import os
+import pytest
+import logging
+import importlib
+from helpers.utils import read_data
+
+script = importlib.import_module("aci-preupgrade-validation-script")
+
+log = logging.getLogger(__name__)
+dir = os.path.dirname(os.path.abspath(__file__))
+
+# icurl queries
+configpushShardCont_api = 'configpushShardCont.json'
+configpushShardCont_api += '?query-target-filter=and(eq(configpushShardCont.tailTx,"0"),ne(configpushShardCont.headTx,"0"))'
+
+@pytest.mark.parametrize(
+    "icurl_outputs, tversion, expected_result",
+    [
+        # tversion not given 
+        (
+            {configpushShardCont_api: []},
+            None,
+            script.MANUAL,
+        ),
+        # Non-fixed Versions
+        (
+            # affected tversion, configpushShardCont_api has non-zero headTx / tailTx
+            {configpushShardCont_api: read_data(dir, "configpushShardCont_pos.json")},
+            "6.0(3a)",
+            script.FAIL_O,
+        ),
+        (
+            # affected tversion, all configpushShardCont_api tx are 0
+            {configpushShardCont_api: []},
+            "5.2(6a)",
+            script.PASS,
+        ),
+        # Fixed Versions
+        (
+            # non-affected tversion, configpushShardCont_api has non-zero headTx / tailTx
+            {configpushShardCont_api: read_data(dir, "configpushShardCont_pos.json")},
+            "6.1(6b)",
+            script.NA,
+        ),
+        (
+            # non-affected tversion, all configpushShardCont_api tx are 0
+            {configpushShardCont_api: []},
+            "6.1(4b)",
+            script.NA,
+        ),
+    ],
+)
+def test_logic(mock_icurl, tversion, expected_result):
+    tversion = script.AciVersion(tversion) if tversion else None
+    result = script.configpush_shard_check(1, 1, tversion)
+    assert result == expected_result