Added validation for multipod_modular_spine_bootscript_check - CSCwr66848

aci-preupgrade-validation-script.py

@@ -6053,6 +6053,73 @@ def auto_firmware_update_on_switch_check(cversion, tversion, **kwargs):
 
     return Result(result=result, headers=headers, data=data, recommended_action=recommended_action, doc_url=doc_url)
 
+
+@check_wrapper(check_title="Multi-Pod modular spine bootscript check")
+def multipod_modular_spine_bootscript_check(tversion, fabric_nodes, username, password, **kwargs):
+    result = PASS
+    headers = ["Pod ID", "Node ID", "Node Name", "Model", "Bootscript Present", "Bootstrap file Present"]
+    data = []
+    recommended_action = "Bootscript is missing, delete bootstrap.xml from /bootflash folder and do clean reboot"
+    doc_url = "https://datacenter.github.io/ACI-Pre-Upgrade-Validation-Script/validations/#multipod-modular-spine-bootscript-check"
+
+    pod_count_resp = icurl('class', 'fabricSetupP.json?query-target=self&rsp-subtree-include=count')
+    if (int(pod_count_resp[0]['moCount']['attributes']['count'])) < 2:
+        return Result(result=NA, msg="Multi-Pod is not enabled in fabric.")
+
+    if not (tversion.same_as("6.1(4h)") or tversion.same_as("6.1(5e)")):
+        return Result(result=NA, msg="Target version is not affected")
+
+    modular_spine_models = {"N9K-C9408", "N9K-C9504", "N9K-C9508", "N9K-C9516"}
+    found_modular_spine = any(
+        node["fabricNode"]["attributes"].get("role") == "spine" and
+        node["fabricNode"]["attributes"].get("model") in modular_spine_models
+        for node in fabric_nodes
+    )
+    if not found_modular_spine:
+        return Result(result=NA, msg="No modular spine (N9K-C9408, N9K-C9504, N9K-C9508, N9K-C9516) found in fabric.")
+
+    has_error = False
+    for node in fabric_nodes:
+        attr = node["fabricNode"]["attributes"]
+        if attr.get("role") != "spine":
+            continue
+
+        node_id = attr.get("id")
+        node_name = attr.get("name")
+        dn = re.search(node_regex, attr.get("dn", ""))
+        pod_id = dn.group("pod") if dn else "Unknown"
+        mgmt_ip = attr.get("address")
+        model = attr.get("model")
+
+        c = Connection(mgmt_ip)
+        c.username = username
+        c.password = password
+        c.log = LOG_FILE
+        try:
+            c.connect()
+            c.cmd("ls -l bootflash/ | grep boots")
+            bootscript_present = "Yes" if "bootscript" in c.output else "No"
+            bootstrap_present = "Yes" if "bootstrap.xml" in c.output else "No"
+        except Exception as e:
+            ssh_error = f"SSH ERROR: {e}"
+            data.append([pod_id, node_id, node_name, model, ssh_error, ssh_error])
+            has_error = True
+            continue
+        data.append([pod_id, node_id, node_name, model, bootscript_present, bootstrap_present])
+
+    if has_error:
+        result = ERROR
+    elif data:
+        bootscript_missing = any(row[4] == "No" for row in data)
+        bootstrap_missing  = any(row[5] == "No" for row in data)
+        if bootscript_missing and not bootstrap_missing:
+            result = FAIL_O
+        elif bootscript_missing and bootstrap_missing:
+            result = FAIL_UF
+            recommended_action = "bootscript and bootstrap.xml files are not found, Move to Fix version."
+
+    return Result(result=result, headers=headers, data=data, recommended_action=recommended_action, doc_url=doc_url)
+
 # ---- Script Execution ----
 
 
@@ -6216,6 +6283,7 @@ class CheckManager:
         isis_database_byte_check,
         configpush_shard_check,
         auto_firmware_update_on_switch_check,
+        multipod_modular_spine_bootscript_check,
 
     ]
     ssh_checks = [

docs/docs/validations.md

@@ -194,6 +194,7 @@ Items                                           | Defect       | This Script
 [ISIS DTEPs Byte Size][d27]                     | CSCwp15375   | :white_check_mark: | :no_entry_sign:
 [Policydist configpushShardCont Crash][d28]     | CSCwp95515   | :white_check_mark: | :no_entry_sign:
 [Auto Firmware Update on Switch Discovery][d29] | CSCwe83941   | :white_check_mark: | :no_entry_sign:
+[Multi-Pod modular spine bootscript check][d30]       | CSCwr66848   | :white_check_mark: | :no_entry_sign:
 
 [d1]: #ep-announce-compatibility
 [d2]: #eventmgr-db-size-defect-susceptibility
@@ -224,6 +225,7 @@ Items                                           | Defect       | This Script
 [d27]: #isis-dteps-byte-size
 [d28]: #policydist-configpushshardcont-crash
 [d29]: #auto-firmware-update-on-switch-discovery
+[d30]: #multipod-modular-spine-bootscript-check
 
 ## General Check Details
 
@@ -2667,6 +2669,24 @@ To avoid this risk, consider disabling Auto Firmware Update before upgrading to
 !!! note
     This issue occurs because older switch firmware versions are not compatible with switch images 6.0(3) or newer. The APIC version is not a factor.
 
+### Multi-Pod Modular Spine Bootscript
+
+Due to [CSCwr66848][64], in a Multi-Pod fabric, modular spine switches have `bootscript` file present in their bootflash from the initial bootstrap process. When upgrading to 6.1(4h) or 6.1(5e), if `bootscript` file is missing that can cause traffic loss across pods until the spine in this condition is clean reloaded.
+
+To avoid this issue, verify that `bootscript` file exists in the bootflash of each spine switch prior to upgrading to 6.1(4h) or 6.1(5e). If not found, the `bootstrap.xml` file must be removed before proceeding with the upgrade.
+
+!!! tip
+    You can manually check for the presence of `bootscript` on a spine switch by logging into the switch CLI and running the following command:
+    ```
+    spine1# ls -l bootflash/ | grep boots
+    -rw-rw-rw- 1 root  admin   152 Jan  5 11:51 bootscript
+    -rw-r--r-- 1   600 admin 14119 Jan  5 11:51 bootstrap.xml
+    ```
+    If `bootscript` is not present, delete the `bootstrap.xml` file from bootflash to resolve the issue:
+    ```
+    spine1# delete bootflash/bootstrap.xml
+    ```
+
 
 [0]: https://github.com/datacenter/ACI-Pre-Upgrade-Validation-Script
 [1]: https://www.cisco.com/c/dam/en/us/td/docs/Website/datacenter/apicmatrix/index.html
@@ -2731,4 +2751,5 @@ To avoid this risk, consider disabling Auto Firmware Update before upgrading to
 [60]: https://www.cisco.com/c/en/us/solutions/collateral/data-center-virtualization/application-centric-infrastructure/white-paper-c11-743951.html#Inter
 [61]: https://www.cisco.com/c/en/us/solutions/collateral/data-center-virtualization/application-centric-infrastructure/white-paper-c11-743951.html#EnablePolicyCompression
 [62]: https://bst.cloudapps.cisco.com/bugsearch/bug/CSCwe83941
-[63]: https://www.cisco.com/c/en/us/td/docs/dcn/aci/apic/all/apic-installation-aci-upgrade-downgrade/Cisco-APIC-Installation-ACI-Upgrade-Downgrade-Guide/m-auto-firmware-update.html
+[63]: https://www.cisco.com/c/en/us/td/docs/dcn/aci/apic/all/apic-installation-aci-upgrade-downgrade/Cisco-APIC-Installation-ACI-Upgrade-Downgrade-Guide/m-auto-firmware-update.html
+[64]: https://bst.cloudapps.cisco.com/bugsearch/bug/CSCwr66848

tests/checks/modular_spine_bootscript_check/fabricNode_no_spine.json

@@ -0,0 +1,32 @@
+[
+  {
+    "fabricNode": {
+      "attributes": {
+        "address": "10.0.0.1",
+        "dn": "topology/pod-1/node-1",
+        "fabricSt": "commissioned",
+        "id": "1",
+        "model": "APIC-SERVER-L2",
+        "monPolDn": "uni/fabric/monfab-default",
+        "name": "apic1",
+        "nodeType": "unspecified",
+        "role": "controller"
+      }
+    }
+  },
+  {
+    "fabricNode": {
+      "attributes": {
+        "address": "10.0.0.101",
+        "dn": "topology/pod-1/node-101",
+        "fabricSt": "active",
+        "id": "101",
+        "model": "N9K-C93180YC-FX",
+        "monPolDn": "uni/fabric/monfab-default",
+        "name": "leaf101",
+        "nodeType": "unspecified",
+        "role": "leaf"
+      }
+    }
+  }
+]

tests/checks/modular_spine_bootscript_check/fabricNode_with_fixed_spine.json

@@ -0,0 +1,47 @@
+[
+  {
+    "fabricNode": {
+      "attributes": {
+        "address": "10.0.0.1",
+        "dn": "topology/pod-1/node-1",
+        "fabricSt": "commissioned",
+        "id": "1",
+        "model": "APIC-SERVER-L2",
+        "monPolDn": "uni/fabric/monfab-default",
+        "name": "apic1",
+        "nodeType": "unspecified",
+        "role": "controller"
+      }
+    }
+  },
+  {
+    "fabricNode": {
+      "attributes": {
+        "address": "10.0.0.201",
+        "dn": "topology/pod-1/node-201",
+        "fabricSt": "active",
+        "id": "201",
+        "model": "N9K-C9316D-GX2",
+        "monPolDn": "uni/fabric/monfab-default",
+        "name": "spine201",
+        "nodeType": "unspecified",
+        "role": "spine"
+      }
+    }
+  },
+  {
+    "fabricNode": {
+      "attributes": {
+        "address": "10.0.0.202",
+        "dn": "topology/pod-2/node-202",
+        "fabricSt": "active",
+        "id": "202",
+        "model": "N9K-C9316D-GX",
+        "monPolDn": "uni/fabric/monfab-default",
+        "name": "spine202",
+        "nodeType": "unspecified",
+        "role": "spine"
+      }
+    }
+  }
+]

tests/checks/modular_spine_bootscript_check/fabricNode_with_modular_spine.json

@@ -0,0 +1,47 @@
+[
+  {
+    "fabricNode": {
+      "attributes": {
+        "address": "10.0.0.1",
+        "dn": "topology/pod-1/node-1",
+        "fabricSt": "commissioned",
+        "id": "1",
+        "model": "APIC-SERVER-L2",
+        "monPolDn": "uni/fabric/monfab-default",
+        "name": "apic1",
+        "nodeType": "unspecified",
+        "role": "controller"
+      }
+    }
+  },
+  {
+    "fabricNode": {
+      "attributes": {
+        "address": "10.0.0.201",
+        "dn": "topology/pod-1/node-201",
+        "fabricSt": "active",
+        "id": "201",
+        "model": "N9K-C9504",
+        "monPolDn": "uni/fabric/monfab-default",
+        "name": "spine201",
+        "nodeType": "unspecified",
+        "role": "spine"
+      }
+    }
+  },
+  {
+    "fabricNode": {
+      "attributes": {
+        "address": "10.0.0.202",
+        "dn": "topology/pod-2/node-202",
+        "fabricSt": "active",
+        "id": "202",
+        "model": "N9K-C9508",
+        "monPolDn": "uni/fabric/monfab-default",
+        "name": "spine202",
+        "nodeType": "unspecified",
+        "role": "spine"
+      }
+    }
+  }
+]