From 173ff863a4dba562b109b519c1aa6160c2e6fac1 Mon Sep 17 00:00:00 2001 From: sanjanch Date: Sun, 8 Mar 2026 17:01:51 +0000 Subject: [PATCH 1/8] Resolved the merge conflicts --- aci-preupgrade-validation-script.py | 41 +++++++++ docs/docs/validations.md | 1 + .../faultInst_neg_cleared.json | 30 +++++++ .../faultInst_pos_f4390.json | 30 +++++++ .../faultInst_pos_multiple.json | 86 +++++++++++++++++++ .../faultInst_pos_raised.json | 30 +++++++ .../faultInst_pos_unparseable.json | 30 +++++++ .../test_apic_storage_inode_full_check.py | 76 ++++++++++++++++ 8 files changed, 324 insertions(+) create mode 100644 tests/checks/apic_storage_inode_full_check/faultInst_neg_cleared.json create mode 100644 tests/checks/apic_storage_inode_full_check/faultInst_pos_f4390.json create mode 100644 tests/checks/apic_storage_inode_full_check/faultInst_pos_multiple.json create mode 100644 tests/checks/apic_storage_inode_full_check/faultInst_pos_raised.json create mode 100644 tests/checks/apic_storage_inode_full_check/faultInst_pos_unparseable.json create mode 100644 tests/checks/apic_storage_inode_full_check/test_apic_storage_inode_full_check.py diff --git a/aci-preupgrade-validation-script.py b/aci-preupgrade-validation-script.py index f29c66bd..57a4d9f1 100644 --- a/aci-preupgrade-validation-script.py +++ b/aci-preupgrade-validation-script.py @@ -6053,6 +6053,46 @@ def auto_firmware_update_on_switch_check(cversion, tversion, **kwargs): return Result(result=result, headers=headers, data=data, recommended_action=recommended_action, doc_url=doc_url) + +@check_wrapper(check_title="APIC Storage Inode Check (F4388, F4390 equipment-full)") +def apic_storage_inode_check(**kwargs): + result = FAIL_UF + headers = ['Fault', 'Pod', 'Node', 'Mount Point', 'Usage %', 'Recommended Action'] + data = [] + unformatted_headers = ['Fault', 'Fault DN', 'Recommended Action'] + unformatted_data = [] + recommended_action = 'Contact Cisco TAC to remove the files in the mount point to free up space and clear the fault' + doc_url = 'https://datacenter.github.io/ACI-Pre-Upgrade-Validation-Script/validations/#apic-storage-inode-check' + + dn_regex = node_regex + r'/.+p-\[(?P.+)\]-f' + desc_regex = r'is (?P\d{1,3}%) full for Inodes' + + faultInsts = icurl('class', 'faultInst.json?query-target-filter=or(eq(faultInst.code,"F4388"),eq(faultInst.code,"F4390"))') + for faultInst in faultInsts: + lc = faultInst['faultInst']['attributes']['lc'] + if lc not in ["raised", "soaking"]: + continue + fc = faultInst['faultInst']['attributes']['code'] + dn = re.search(dn_regex, faultInst['faultInst']['attributes']['dn']) + desc = re.search(desc_regex, faultInst['faultInst']['attributes']['descr']) + if dn and desc: + data.append([fc, dn.group('pod'), dn.group('node'), dn.group('mountpoint'), + desc.group('usage'), recommended_action]) + else: + unformatted_data.append([fc, faultInst['faultInst']['attributes']['dn'], recommended_action]) + if not data and not unformatted_data: + result = PASS + + return Result( + result=result, + headers=headers, + data=data, + unformatted_headers=unformatted_headers, + unformatted_data=unformatted_data, + recommended_action=recommended_action, + doc_url=doc_url, + ) + # ---- Script Execution ---- @@ -6162,6 +6202,7 @@ class CheckManager: fabric_port_down_check, equipment_disk_limits_exceeded, apic_vmm_inventory_sync_faults_check, + apic_storage_inode_check, # Configurations vpc_paired_switches_check, diff --git a/docs/docs/validations.md b/docs/docs/validations.md index f46e03de..d884e84f 100644 --- a/docs/docs/validations.md +++ b/docs/docs/validations.md @@ -2648,6 +2648,7 @@ Due to [CSCwp95515][59], upgrading to an affected version while having any `conf If any instances of `configpushShardCont` are flagged by this script, Cisco TAC must be contacted to identify and resolve the underlying issue before performing the upgrade. + ### Auto Firmware Update on Switch Discovery [Auto Firmware Update on Switch Discovery][63] automatically upgrades a new switch to the target firmware version before registering it to the ACI fabric. This feature activates in three scenarios: diff --git a/tests/checks/apic_storage_inode_full_check/faultInst_neg_cleared.json b/tests/checks/apic_storage_inode_full_check/faultInst_neg_cleared.json new file mode 100644 index 00000000..6c790da2 --- /dev/null +++ b/tests/checks/apic_storage_inode_full_check/faultInst_neg_cleared.json @@ -0,0 +1,30 @@ +[ + { + "faultInst": { + "attributes": { + "code": "F4388", + "ack": "yes", + "alert": "no", + "cause": "equipment-full", + "changeSet": "", + "created": "2024-07-20T08:15:30.123+02:00", + "delegated": "no", + "descr": "Storage unit / on Node 2 mounted at / is 77% full for Inodes", + "dn": "topology/pod-1/node-2/sys/ch/p-[/]-f-[overlayfs]/fault-F4388", + "domain": "infra", + "highestSeverity": "warning", + "lastTransition": "2024-07-25T12:30:45.789+02:00", + "lc": "retaining", + "modTs": "never", + "occur": "1", + "origSeverity": "warning", + "prevSeverity": "warning", + "rn": "fault-F4388", + "rule": "eqpt-storage-inode-warning", + "severity": "warning", + "status": "", + "subject": "equipment-full" + } + } + } +] diff --git a/tests/checks/apic_storage_inode_full_check/faultInst_pos_f4390.json b/tests/checks/apic_storage_inode_full_check/faultInst_pos_f4390.json new file mode 100644 index 00000000..c752aa39 --- /dev/null +++ b/tests/checks/apic_storage_inode_full_check/faultInst_pos_f4390.json @@ -0,0 +1,30 @@ +[ + { + "faultInst": { + "attributes": { + "code": "F4390", + "ack": "no", + "alert": "yes", + "cause": "equipment-full", + "changeSet": "available (Old: 10267888, New: 9267684), inodesFree (Old: 291336, New: 191298), inodesUsed (Old: 3030104, New: 3130142), used (Old: 27576548, New: 28576752)", + "created": "2024-08-05T10:30:22.456+02:00", + "delegated": "no", + "descr": "Storage unit /firmware on Node 1 mounted at /firmware is 92% full for Inodes", + "dn": "topology/pod-1/node-1/sys/ch/p-[/firmware]-f-[overlayfs]/fault-F4390", + "domain": "infra", + "highestSeverity": "critical", + "lastTransition": "2024-08-05T11:15:45.123+02:00", + "lc": "raised", + "modTs": "never", + "occur": "3", + "origSeverity": "critical", + "prevSeverity": "critical", + "rn": "fault-F4390", + "rule": "eqpt-storage-inode-critical", + "severity": "critical", + "status": "", + "subject": "equipment-full" + } + } + } +] diff --git a/tests/checks/apic_storage_inode_full_check/faultInst_pos_multiple.json b/tests/checks/apic_storage_inode_full_check/faultInst_pos_multiple.json new file mode 100644 index 00000000..bb3e135a --- /dev/null +++ b/tests/checks/apic_storage_inode_full_check/faultInst_pos_multiple.json @@ -0,0 +1,86 @@ +[ + { + "faultInst": { + "attributes": { + "code": "F4388", + "ack": "yes", + "alert": "no", + "cause": "equipment-full", + "changeSet": "available (Old: 21267888, New: 21267684), inodesFree (Old: 591336, New: 591298), inodesUsed (Old: 2030104, New: 2030142), used (Old: 17576548, New: 17576752)", + "created": "2024-08-01T15:58:47.786+02:00", + "delegated": "no", + "descr": "Storage unit / on Node 3 mounted at / is 78% full for Inodes", + "dn": "topology/pod-2/node-3/sys/ch/p-[/]-f-[overlayfs]/fault-F4388", + "domain": "infra", + "highestSeverity": "warning", + "lastTransition": "2024-08-01T19:42:15.784+02:00", + "lc": "raised", + "modTs": "never", + "occur": "1", + "origSeverity": "warning", + "prevSeverity": "warning", + "rn": "fault-F4388", + "rule": "eqpt-storage-inode-warning", + "severity": "warning", + "status": "", + "subject": "equipment-full" + } + } + }, + { + "faultInst": { + "attributes": { + "code": "F4390", + "ack": "no", + "alert": "yes", + "cause": "equipment-full", + "changeSet": "available (Old: 8267888, New: 7267684), inodesFree (Old: 191336, New: 91298), inodesUsed (Old: 4030104, New: 4130142)", + "created": "2024-08-02T09:30:15.123+02:00", + "delegated": "no", + "descr": "Storage unit /data/techsupport on Node 5 mounted at /data/techsupport is 95% full for Inodes", + "dn": "topology/pod-1/node-5/sys/ch/p-[/data/techsupport]-f-[overlayfs]/fault-F4390", + "domain": "infra", + "highestSeverity": "critical", + "lastTransition": "2024-08-02T10:15:30.456+02:00", + "lc": "raised", + "modTs": "never", + "occur": "2", + "origSeverity": "critical", + "prevSeverity": "critical", + "rn": "fault-F4390", + "rule": "eqpt-storage-inode-critical", + "severity": "critical", + "status": "", + "subject": "equipment-full" + } + } + }, + { + "faultInst": { + "attributes": { + "code": "F4388", + "ack": "yes", + "alert": "no", + "cause": "equipment-full", + "changeSet": "inodesFree (Old: 491336, New: 391298), inodesUsed (Old: 2530104, New: 2630142)", + "created": "2024-08-03T14:22:33.789+02:00", + "delegated": "no", + "descr": "Storage unit /firmware on Node 101 mounted at /firmware is 82% full for Inodes", + "dn": "topology/pod-3/node-101/sys/ch/p-[/firmware]-f-[overlayfs]/fault-F4388", + "domain": "infra", + "highestSeverity": "warning", + "lastTransition": "2024-08-03T15:45:12.345+02:00", + "lc": "raised", + "modTs": "never", + "occur": "1", + "origSeverity": "warning", + "prevSeverity": "warning", + "rn": "fault-F4388", + "rule": "eqpt-storage-inode-warning", + "severity": "warning", + "status": "", + "subject": "equipment-full" + } + } + } +] diff --git a/tests/checks/apic_storage_inode_full_check/faultInst_pos_raised.json b/tests/checks/apic_storage_inode_full_check/faultInst_pos_raised.json new file mode 100644 index 00000000..918b0299 --- /dev/null +++ b/tests/checks/apic_storage_inode_full_check/faultInst_pos_raised.json @@ -0,0 +1,30 @@ +[ + { + "faultInst": { + "attributes": { + "code": "F4388", + "ack": "yes", + "alert": "no", + "cause": "equipment-full", + "changeSet": "available (Old: 21267888, New: 21267684), inodesFree (Old: 591336, New: 591298), inodesUsed (Old: 2030104, New: 2030142), used (Old: 17576548, New: 17576752)", + "created": "2024-08-01T15:58:47.786+02:00", + "delegated": "no", + "descr": "Storage unit / on Node 3 mounted at / is 78% full for Inodes", + "dn": "topology/pod-2/node-3/sys/ch/p-[/]-f-[overlayfs]/fault-F4388", + "domain": "infra", + "highestSeverity": "warning", + "lastTransition": "2024-08-01T19:42:15.784+02:00", + "lc": "raised", + "modTs": "never", + "occur": "1", + "origSeverity": "warning", + "prevSeverity": "warning", + "rn": "fault-F4388", + "rule": "eqpt-storage-inode-warning", + "severity": "warning", + "status": "", + "subject": "equipment-full" + } + } + } +] diff --git a/tests/checks/apic_storage_inode_full_check/faultInst_pos_unparseable.json b/tests/checks/apic_storage_inode_full_check/faultInst_pos_unparseable.json new file mode 100644 index 00000000..5c9161f1 --- /dev/null +++ b/tests/checks/apic_storage_inode_full_check/faultInst_pos_unparseable.json @@ -0,0 +1,30 @@ +[ + { + "faultInst": { + "attributes": { + "code": "F4388", + "ack": "yes", + "alert": "no", + "cause": "equipment-full", + "changeSet": "inodesFree (Old: 291336, New: 191298)", + "created": "2024-08-04T11:15:22.456+02:00", + "delegated": "no", + "descr": "Storage unit mounted at some-weird-path is 85 percent full for Inodes", + "dn": "some/unparseable/dn/format/fault-F4388", + "domain": "infra", + "highestSeverity": "warning", + "lastTransition": "2024-08-04T12:30:45.789+02:00", + "lc": "raised", + "modTs": "never", + "occur": "1", + "origSeverity": "warning", + "prevSeverity": "warning", + "rn": "fault-F4388", + "rule": "eqpt-storage-inode-warning", + "severity": "warning", + "status": "", + "subject": "equipment-full" + } + } + } +] diff --git a/tests/checks/apic_storage_inode_full_check/test_apic_storage_inode_full_check.py b/tests/checks/apic_storage_inode_full_check/test_apic_storage_inode_full_check.py new file mode 100644 index 00000000..39c37e0f --- /dev/null +++ b/tests/checks/apic_storage_inode_full_check/test_apic_storage_inode_full_check.py @@ -0,0 +1,76 @@ +import os +import pytest +import logging +import importlib +from helpers.utils import read_data + +script = importlib.import_module("aci-preupgrade-validation-script") + +log = logging.getLogger(__name__) +dir = os.path.dirname(os.path.abspath(__file__)) + +test_function = "apic_storage_inode_check" + +# icurl queries +faultInst_api = 'faultInst.json' +faultInst_api += '?query-target-filter=or(eq(faultInst.code,"F4388"),eq(faultInst.code,"F4390"))' + + +@pytest.mark.parametrize( + "icurl_outputs, expected_result", + [ + # Target version in affected range (>= 5.3(2b)) and < 6.0(8f)) with raised faults + ( + {faultInst_api: read_data(dir, "faultInst_pos_raised.json")}, + script.FAIL_O, + ), + # Target version in affected range with raised faults (F4390 - critical) + ( + {faultInst_api: read_data(dir, "faultInst_pos_f4390.json")}, + script.FAIL_O, + ), + # Target version in affected range with NO raised faults (cleared faults) + ( + {faultInst_api: read_data(dir, "faultInst_neg_cleared.json")}, + script.PASS, + ), + # Target version in affected range with NO faults at all + ( + {faultInst_api: []}, + script.PASS, + ), + # Target version BELOW affected range (< 5.2(8i)) + ( + {faultInst_api: read_data(dir, "faultInst_pos_raised.json")}, + script.NA, + ), + # Target version ABOVE affected range (>= 6.0(8f)) + ( + {faultInst_api: read_data(dir, "faultInst_pos_raised.json")}, + script.NA, + ), + # Target version way above affected range + ( + {faultInst_api: read_data(dir, "faultInst_pos_raised.json")}, + script.NA, + ), + # NO target version provided + ( + {faultInst_api: []}, + script.MANUAL, + ), + # Multiple raised faults from different nodes + ( + {faultInst_api: read_data(dir, "faultInst_pos_multiple.json")}, + script.FAIL_O, + ), + # Fault with unparseable DN (should go to unformatted_data) + ( + {faultInst_api: read_data(dir, "faultInst_pos_unparseable.json")}, + script.FAIL_O, + ), + ], +) +def test_logic(run_check, mock_icurl, expected_result): + result = run_check() + assert result.result == expected_result From a1c844d7edd0bd7e91ef93a5901e233348fdbf8e Mon Sep 17 00:00:00 2001 From: sanjanch Date: Mon, 9 Mar 2026 19:06:38 +0000 Subject: [PATCH 2/8] Added validation for CSCwm03003 --- aci-preupgrade-validation-script.py | 21 ++--- docs/docs/validations.md | 51 ++++++++++- .../Fault_combination.json | 80 +++++++++++++++++ .../Fault_exists_not_raised.json | 54 ++++++++++++ .../Fault_raised.json | 80 +++++++++++++++++ .../Fault_soaking.json | 81 +++++++++++++++++ .../Fault_unformatted_data.json | 28 ++++++ .../faultInst_neg_cleared.json | 30 ------- .../faultInst_pos_f4390.json | 30 ------- .../faultInst_pos_multiple.json | 86 ------------------- .../faultInst_pos_raised.json | 30 ------- .../faultInst_pos_unparseable.json | 30 ------- .../test_apic_storage_inode_full_check.py | 84 +++++++++--------- 13 files changed, 422 insertions(+), 263 deletions(-) create mode 100644 tests/checks/apic_storage_inode_full_check/Fault_combination.json create mode 100644 tests/checks/apic_storage_inode_full_check/Fault_exists_not_raised.json create mode 100644 tests/checks/apic_storage_inode_full_check/Fault_raised.json create mode 100644 tests/checks/apic_storage_inode_full_check/Fault_soaking.json create mode 100644 tests/checks/apic_storage_inode_full_check/Fault_unformatted_data.json delete mode 100644 tests/checks/apic_storage_inode_full_check/faultInst_neg_cleared.json delete mode 100644 tests/checks/apic_storage_inode_full_check/faultInst_pos_f4390.json delete mode 100644 tests/checks/apic_storage_inode_full_check/faultInst_pos_multiple.json delete mode 100644 tests/checks/apic_storage_inode_full_check/faultInst_pos_raised.json delete mode 100644 tests/checks/apic_storage_inode_full_check/faultInst_pos_unparseable.json diff --git a/aci-preupgrade-validation-script.py b/aci-preupgrade-validation-script.py index 57a4d9f1..81aeb485 100644 --- a/aci-preupgrade-validation-script.py +++ b/aci-preupgrade-validation-script.py @@ -6054,7 +6054,7 @@ def auto_firmware_update_on_switch_check(cversion, tversion, **kwargs): return Result(result=result, headers=headers, data=data, recommended_action=recommended_action, doc_url=doc_url) -@check_wrapper(check_title="APIC Storage Inode Check (F4388, F4390 equipment-full)") +@check_wrapper(check_title="APIC Storage Inode Check (F4388, F4389, F4390 equipment-full)") def apic_storage_inode_check(**kwargs): result = FAIL_UF headers = ['Fault', 'Pod', 'Node', 'Mount Point', 'Usage %', 'Recommended Action'] @@ -6063,11 +6063,9 @@ def apic_storage_inode_check(**kwargs): unformatted_data = [] recommended_action = 'Contact Cisco TAC to remove the files in the mount point to free up space and clear the fault' doc_url = 'https://datacenter.github.io/ACI-Pre-Upgrade-Validation-Script/validations/#apic-storage-inode-check' - dn_regex = node_regex + r'/.+p-\[(?P.+)\]-f' - desc_regex = r'is (?P\d{1,3}%) full for Inodes' - - faultInsts = icurl('class', 'faultInst.json?query-target-filter=or(eq(faultInst.code,"F4388"),eq(faultInst.code,"F4390"))') + desc_regex = r'is (?P\d{2,3}%) full for Inodes' + faultInsts = icurl('class', 'faultInst.json?query-target-filter=or(eq(faultInst.code,"F4388"),eq(faultInst.code,"F4389"),eq(faultInst.code,"F4390"))') for faultInst in faultInsts: lc = faultInst['faultInst']['attributes']['lc'] if lc not in ["raised", "soaking"]: @@ -6076,22 +6074,13 @@ def apic_storage_inode_check(**kwargs): dn = re.search(dn_regex, faultInst['faultInst']['attributes']['dn']) desc = re.search(desc_regex, faultInst['faultInst']['attributes']['descr']) if dn and desc: - data.append([fc, dn.group('pod'), dn.group('node'), dn.group('mountpoint'), - desc.group('usage'), recommended_action]) + data.append([fc, dn.group('pod'), dn.group('node'), dn.group('mountpoint'), desc.group('usage'), recommended_action]) else: unformatted_data.append([fc, faultInst['faultInst']['attributes']['dn'], recommended_action]) if not data and not unformatted_data: result = PASS + return Result(result=result, headers=headers, data=data, unformatted_headers=unformatted_headers, unformatted_data=unformatted_data, recommended_action=recommended_action, doc_url=doc_url) - return Result( - result=result, - headers=headers, - data=data, - unformatted_headers=unformatted_headers, - unformatted_data=unformatted_data, - recommended_action=recommended_action, - doc_url=doc_url, - ) # ---- Script Execution ---- diff --git a/docs/docs/validations.md b/docs/docs/validations.md index d884e84f..36aa784a 100644 --- a/docs/docs/validations.md +++ b/docs/docs/validations.md @@ -82,7 +82,7 @@ Items | Faults | This Script [Fabric Port Status][f19] | F1394: ethpm-if-port-down-fabric | :white_check_mark: | :no_entry_sign: [Equipment Disk Limits][f20] | F1820: 80% -minor
F1821: -major
F1822: -critical | :white_check_mark: | :no_entry_sign: [VMM Inventory Partially Synced][f21] | F0132: comp-ctrlr-operational-issues | :white_check_mark: | :no_entry_sign: - +[APIC Storage Inode Check][f22] | F4388: 75% - 85% -warning
F4389: 85% - 90% -major
F4390: 90% or more -critical | :white_check_mark: | :no_entry_sign: [f1]: #apic-disk-space-usage [f2]: #standby-apic-disk-space-usage @@ -105,6 +105,7 @@ Items | Faults | This Script [f19]: #fabric-port-status [f20]: #equipment-disk-limits [f21]: #vmm-inventory-partially-synced +[f22]: #apic-storage-inode-check ### Configuration Checks @@ -1551,6 +1552,54 @@ EPGs using the `pre-provision` resolution immediacy do not rely on the VMM inven This check returns a `MANUAL` result as there are many reasons for a partial inventory sync to be reported. The goal is to ensure that the VMM inventory sync has fully completed before triggering the APIC upgrade to reduce any chance for unexpected inventory changes to occur. + +### APIC Storage Inode Check + +If a Cisco APIC is running low on inode capacity for any reason, the Cisco APIC upgrade can fail. The Cisco APIC will raise three different faults depending on inode utilization. If any of these faults are raised on the system, the issue should be resolved prior to performing the upgrade. + +* **F4388**: A warning level fault for Cisco APIC storage inode utilization. This is raised when utilization is greater than 75%. + +* **F4389**: A major level fault for Cisco APIC storage inode utilization. This is raised when utilization is between 85% and 90%. + +* **F4390**: A critical level fault for Cisco APIC storage inode utilization. This is raised when utilization is greater than 90%. + +Recommended Action: + +To recover from this fault, try the following action + +1. Free up space from affected disk partition . +2. TAC may be required to analyze and cleanup certain directories due to filesystem permissions. Cleanup of `/` is one such example. + +!!! example "Fault Example (F4390: " Critical fault for APIC Inode Utilisation) + ``` + moquery -c faultInst -f 'fault.Inst.code=="F4390"' + Total Objects shown: 1 + + # faultInst + ack : yes + alert : no + cause : equipment-full + changeSet : available (Old: 19408344, New: 19407972), inodesFree (Old: 263915, New: 263842), inodesUsed (Old: 2357525, New: 2357598), + used (Old: 19436092, New: 19436464) + code : F4390 + created : 2024-08-05T05:42:31.975+02:00 + delegated : no + descr : Storage unit /scratch-writes on node 3 with hostname b001nnc000003 mounted at /scratch-writes is 90% full for Inodes + dn : topology/pod-2/node-3/sys/ch/p-[/scratch-writes]-f-[/dev/mapper/atx-scratch]/fault-F4390 + domain : infra + highestSeverity : critical + lastTransition : 2024-08-05T09:41:18.152+02:00 + lc : raised + occur : 2 + origSeverity : critical + prevSeverity : cleared + rule : eqpt-storage-inode-critical + severity : critical + subject : equipment-full + type : operational + ``` + + ## Configuration Check Details ### VPC-paired Leaf switches diff --git a/tests/checks/apic_storage_inode_full_check/Fault_combination.json b/tests/checks/apic_storage_inode_full_check/Fault_combination.json new file mode 100644 index 00000000..5f1bb267 --- /dev/null +++ b/tests/checks/apic_storage_inode_full_check/Fault_combination.json @@ -0,0 +1,80 @@ +[ + { + "faultInst": { + "attributes": { + "ack": "no", + "alert": "no", + "cause": "equipment-full", + "changeSet": "available (Old: 37868344, New: 37859228), inodesFree (Old: 810163, New: 479339), inodesUsed (Old: 1811277, New: 2142101), inodesUtilized (Old: 70, New: 82), used (Old: 976092, New: 985208)", + "code": "F4388", + "created": "2026-03-06T11:58:43.579+00:00", + "delegated": "no", + "descr": "Storage unit /data/admin/bin/avread on Node 1 mounted at /data/admin/bin/avread is 82% full for Inodes", + "dn": "topology/pod-1/node-1/sys/ch/p-[/data/admin/bin/avread]-f-[overlayfs]/fault-F4388", + "domain": "infra", + "highestSeverity": "warning", + "lastTransition": "2026-03-06T11:58:43.579+00:00", + "lc": "raised", + "occur": "1", + "origSeverity": "warning", + "prevSeverity": "warning", + "rule": "eqpt-storage-inode-warning", + "severity": "warning", + "subject": "equipment-full", + "type": "operational" + } + } + }, + { + "faultInst": { + "attributes": { + "ack": "no", + "alert": "no", + "cause": "equipment-full", + "changeSet": "available (Old: 37868344, New: 37859228), inodesFree (Old: 810163, New: 479339), inodesUsed (Old: 1811277, New: 2142101), inodesUtilized (Old: 70, New: 82), used (Old: 976092, New: 985208)", + "code": "F4388", + "created": "2026-03-06T11:58:43.587+00:00", + "delegated": "no", + "descr": "Storage unit /etc/hosts on Node 1 mounted at /etc/hosts is 82% full for Inodes", + "dn": "topology/pod-1/node-1/sys/ch/p-[/etc/hosts]-f-[overlayfs]/fault-F4388", + "domain": "infra", + "highestSeverity": "warning", + "lastTransition": "2026-03-06T11:58:43.587+00:00", + "lc": "soaking", + "occur": "1", + "origSeverity": "warning", + "prevSeverity": "warning", + "rule": "eqpt-storage-inode-warning", + "severity": "warning", + "subject": "equipment-full", + "type": "operational" + } + } + }, + { + "faultInst": { + "attributes": { + "ack": "no", + "alert": "no", + "cause": "equipment-full", + "changeSet": "available (Old: 37868344, New: 37859228), inodesFree (Old: 810163, New: 479339), inodesUsed (Old: 1811277, New: 2142101), inodesUtilized (Old: 70, New: 82), used (Old: 976092, New: 985208)", + "code": "F4388", + "created": "2026-03-06T11:58:43.595+00:00", + "delegated": "no", + "descr": "Storage unit /scratch-writes on Node 1 mounted at /scratch-writes is 82% full for Inodes", + "dn": "topology/pod-1/node-1/sys/ch/p-[/scratch-writes]-f-[/dev/mapper/atx-scratch]/fault-F4388", + "domain": "infra", + "highestSeverity": "warning", + "lastTransition": "2026-03-06T11:58:43.595+00:00", + "lc": "raised-clearing", + "occur": "1", + "origSeverity": "warning", + "prevSeverity": "warning", + "rule": "eqpt-storage-inode-warning", + "severity": "warning", + "subject": "equipment-full", + "type": "operational" + } + } + } +] \ No newline at end of file diff --git a/tests/checks/apic_storage_inode_full_check/Fault_exists_not_raised.json b/tests/checks/apic_storage_inode_full_check/Fault_exists_not_raised.json new file mode 100644 index 00000000..468facd3 --- /dev/null +++ b/tests/checks/apic_storage_inode_full_check/Fault_exists_not_raised.json @@ -0,0 +1,54 @@ +[ + { + "faultInst": { + "attributes": { + "ack": "no", + "alert": "no", + "cause": "equipment-full", + "changeSet": "available (Old: 37868344, New: 37859228), inodesFree (Old: 810163, New: 479339), inodesUsed (Old: 1811277, New: 2142101), inodesUtilized (Old: 70, New: 82), used (Old: 976092, New: 985208)", + "code": "F4388", + "created": "2026-03-06T11:58:43.579+00:00", + "delegated": "no", + "descr": "Storage unit /data/admin/bin/avread on Node 1 mounted at /data/admin/bin/avread is 82% full for Inodes", + "dn": "topology/pod-1/node-1/sys/ch/p-[/data/admin/bin/avread]-f-[overlayfs]/fault-F4388", + "domain": "infra", + "highestSeverity": "warning", + "lastTransition": "2026-03-06T11:58:43.579+00:00", + "lc": "cleared", + "occur": "1", + "origSeverity": "warning", + "prevSeverity": "warning", + "rule": "eqpt-storage-inode-warning", + "severity": "warning", + "subject": "equipment-full", + "type": "operational" + } + } + }, + { + "faultInst": { + "attributes": { + "ack": "no", + "alert": "no", + "cause": "equipment-full", + "changeSet": "available (Old: 37868344, New: 37859228), inodesFree (Old: 810163, New: 479339), inodesUsed (Old: 1811277, New: 2142101), inodesUtilized (Old: 70, New: 82), used (Old: 976092, New: 985208)", + "code": "F4388", + "created": "2026-03-06T11:58:43.587+00:00", + "delegated": "no", + "descr": "Storage unit /etc/hosts on Node 1 mounted at /etc/hosts is 82% full for Inodes", + "dn": "topology/pod-1/node-1/sys/ch/p-[/etc/hosts]-f-[overlayfs]/fault-F4388", + "domain": "infra", + "highestSeverity": "warning", + "lastTransition": "2026-03-06T11:58:43.587+00:00", + "lc": "retaining", + "occur": "1", + "origSeverity": "warning", + "prevSeverity": "warning", + "rule": "eqpt-storage-inode-warning", + "severity": "warning", + "subject": "equipment-full", + "type": "operational" + } + } + } +] diff --git a/tests/checks/apic_storage_inode_full_check/Fault_raised.json b/tests/checks/apic_storage_inode_full_check/Fault_raised.json new file mode 100644 index 00000000..ec5760ab --- /dev/null +++ b/tests/checks/apic_storage_inode_full_check/Fault_raised.json @@ -0,0 +1,80 @@ +[ + { + "faultInst": { + "attributes": { + "ack": "no", + "alert": "no", + "cause": "equipment-full", + "changeSet": "available (Old: 37868344, New: 37859228), inodesFree (Old: 810163, New: 479339), inodesUsed (Old: 1811277, New: 2142101), inodesUtilized (Old: 70, New: 82), used (Old: 976092, New: 985208)", + "code": "F4388", + "created": "2026-03-06T11:58:43.579+00:00", + "delegated": "no", + "descr": "Storage unit /data/admin/bin/avread on Node 1 mounted at /data/admin/bin/avread is 82% full for Inodes", + "dn": "topology/pod-1/node-1/sys/ch/p-[/data/admin/bin/avread]-f-[overlayfs]/fault-F4388", + "domain": "infra", + "highestSeverity": "warning", + "lastTransition": "2026-03-06T12:00:53.560+00:00", + "lc": "raised", + "occur": "1", + "origSeverity": "warning", + "prevSeverity": "warning", + "rule": "eqpt-storage-inode-warning", + "severity": "warning", + "subject": "equipment-full", + "type": "operational" + } + } + }, + { + "faultInst": { + "attributes": { + "ack": "no", + "alert": "no", + "cause": "equipment-full", + "changeSet": "available (Old: 37868344, New: 37859228), inodesFree (Old: 810163, New: 479339), inodesUsed (Old: 1811277, New: 2142101), inodesUtilized (Old: 70, New: 82), used (Old: 976092, New: 985208)", + "code": "F4388", + "created": "2026-03-06T11:58:43.587+00:00", + "delegated": "no", + "descr": "Storage unit /etc/hosts on Node 1 mounted at /etc/hosts is 82% full for Inodes", + "dn": "topology/pod-1/node-1/sys/ch/p-[/etc/hosts]-f-[overlayfs]/fault-F4388", + "domain": "infra", + "highestSeverity": "warning", + "lastTransition": "2026-03-06T12:00:53.560+00:00", + "lc": "raised", + "occur": "1", + "origSeverity": "warning", + "prevSeverity": "warning", + "rule": "eqpt-storage-inode-warning", + "severity": "warning", + "subject": "equipment-full", + "type": "operational" + } + } + }, + { + "faultInst": { + "attributes": { + "ack": "no", + "alert": "no", + "cause": "equipment-full", + "changeSet": "available (Old: 37868344, New: 37859228), inodesFree (Old: 810163, New: 479339), inodesUsed (Old: 1811277, New: 2142101), inodesUtilized (Old: 70, New: 82), used (Old: 976092, New: 985208)", + "code": "F4388", + "created": "2026-03-06T11:58:43.602+00:00", + "delegated": "no", + "descr": "Storage unit / on Node 1 mounted at / is 82% full for Inodes", + "dn": "topology/pod-1/node-1/sys/ch/p-[/]-f-[overlayfs]/fault-F4388", + "domain": "infra", + "highestSeverity": "warning", + "lastTransition": "2026-03-06T12:00:53.560+00:00", + "lc": "raised", + "occur": "1", + "origSeverity": "warning", + "prevSeverity": "warning", + "rule": "eqpt-storage-inode-warning", + "severity": "warning", + "subject": "equipment-full", + "type": "operational" + } + } + } + ] diff --git a/tests/checks/apic_storage_inode_full_check/Fault_soaking.json b/tests/checks/apic_storage_inode_full_check/Fault_soaking.json new file mode 100644 index 00000000..3d4bf800 --- /dev/null +++ b/tests/checks/apic_storage_inode_full_check/Fault_soaking.json @@ -0,0 +1,81 @@ +[ + { + "faultInst": { + "attributes": { + "ack": "no", + "alert": "no", + "cause": "equipment-full", + "changeSet": "available (Old: 37868344, New: 37859228), inodesFree (Old: 810163, New: 479339), inodesUsed (Old: 1811277, New: 2142101), inodesUtilized (Old: 70, New: 82), used (Old: 976092, New: 985208)", + "code": "F4388", + "created": "2026-03-06T11:58:43.579+00:00", + "delegated": "no", + "descr": "Storage unit /data/admin/bin/avread on Node 1 mounted at /data/admin/bin/avread is 82% full for Inodes", + "dn": "topology/pod-1/node-1/sys/ch/p-[/data/admin/bin/avread]-f-[overlayfs]/fault-F4388", + "domain": "infra", + "highestSeverity": "warning", + "lastTransition": "2026-03-06T11:58:43.579+00:00", + "lc": "soaking", + "occur": "1", + "origSeverity": "warning", + "prevSeverity": "warning", + "rule": "eqpt-storage-inode-warning", + "severity": "warning", + "subject": "equipment-full", + "type": "operational" + } + } + }, + { + "faultInst": { + "attributes": { + "ack": "no", + "alert": "no", + "cause": "equipment-full", + "changeSet": "available (Old: 37868344, New: 37859228), inodesFree (Old: 810163, New: 479339), inodesUsed (Old: 1811277, New: 2142101), inodesUtilized (Old: 70, New: 82), used (Old: 976092, New: 985208)", + "code": "F4388", + "created": "2026-03-06T11:58:43.587+00:00", + "delegated": "no", + "descr": "Storage unit /etc/hosts on Node 1 mounted at /etc/hosts is 82% full for Inodes", + "dn": "topology/pod-1/node-1/sys/ch/p-[/etc/hosts]-f-[overlayfs]/fault-F4388", + "domain": "infra", + "highestSeverity": "warning", + "lastTransition": "2026-03-06T11:58:43.587+00:00", + "lc": "soaking", + "occur": "1", + "origSeverity": "warning", + "prevSeverity": "warning", + "rule": "eqpt-storage-inode-warning", + "severity": "warning", + "subject": "equipment-full", + "type": "operational" + } + } + }, + + { + "faultInst": { + "attributes": { + "ack": "no", + "alert": "no", + "cause": "equipment-full", + "changeSet": "available (Old: 37868344, New: 37859228), inodesFree (Old: 810163, New: 479339), inodesUsed (Old: 1811277, New: 2142101), inodesUtilized (Old: 70, New: 82), used (Old: 976092, New: 985208)", + "code": "F4388", + "created": "2026-03-06T11:58:43.602+00:00", + "delegated": "no", + "descr": "Storage unit / on Node 1 mounted at / is 82% full for Inodes", + "dn": "topology/pod-1/node-1/sys/ch/p-[/]-f-[overlayfs]/fault-F4388", + "domain": "infra", + "highestSeverity": "warning", + "lastTransition": "2026-03-06T11:58:43.602+00:00", + "lc": "soaking", + "occur": "1", + "origSeverity": "warning", + "prevSeverity": "warning", + "rule": "eqpt-storage-inode-warning", + "severity": "warning", + "subject": "equipment-full", + "type": "operational" + } + } + } + ] diff --git a/tests/checks/apic_storage_inode_full_check/Fault_unformatted_data.json b/tests/checks/apic_storage_inode_full_check/Fault_unformatted_data.json new file mode 100644 index 00000000..0c6463ca --- /dev/null +++ b/tests/checks/apic_storage_inode_full_check/Fault_unformatted_data.json @@ -0,0 +1,28 @@ +[ + { + "faultInst": { + "attributes": { + "ack": "no", + "alert": "no", + "cause": "equipment-full", + "changeSet": "available (Old: 37868344, New: 37859228), inodesFree (Old: 810163, New: 479339), inodesUsed (Old: 1811277, New: 2142101), inodesUtilized (Old: 70, New: 82), used (Old: 976092, New: 985208)", + "code": "F4388", + "created": "2026-03-06T11:58:43.602+00:00", + "delegated": "no", + "descr": "Storage unit /unknown on Node 1 mounted at /unknown is 82% full", + "dn": "topology/pod-1/node-1/sys/ch/invalid/fault-F4388", + "domain": "infra", + "highestSeverity": "warning", + "lastTransition": "2026-03-06T11:58:43.602+00:00", + "lc": "raised", + "occur": "1", + "origSeverity": "warning", + "prevSeverity": "warning", + "rule": "eqpt-storage-inode-warning", + "severity": "warning", + "subject": "equipment-full", + "type": "operational" + } + } + } + ] diff --git a/tests/checks/apic_storage_inode_full_check/faultInst_neg_cleared.json b/tests/checks/apic_storage_inode_full_check/faultInst_neg_cleared.json deleted file mode 100644 index 6c790da2..00000000 --- a/tests/checks/apic_storage_inode_full_check/faultInst_neg_cleared.json +++ /dev/null @@ -1,30 +0,0 @@ -[ - { - "faultInst": { - "attributes": { - "code": "F4388", - "ack": "yes", - "alert": "no", - "cause": "equipment-full", - "changeSet": "", - "created": "2024-07-20T08:15:30.123+02:00", - "delegated": "no", - "descr": "Storage unit / on Node 2 mounted at / is 77% full for Inodes", - "dn": "topology/pod-1/node-2/sys/ch/p-[/]-f-[overlayfs]/fault-F4388", - "domain": "infra", - "highestSeverity": "warning", - "lastTransition": "2024-07-25T12:30:45.789+02:00", - "lc": "retaining", - "modTs": "never", - "occur": "1", - "origSeverity": "warning", - "prevSeverity": "warning", - "rn": "fault-F4388", - "rule": "eqpt-storage-inode-warning", - "severity": "warning", - "status": "", - "subject": "equipment-full" - } - } - } -] diff --git a/tests/checks/apic_storage_inode_full_check/faultInst_pos_f4390.json b/tests/checks/apic_storage_inode_full_check/faultInst_pos_f4390.json deleted file mode 100644 index c752aa39..00000000 --- a/tests/checks/apic_storage_inode_full_check/faultInst_pos_f4390.json +++ /dev/null @@ -1,30 +0,0 @@ -[ - { - "faultInst": { - "attributes": { - "code": "F4390", - "ack": "no", - "alert": "yes", - "cause": "equipment-full", - "changeSet": "available (Old: 10267888, New: 9267684), inodesFree (Old: 291336, New: 191298), inodesUsed (Old: 3030104, New: 3130142), used (Old: 27576548, New: 28576752)", - "created": "2024-08-05T10:30:22.456+02:00", - "delegated": "no", - "descr": "Storage unit /firmware on Node 1 mounted at /firmware is 92% full for Inodes", - "dn": "topology/pod-1/node-1/sys/ch/p-[/firmware]-f-[overlayfs]/fault-F4390", - "domain": "infra", - "highestSeverity": "critical", - "lastTransition": "2024-08-05T11:15:45.123+02:00", - "lc": "raised", - "modTs": "never", - "occur": "3", - "origSeverity": "critical", - "prevSeverity": "critical", - "rn": "fault-F4390", - "rule": "eqpt-storage-inode-critical", - "severity": "critical", - "status": "", - "subject": "equipment-full" - } - } - } -] diff --git a/tests/checks/apic_storage_inode_full_check/faultInst_pos_multiple.json b/tests/checks/apic_storage_inode_full_check/faultInst_pos_multiple.json deleted file mode 100644 index bb3e135a..00000000 --- a/tests/checks/apic_storage_inode_full_check/faultInst_pos_multiple.json +++ /dev/null @@ -1,86 +0,0 @@ -[ - { - "faultInst": { - "attributes": { - "code": "F4388", - "ack": "yes", - "alert": "no", - "cause": "equipment-full", - "changeSet": "available (Old: 21267888, New: 21267684), inodesFree (Old: 591336, New: 591298), inodesUsed (Old: 2030104, New: 2030142), used (Old: 17576548, New: 17576752)", - "created": "2024-08-01T15:58:47.786+02:00", - "delegated": "no", - "descr": "Storage unit / on Node 3 mounted at / is 78% full for Inodes", - "dn": "topology/pod-2/node-3/sys/ch/p-[/]-f-[overlayfs]/fault-F4388", - "domain": "infra", - "highestSeverity": "warning", - "lastTransition": "2024-08-01T19:42:15.784+02:00", - "lc": "raised", - "modTs": "never", - "occur": "1", - "origSeverity": "warning", - "prevSeverity": "warning", - "rn": "fault-F4388", - "rule": "eqpt-storage-inode-warning", - "severity": "warning", - "status": "", - "subject": "equipment-full" - } - } - }, - { - "faultInst": { - "attributes": { - "code": "F4390", - "ack": "no", - "alert": "yes", - "cause": "equipment-full", - "changeSet": "available (Old: 8267888, New: 7267684), inodesFree (Old: 191336, New: 91298), inodesUsed (Old: 4030104, New: 4130142)", - "created": "2024-08-02T09:30:15.123+02:00", - "delegated": "no", - "descr": "Storage unit /data/techsupport on Node 5 mounted at /data/techsupport is 95% full for Inodes", - "dn": "topology/pod-1/node-5/sys/ch/p-[/data/techsupport]-f-[overlayfs]/fault-F4390", - "domain": "infra", - "highestSeverity": "critical", - "lastTransition": "2024-08-02T10:15:30.456+02:00", - "lc": "raised", - "modTs": "never", - "occur": "2", - "origSeverity": "critical", - "prevSeverity": "critical", - "rn": "fault-F4390", - "rule": "eqpt-storage-inode-critical", - "severity": "critical", - "status": "", - "subject": "equipment-full" - } - } - }, - { - "faultInst": { - "attributes": { - "code": "F4388", - "ack": "yes", - "alert": "no", - "cause": "equipment-full", - "changeSet": "inodesFree (Old: 491336, New: 391298), inodesUsed (Old: 2530104, New: 2630142)", - "created": "2024-08-03T14:22:33.789+02:00", - "delegated": "no", - "descr": "Storage unit /firmware on Node 101 mounted at /firmware is 82% full for Inodes", - "dn": "topology/pod-3/node-101/sys/ch/p-[/firmware]-f-[overlayfs]/fault-F4388", - "domain": "infra", - "highestSeverity": "warning", - "lastTransition": "2024-08-03T15:45:12.345+02:00", - "lc": "raised", - "modTs": "never", - "occur": "1", - "origSeverity": "warning", - "prevSeverity": "warning", - "rn": "fault-F4388", - "rule": "eqpt-storage-inode-warning", - "severity": "warning", - "status": "", - "subject": "equipment-full" - } - } - } -] diff --git a/tests/checks/apic_storage_inode_full_check/faultInst_pos_raised.json b/tests/checks/apic_storage_inode_full_check/faultInst_pos_raised.json deleted file mode 100644 index 918b0299..00000000 --- a/tests/checks/apic_storage_inode_full_check/faultInst_pos_raised.json +++ /dev/null @@ -1,30 +0,0 @@ -[ - { - "faultInst": { - "attributes": { - "code": "F4388", - "ack": "yes", - "alert": "no", - "cause": "equipment-full", - "changeSet": "available (Old: 21267888, New: 21267684), inodesFree (Old: 591336, New: 591298), inodesUsed (Old: 2030104, New: 2030142), used (Old: 17576548, New: 17576752)", - "created": "2024-08-01T15:58:47.786+02:00", - "delegated": "no", - "descr": "Storage unit / on Node 3 mounted at / is 78% full for Inodes", - "dn": "topology/pod-2/node-3/sys/ch/p-[/]-f-[overlayfs]/fault-F4388", - "domain": "infra", - "highestSeverity": "warning", - "lastTransition": "2024-08-01T19:42:15.784+02:00", - "lc": "raised", - "modTs": "never", - "occur": "1", - "origSeverity": "warning", - "prevSeverity": "warning", - "rn": "fault-F4388", - "rule": "eqpt-storage-inode-warning", - "severity": "warning", - "status": "", - "subject": "equipment-full" - } - } - } -] diff --git a/tests/checks/apic_storage_inode_full_check/faultInst_pos_unparseable.json b/tests/checks/apic_storage_inode_full_check/faultInst_pos_unparseable.json deleted file mode 100644 index 5c9161f1..00000000 --- a/tests/checks/apic_storage_inode_full_check/faultInst_pos_unparseable.json +++ /dev/null @@ -1,30 +0,0 @@ -[ - { - "faultInst": { - "attributes": { - "code": "F4388", - "ack": "yes", - "alert": "no", - "cause": "equipment-full", - "changeSet": "inodesFree (Old: 291336, New: 191298)", - "created": "2024-08-04T11:15:22.456+02:00", - "delegated": "no", - "descr": "Storage unit mounted at some-weird-path is 85 percent full for Inodes", - "dn": "some/unparseable/dn/format/fault-F4388", - "domain": "infra", - "highestSeverity": "warning", - "lastTransition": "2024-08-04T12:30:45.789+02:00", - "lc": "raised", - "modTs": "never", - "occur": "1", - "origSeverity": "warning", - "prevSeverity": "warning", - "rn": "fault-F4388", - "rule": "eqpt-storage-inode-warning", - "severity": "warning", - "status": "", - "subject": "equipment-full" - } - } - } -] diff --git a/tests/checks/apic_storage_inode_full_check/test_apic_storage_inode_full_check.py b/tests/checks/apic_storage_inode_full_check/test_apic_storage_inode_full_check.py index 39c37e0f..da651e8f 100644 --- a/tests/checks/apic_storage_inode_full_check/test_apic_storage_inode_full_check.py +++ b/tests/checks/apic_storage_inode_full_check/test_apic_storage_inode_full_check.py @@ -13,64 +13,68 @@ # icurl queries faultInst_api = 'faultInst.json' -faultInst_api += '?query-target-filter=or(eq(faultInst.code,"F4388"),eq(faultInst.code,"F4390"))' - +faultInst_api += '?query-target-filter=or(eq(faultInst.code,"F4388"),eq(faultInst.code,"F4389"),eq(faultInst.code,"F4390"))' @pytest.mark.parametrize( - "icurl_outputs, expected_result", + "icurl_outputs, expected_result, expected_data", [ - # Target version in affected range (>= 5.3(2b)) and < 6.0(8f)) with raised faults - ( - {faultInst_api: read_data(dir, "faultInst_pos_raised.json")}, - script.FAIL_O, - ), - # Target version in affected range with raised faults (F4390 - critical) - ( - {faultInst_api: read_data(dir, "faultInst_pos_f4390.json")}, - script.FAIL_O, - ), - # Target version in affected range with NO raised faults (cleared faults) - ( - {faultInst_api: read_data(dir, "faultInst_neg_cleared.json")}, - script.PASS, - ), - # Target version in affected range with NO faults at all + # PASS - No raised faults ( {faultInst_api: []}, script.PASS, + [], ), - # Target version BELOW affected range (< 5.2(8i)) - ( - {faultInst_api: read_data(dir, "faultInst_pos_raised.json")}, - script.NA, - ), - # Target version ABOVE affected range (>= 6.0(8f)) + # FAIL - Soaking faults ( - {faultInst_api: read_data(dir, "faultInst_pos_raised.json")}, - script.NA, + {faultInst_api: read_data(dir, "Fault_soaking.json")}, + script.FAIL_UF, + [ + ["F4388", "1", "1", "/data/admin/bin/avread", "82%", "Contact Cisco TAC to remove the files in the mount point to free up space and clear the fault"], + ["F4388", "1", "1", "/etc/hosts", "82%", "Contact Cisco TAC to remove the files in the mount point to free up space and clear the fault"], + ["F4388", "1", "1", "/", "82%", "Contact Cisco TAC to remove the files in the mount point to free up space and clear the fault"], + ], ), - # Target version way above affected range + # FAIL - Raised faults ( - {faultInst_api: read_data(dir, "faultInst_pos_raised.json")}, - script.NA, + {faultInst_api: read_data(dir, "Fault_raised.json")}, + script.FAIL_UF, + [ + ["F4388", "1", "1", "/data/admin/bin/avread", "82%", "Contact Cisco TAC to remove the files in the mount point to free up space and clear the fault"], + ["F4388", "1", "1", "/etc/hosts", "82%", "Contact Cisco TAC to remove the files in the mount point to free up space and clear the fault"], + ["F4388", "1", "1", "/", "82%", "Contact Cisco TAC to remove the files in the mount point to free up space and clear the fault"], + ], ), - # NO target version provided + # PASS - Faults exist but not raised nor soaking (cleared) ( - {faultInst_api: []}, - script.MANUAL, + {faultInst_api: read_data(dir, "Fault_exists_not_raised.json")}, + script.PASS, + [], ), - # Multiple raised faults from different nodes + # FAIL - Raised faults with multiple status - Cleared and Active ( - {faultInst_api: read_data(dir, "faultInst_pos_multiple.json")}, - script.FAIL_O, + {faultInst_api: read_data(dir, "Fault_combination.json")}, + script.FAIL_UF, + [ + ["F4388", "1", "1", "/data/admin/bin/avread", "82%", "Contact Cisco TAC to remove the files in the mount point to free up space and clear the fault"], + ["F4388", "1", "1", "/etc/hosts", "82%", "Contact Cisco TAC to remove the files in the mount point to free up space and clear the fault"], + ], ), - # Fault with unparseable DN (should go to unformatted_data) + # FAIL - Raised faults with unknown mount point (unformatted data) ( - {faultInst_api: read_data(dir, "faultInst_pos_unparseable.json")}, - script.FAIL_O, + {faultInst_api: read_data(dir, "Fault_unformatted_data.json")}, + script.FAIL_UF, + [ + ["F4388", "topology/pod-1/node-1/sys/ch/invalid/fault-F4388", "Contact Cisco TAC to remove the files in the mount point to free up space and clear the fault"], + ], ), ], ) -def test_logic(run_check, mock_icurl, expected_result): + +def test_logic(run_check, mock_icurl, expected_result, expected_data): result = run_check() assert result.result == expected_result + if result.data: + assert result.data == expected_data + else: + assert result.unformatted_data == expected_data + \ No newline at end of file From d2ae53700c0b5266f9e12b55d68fcb8018498575 Mon Sep 17 00:00:00 2001 From: sanjanch Date: Wed, 11 Mar 2026 12:24:46 +0000 Subject: [PATCH 3/8] Updated the validation.md file for APIC inode check --- docs/docs/validations.md | 2 ++ .../test_apic_storage_inode_full_check.py | 3 --- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/docs/docs/validations.md b/docs/docs/validations.md index 36aa784a..8a32c141 100644 --- a/docs/docs/validations.md +++ b/docs/docs/validations.md @@ -1563,6 +1563,8 @@ If a Cisco APIC is running low on inode capacity for any reason, the Cisco APIC * **F4390**: A critical level fault for Cisco APIC storage inode utilization. This is raised when utilization is greater than 90%. +Although the storage space for the filesystem might be adequate we might still see issues with inode usage, this happens when we have more number of files or directories created with lower file sizes. + Recommended Action: To recover from this fault, try the following action diff --git a/tests/checks/apic_storage_inode_full_check/test_apic_storage_inode_full_check.py b/tests/checks/apic_storage_inode_full_check/test_apic_storage_inode_full_check.py index da651e8f..d0e5ff66 100644 --- a/tests/checks/apic_storage_inode_full_check/test_apic_storage_inode_full_check.py +++ b/tests/checks/apic_storage_inode_full_check/test_apic_storage_inode_full_check.py @@ -8,10 +8,7 @@ log = logging.getLogger(__name__) dir = os.path.dirname(os.path.abspath(__file__)) - test_function = "apic_storage_inode_check" - -# icurl queries faultInst_api = 'faultInst.json' faultInst_api += '?query-target-filter=or(eq(faultInst.code,"F4388"),eq(faultInst.code,"F4389"),eq(faultInst.code,"F4390"))' From 15f5f48f34727c0faba0a73701a9e93b8d95f1ee Mon Sep 17 00:00:00 2001 From: sanjanch Date: Wed, 11 Mar 2026 12:35:01 +0000 Subject: [PATCH 4/8] Removed unnecessary spaces --- docs/docs/validations.md | 1 - .../test_apic_storage_inode_full_check.py | 3 +-- 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/docs/docs/validations.md b/docs/docs/validations.md index 8a32c141..ba63890b 100644 --- a/docs/docs/validations.md +++ b/docs/docs/validations.md @@ -2699,7 +2699,6 @@ Due to [CSCwp95515][59], upgrading to an affected version while having any `conf If any instances of `configpushShardCont` are flagged by this script, Cisco TAC must be contacted to identify and resolve the underlying issue before performing the upgrade. - ### Auto Firmware Update on Switch Discovery [Auto Firmware Update on Switch Discovery][63] automatically upgrades a new switch to the target firmware version before registering it to the ACI fabric. This feature activates in three scenarios: diff --git a/tests/checks/apic_storage_inode_full_check/test_apic_storage_inode_full_check.py b/tests/checks/apic_storage_inode_full_check/test_apic_storage_inode_full_check.py index d0e5ff66..7a5e58c1 100644 --- a/tests/checks/apic_storage_inode_full_check/test_apic_storage_inode_full_check.py +++ b/tests/checks/apic_storage_inode_full_check/test_apic_storage_inode_full_check.py @@ -73,5 +73,4 @@ def test_logic(run_check, mock_icurl, expected_result, expected_data): if result.data: assert result.data == expected_data else: - assert result.unformatted_data == expected_data - \ No newline at end of file + assert result.unformatted_data == expected_data \ No newline at end of file From cc8da56bbc1411f7f2e4e3de2178780e14bbeb69 Mon Sep 17 00:00:00 2001 From: tkishida Date: Fri, 3 Apr 2026 11:56:58 -0700 Subject: [PATCH 5/8] fix: Remove recommended_action from data when all rows use the same text --- aci-preupgrade-validation-script.py | 10 +++---- .../test_apic_storage_inode_full_check.py | 28 ++++++++++--------- 2 files changed, 20 insertions(+), 18 deletions(-) diff --git a/aci-preupgrade-validation-script.py b/aci-preupgrade-validation-script.py index 1d99105b..4d259880 100644 --- a/aci-preupgrade-validation-script.py +++ b/aci-preupgrade-validation-script.py @@ -6195,15 +6195,15 @@ def n9k_c9408_model_lem_count_check(tversion, fabric_nodes, **kwargs): @check_wrapper(check_title="APIC Storage Inode Check (F4388, F4389, F4390 equipment-full)") def apic_storage_inode_check(**kwargs): result = FAIL_UF - headers = ['Fault', 'Pod', 'Node', 'Mount Point', 'Usage %', 'Recommended Action'] + headers = ['Fault', 'Pod', 'Node', 'Mount Point', 'Usage %'] data = [] - unformatted_headers = ['Fault', 'Fault DN', 'Recommended Action'] + unformatted_headers = ['Fault', 'Fault DN'] unformatted_data = [] recommended_action = 'Contact Cisco TAC to remove the files in the mount point to free up space and clear the fault' doc_url = 'https://datacenter.github.io/ACI-Pre-Upgrade-Validation-Script/validations/#apic-storage-inode-check' dn_regex = node_regex + r'/.+p-\[(?P.+)\]-f' desc_regex = r'is (?P\d{2,3}%) full for Inodes' - faultInsts = icurl('class', 'faultInst.json?query-target-filter=or(eq(faultInst.code,"F4388"),eq(faultInst.code,"F4389"),eq(faultInst.code,"F4390"))') + faultInsts = icurl('class', 'faultInst.json?query-target-filter=or(eq(faultInst.code,"F4388"),eq(faultInst.code,"F4389"),eq(faultInst.code,"F4390"))') for faultInst in faultInsts: lc = faultInst['faultInst']['attributes']['lc'] if lc not in ["raised", "soaking"]: @@ -6212,9 +6212,9 @@ def apic_storage_inode_check(**kwargs): dn = re.search(dn_regex, faultInst['faultInst']['attributes']['dn']) desc = re.search(desc_regex, faultInst['faultInst']['attributes']['descr']) if dn and desc: - data.append([fc, dn.group('pod'), dn.group('node'), dn.group('mountpoint'), desc.group('usage'), recommended_action]) + data.append([fc, dn.group('pod'), dn.group('node'), dn.group('mountpoint'), desc.group('usage')]) else: - unformatted_data.append([fc, faultInst['faultInst']['attributes']['dn'], recommended_action]) + unformatted_data.append([fc, faultInst['faultInst']['attributes']['dn']]) if not data and not unformatted_data: result = PASS return Result(result=result, headers=headers, data=data, unformatted_headers=unformatted_headers, unformatted_data=unformatted_data, recommended_action=recommended_action, doc_url=doc_url) diff --git a/tests/checks/apic_storage_inode_full_check/test_apic_storage_inode_full_check.py b/tests/checks/apic_storage_inode_full_check/test_apic_storage_inode_full_check.py index 7a5e58c1..e9ca9ff8 100644 --- a/tests/checks/apic_storage_inode_full_check/test_apic_storage_inode_full_check.py +++ b/tests/checks/apic_storage_inode_full_check/test_apic_storage_inode_full_check.py @@ -9,8 +9,11 @@ log = logging.getLogger(__name__) dir = os.path.dirname(os.path.abspath(__file__)) test_function = "apic_storage_inode_check" -faultInst_api = 'faultInst.json' -faultInst_api += '?query-target-filter=or(eq(faultInst.code,"F4388"),eq(faultInst.code,"F4389"),eq(faultInst.code,"F4390"))' +faultInst_api = "faultInst.json" +faultInst_api += ( + '?query-target-filter=or(eq(faultInst.code,"F4388"),eq(faultInst.code,"F4389"),eq(faultInst.code,"F4390"))' +) + @pytest.mark.parametrize( "icurl_outputs, expected_result, expected_data", @@ -26,9 +29,9 @@ {faultInst_api: read_data(dir, "Fault_soaking.json")}, script.FAIL_UF, [ - ["F4388", "1", "1", "/data/admin/bin/avread", "82%", "Contact Cisco TAC to remove the files in the mount point to free up space and clear the fault"], - ["F4388", "1", "1", "/etc/hosts", "82%", "Contact Cisco TAC to remove the files in the mount point to free up space and clear the fault"], - ["F4388", "1", "1", "/", "82%", "Contact Cisco TAC to remove the files in the mount point to free up space and clear the fault"], + ["F4388", "1", "1", "/data/admin/bin/avread", "82%"], + ["F4388", "1", "1", "/etc/hosts", "82%"], + ["F4388", "1", "1", "/", "82%"], ], ), # FAIL - Raised faults @@ -36,9 +39,9 @@ {faultInst_api: read_data(dir, "Fault_raised.json")}, script.FAIL_UF, [ - ["F4388", "1", "1", "/data/admin/bin/avread", "82%", "Contact Cisco TAC to remove the files in the mount point to free up space and clear the fault"], - ["F4388", "1", "1", "/etc/hosts", "82%", "Contact Cisco TAC to remove the files in the mount point to free up space and clear the fault"], - ["F4388", "1", "1", "/", "82%", "Contact Cisco TAC to remove the files in the mount point to free up space and clear the fault"], + ["F4388", "1", "1", "/data/admin/bin/avread", "82%"], + ["F4388", "1", "1", "/etc/hosts", "82%"], + ["F4388", "1", "1", "/", "82%"], ], ), # PASS - Faults exist but not raised nor soaking (cleared) @@ -52,8 +55,8 @@ {faultInst_api: read_data(dir, "Fault_combination.json")}, script.FAIL_UF, [ - ["F4388", "1", "1", "/data/admin/bin/avread", "82%", "Contact Cisco TAC to remove the files in the mount point to free up space and clear the fault"], - ["F4388", "1", "1", "/etc/hosts", "82%", "Contact Cisco TAC to remove the files in the mount point to free up space and clear the fault"], + ["F4388", "1", "1", "/data/admin/bin/avread", "82%"], + ["F4388", "1", "1", "/etc/hosts", "82%"], ], ), # FAIL - Raised faults with unknown mount point (unformatted data) @@ -61,16 +64,15 @@ {faultInst_api: read_data(dir, "Fault_unformatted_data.json")}, script.FAIL_UF, [ - ["F4388", "topology/pod-1/node-1/sys/ch/invalid/fault-F4388", "Contact Cisco TAC to remove the files in the mount point to free up space and clear the fault"], + ["F4388", "topology/pod-1/node-1/sys/ch/invalid/fault-F4388"], ], ), ], ) - def test_logic(run_check, mock_icurl, expected_result, expected_data): result = run_check() assert result.result == expected_result if result.data: assert result.data == expected_data else: - assert result.unformatted_data == expected_data \ No newline at end of file + assert result.unformatted_data == expected_data From d66ed2c71dd6ea89881904002b9790a295ff6b71 Mon Sep 17 00:00:00 2001 From: tkishida Date: Fri, 3 Apr 2026 11:58:31 -0700 Subject: [PATCH 6/8] doc: Fix admonitions syntax error --- docs/docs/validations.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/docs/validations.md b/docs/docs/validations.md index 4b1d2b95..f76d6d05 100644 --- a/docs/docs/validations.md +++ b/docs/docs/validations.md @@ -1576,7 +1576,7 @@ To recover from this fault, try the following action 1. Free up space from affected disk partition . 2. TAC may be required to analyze and cleanup certain directories due to filesystem permissions. Cleanup of `/` is one such example. -!!! example "Fault Example (F4390: " Critical fault for APIC Inode Utilisation) +!!! example "Fault Example (F4390: Critical fault for APIC Inode Utilisation)" ``` moquery -c faultInst -f 'fault.Inst.code=="F4390"' Total Objects shown: 1 @@ -1590,7 +1590,7 @@ To recover from this fault, try the following action code : F4390 created : 2024-08-05T05:42:31.975+02:00 delegated : no - descr : Storage unit /scratch-writes on node 3 with hostname b001nnc000003 mounted at /scratch-writes is 90% full for Inodes + descr : Storage unit /scratch-writes on node 3 with hostname apic3 mounted at /scratch-writes is 90% full for Inodes dn : topology/pod-2/node-3/sys/ch/p-[/scratch-writes]-f-[/dev/mapper/atx-scratch]/fault-F4390 domain : infra highestSeverity : critical From d30bdc2491852917412d76b8c8af4c836d8a8b5b Mon Sep 17 00:00:00 2001 From: tkishida Date: Fri, 3 Apr 2026 13:50:49 -0700 Subject: [PATCH 7/8] fix: Handle new fault code on older versions --- aci-preupgrade-validation-script.py | 8 +++++++- .../test_apic_storage_inode_full_check.py | 6 ++++++ tests/test_icurl.py | 14 ++++++++++++++ 3 files changed, 27 insertions(+), 1 deletion(-) diff --git a/aci-preupgrade-validation-script.py b/aci-preupgrade-validation-script.py index 4d259880..936abafc 100644 --- a/aci-preupgrade-validation-script.py +++ b/aci-preupgrade-validation-script.py @@ -1593,6 +1593,8 @@ def _icurl_error_handler(imdata): if imdata and "error" in imdata[0]: if "not found in class" in imdata[0]['error']['attributes']['text']: raise OldVerPropNotFound('Your current ACI version does not have requested property') + elif "Incorrect filter format for" in imdata[0]['error']['attributes']['text']: + raise OldVerPropNotFound('Your current ACI version does not have requested value for the property in the filter') elif "unresolved class for" in imdata[0]['error']['attributes']['text']: raise OldVerClassNotFound('Your current ACI version does not have requested class') elif "not found" in imdata[0]['error']['attributes']['text']: @@ -6203,7 +6205,11 @@ def apic_storage_inode_check(**kwargs): doc_url = 'https://datacenter.github.io/ACI-Pre-Upgrade-Validation-Script/validations/#apic-storage-inode-check' dn_regex = node_regex + r'/.+p-\[(?P.+)\]-f' desc_regex = r'is (?P\d{2,3}%) full for Inodes' - faultInsts = icurl('class', 'faultInst.json?query-target-filter=or(eq(faultInst.code,"F4388"),eq(faultInst.code,"F4389"),eq(faultInst.code,"F4390"))') + try: + faultInsts = icurl('class', 'faultInst.json?query-target-filter=or(eq(faultInst.code,"F4388"),eq(faultInst.code,"F4389"),eq(faultInst.code,"F4390"))') + except OldVerPropNotFound: + # Pre 5.2.6 does not have these fault codes. + return Result(result=NA, msg="cversion does not have fault code F4388, F4389 or F4390.", doc_url=doc_url) for faultInst in faultInsts: lc = faultInst['faultInst']['attributes']['lc'] if lc not in ["raised", "soaking"]: diff --git a/tests/checks/apic_storage_inode_full_check/test_apic_storage_inode_full_check.py b/tests/checks/apic_storage_inode_full_check/test_apic_storage_inode_full_check.py index e9ca9ff8..9800f7f8 100644 --- a/tests/checks/apic_storage_inode_full_check/test_apic_storage_inode_full_check.py +++ b/tests/checks/apic_storage_inode_full_check/test_apic_storage_inode_full_check.py @@ -18,6 +18,12 @@ @pytest.mark.parametrize( "icurl_outputs, expected_result, expected_data", [ + # NA - Fault code not yet supported in the current APIC version + ( + {faultInst_api: [{"error": {"attributes": {"code": "301", "text": "Incorrect filter format for faultInst.code, value 'F4388' is not valid"}}}]}, + script.NA, + [], + ), # PASS - No raised faults ( {faultInst_api: []}, diff --git a/tests/test_icurl.py b/tests/test_icurl.py index 351e3cf5..b70f253b 100644 --- a/tests/test_icurl.py +++ b/tests/test_icurl.py @@ -114,6 +114,20 @@ def test_icurl(mock_icurl, apitype, query, expected_result): ], script.OldVerPropNotFound, ), + # /api/class/faultInfo.json?query-target-filter=eq(faultInst.code,"F999999") + ( + [ + { + "error": { + "attributes": { + "code": "301", + "text": "Incorrect filter format for faultInst.code, value 'F999999' is not valid", + } + } + } + ], + script.OldVerPropNotFound, + ), # /api/class/faultInf.json?query-target-filter=eq(faultInst.code,"F2109") ( [ From f74c934482e58ab05a6dfdc3602ef2f29510e12d Mon Sep 17 00:00:00 2001 From: tkishida Date: Fri, 3 Apr 2026 14:49:02 -0700 Subject: [PATCH 8/8] fix: Remove "check" from the rule title --- aci-preupgrade-validation-script.py | 4 ++-- docs/docs/validations.md | 6 +++--- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/aci-preupgrade-validation-script.py b/aci-preupgrade-validation-script.py index 936abafc..7921ed61 100644 --- a/aci-preupgrade-validation-script.py +++ b/aci-preupgrade-validation-script.py @@ -6194,7 +6194,7 @@ def n9k_c9408_model_lem_count_check(tversion, fabric_nodes, **kwargs): return Result(result=result, headers=headers, data=data, recommended_action=recommended_action, doc_url=doc_url) -@check_wrapper(check_title="APIC Storage Inode Check (F4388, F4389, F4390 equipment-full)") +@check_wrapper(check_title="APIC Storage Inode Usage (F4388, F4389, F4390 equipment-full)") def apic_storage_inode_check(**kwargs): result = FAIL_UF headers = ['Fault', 'Pod', 'Node', 'Mount Point', 'Usage %'] @@ -6202,7 +6202,7 @@ def apic_storage_inode_check(**kwargs): unformatted_headers = ['Fault', 'Fault DN'] unformatted_data = [] recommended_action = 'Contact Cisco TAC to remove the files in the mount point to free up space and clear the fault' - doc_url = 'https://datacenter.github.io/ACI-Pre-Upgrade-Validation-Script/validations/#apic-storage-inode-check' + doc_url = 'https://datacenter.github.io/ACI-Pre-Upgrade-Validation-Script/validations/#apic-storage-inode-usage' dn_regex = node_regex + r'/.+p-\[(?P.+)\]-f' desc_regex = r'is (?P\d{2,3}%) full for Inodes' try: diff --git a/docs/docs/validations.md b/docs/docs/validations.md index f76d6d05..3859348d 100644 --- a/docs/docs/validations.md +++ b/docs/docs/validations.md @@ -82,7 +82,7 @@ Items | Faults | This Script [Fabric Port Status][f19] | F1394: ethpm-if-port-down-fabric | :white_check_mark: | :no_entry_sign: [Equipment Disk Limits][f20] | F1820: 80% -minor
F1821: -major
F1822: -critical | :white_check_mark: | :no_entry_sign: [VMM Inventory Partially Synced][f21] | F0132: comp-ctrlr-operational-issues | :white_check_mark: | :no_entry_sign: -[APIC Storage Inode Check][f22] | F4388: 75% - 85% -warning
F4389: 85% - 90% -major
F4390: 90% or more -critical | :white_check_mark: | :no_entry_sign: +[APIC Storage Inode Usage][f22] | F4388: 75% - 85% -warning
F4389: 85% - 90% -major
F4390: 90% or more -critical | :white_check_mark: | :no_entry_sign: [f1]: #apic-disk-space-usage [f2]: #standby-apic-disk-space-usage @@ -105,7 +105,7 @@ Items | Faults | This Script [f19]: #fabric-port-status [f20]: #equipment-disk-limits [f21]: #vmm-inventory-partially-synced -[f22]: #apic-storage-inode-check +[f22]: #apic-storage-inode-usage ### Configuration Checks @@ -1557,7 +1557,7 @@ EPGs using the `pre-provision` resolution immediacy do not rely on the VMM inven This check returns a `MANUAL` result as there are many reasons for a partial inventory sync to be reported. The goal is to ensure that the VMM inventory sync has fully completed before triggering the APIC upgrade to reduce any chance for unexpected inventory changes to occur. -### APIC Storage Inode Check +### APIC Storage Inode Usage If a Cisco APIC is running low on inode capacity for any reason, the Cisco APIC upgrade can fail. The Cisco APIC will raise three different faults depending on inode utilization. If any of these faults are raised on the system, the issue should be resolved prior to performing the upgrade.