Skip to content

Commit 0d28b80

Browse files
Harinadh-Saladimonrog2takishidapsureshbsanjanch
authored
Added code for CSCwo96334:/tmp directory disk space for snapshot storage (datacenter#368)
* Add arg to define thread limit - to throttle concurrent API calls when required (datacenter#355) * add `--max-threads` arg * fix bad descriptor errs/race conditions * update pytests * 353 the script incorrectly detects vpc and port channel interfaces as cscwh68103 invalid fabricpathep targets (datacenter#357) * specific testing for known failure conditions of cscwh68103 as to not catch valid scenarios * Update aci-preupgrade-validation-script.py mark version * print cleanup * Added code for CSCwo96334:/tmp directory disk space for snapshot storage * Updated fix version * Added clarification comment for /tmp snapshot handling * fix: Add cversion check for post_upgrade_cb_check (datacenter#377) * added validation for CFD CSCwp64296 (datacenter#307) Added rogue ep/coop exception mac check for the CFD CSCwp64296 * Added pre-upgrade validation for N9K-C9408 with more than 6 N9K-X9400-16W LEM's for the bug CSCws82819 (datacenter#354) * Added a new check for the bug 'CSCws82819N9K-C9408 boot loop on 16.1.2f and later with 6 or more LEMs' * New Validation for APIC Storage Inode Usage (F4388, F4389, F4390 equipment-full) (datacenter#361) * New Validation for APIC Storage Inode Usage (F4388, F4389, F4390 equipment-full) * Add new exception handling of invalid query filter in `icurl` due to, for example, a non-supported fault code on older versions * Add validation for multipod_modular_spine_bootscript_check - CSCwr66848 (datacenter#365) * Add check for CFD - CSCwr66848 * Update pytest.yml to run on vX.Y.Z branches * adding of cli parameters for user and password (datacenter#335) * added cli parameter for user (-u) and password (-p) to be used in an eased way for fully automated execution of 'aci-preupgrade-validation-script.py' * test: Add pytest --------- Co-authored-by: Detlef Sass <detlef.sass@de.bosch.com> * Corrected the typo for snapshot --------- Co-authored-by: GM <monrog2@gmail.com> Co-authored-by: Gabriel <gmonroy@cisco.com> Co-authored-by: tkishida <tkishida@cisco.com> Co-authored-by: takishida <38262981+takishida@users.noreply.github.com> Co-authored-by: psureshb <psureshb@cisco.com> Co-authored-by: sanjanch <sanjanch@cisco.com> Co-authored-by: asraf-khan <anazar@cisco.com> Co-authored-by: Thatleft <detlef.sass@web.de> Co-authored-by: Detlef Sass <detlef.sass@de.bosch.com>
1 parent 916ca44 commit 0d28b80

5 files changed

Lines changed: 329 additions & 11 deletions

File tree

‎aci-preupgrade-validation-script.py‎

Lines changed: 21 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -3010,17 +3010,19 @@ def scalability_faults_check(**kwargs):
30103010

30113011

30123012
@check_wrapper(check_title="APIC Disk Space Usage (F1527, F1528, F1529 equipment-full)")
3013-
def apic_disk_space_faults_check(cversion, **kwargs):
3013+
def apic_disk_space_faults_check(cversion, tversion, **kwargs):
30143014
result = FAIL_UF
30153015
headers = ['Fault', 'Pod', 'Node', 'Mount Point', 'Current Usage %', 'Recommended Action']
30163016
data = []
30173017
unformatted_headers = ['Fault', 'Fault DN', 'Recommended Action']
30183018
unformatted_data = []
30193019
doc_url = "https://datacenter.github.io/ACI-Pre-Upgrade-Validation-Script/validations/#apic-disk-space-usage"
3020+
# we are checking /tmp utilization because high usage can lead to snaphshot corruption during an upgrade. After the fix version, snapshot storage location moved to /data.
30203021
recommended_action = {
30213022
'/firmware': 'Remove unneeded images',
30223023
'/techsupport': 'Remove unneeded techsupports/cores',
3023-
'/data/log': 'Remove unneeded logs in var/log/dme/log'
3024+
'/data/log': 'Remove unneeded logs in var/log/dme/log',
3025+
'/tmp': 'Remove unneeded logs in /tmp directory'
30243026
}
30253027
default_action = 'Contact Cisco TAC.'
30263028
if cversion.same_as('4.0(1h)') or cversion.older_than('3.2(6i)'):
@@ -3029,6 +3031,8 @@ def apic_disk_space_faults_check(cversion, **kwargs):
30293031
dn_regex = node_regex + r'/.+p-\[(?P<mountpoint>.+)\]-f'
30303032
desc_regex = r'is (?P<usage>\d{2,3}%) full'
30313033

3034+
tmp_faults_skip_versions = ["6.0(9f)", "6.1(4h)", "6.2(1g)"]
3035+
tmp_faults_skipped = False # Track if we skip /tmp faults for CSCwo96334 versions
30323036
faultInsts = icurl('class',
30333037
'faultInst.json?query-target-filter=or(eq(faultInst.code,"F1527"),eq(faultInst.code,"F1528"),eq(faultInst.code,"F1529"))')
30343038
for faultInst in faultInsts:
@@ -3038,14 +3042,24 @@ def apic_disk_space_faults_check(cversion, **kwargs):
30383042
fc = faultInst['faultInst']['attributes']['code']
30393043
dn = re.search(dn_regex, faultInst['faultInst']['attributes']['dn'])
30403044
desc = re.search(desc_regex, faultInst['faultInst']['attributes']['descr'])
3041-
if dn and desc:
3042-
data.append([fc, dn.group('pod'), dn.group('node'), dn.group('mountpoint'),
3045+
if dn:
3046+
mountpoint = dn.group('mountpoint')
3047+
# CSCwo96334: Skip /tmp faults when target is >= 6.1(4h) or any unaffected versions
3048+
if mountpoint == '/tmp' and (not tversion.older_than("6.1(4h)") or any(tversion.same_as(version) for version in tmp_faults_skip_versions)):
3049+
tmp_faults_skipped = True
3050+
continue
3051+
if desc:
3052+
data.append([fc, dn.group('pod'), dn.group('node'), dn.group('mountpoint'),
30433053
desc.group('usage'),
30443054
recommended_action.get(dn.group('mountpoint'), default_action)])
3045-
else:
3046-
unformatted_data.append([fc, faultInst['faultInst']['attributes']['dn'], default_action])
3055+
else:
3056+
unformatted_data.append([fc, faultInst['faultInst']['attributes']['dn'], default_action])
30473057
if not data and not unformatted_data:
3048-
result = PASS
3058+
# If we only found /tmp faults that were skipped (CSCwo96334 fixed target versions), return NA
3059+
if tmp_faults_skipped:
3060+
result = NA
3061+
else:
3062+
result = PASS
30493063
return Result(
30503064
result=result,
30513065
headers=headers,

‎tests/checks/apic_disk_space_faults_check/Fault_combination.json‎

Lines changed: 90 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -88,5 +88,95 @@
8888
"type": "operational"
8989
}
9090
}
91+
},
92+
93+
{
94+
"faultInst": {
95+
"attributes": {
96+
"ack": "no",
97+
"alert": "no",
98+
"cause": "equipment-full",
99+
"changeSet": "inodesFree (Old: 12167881, New: 12167880), inodesUsed (Old: 64, New: 65)",
100+
"childAction": "",
101+
"code": "F1529",
102+
"created": "2026-03-13T11:46:02.307+00:00",
103+
"delegated": "no",
104+
"descr": "Storage unit /tmp on node 1 with hostname fab3-apic1 mounted at /tmp is 100% full",
105+
"dn": "topology/pod-1/node-1/sys/ch/p-[/tmp]-f-[tmpfs]/fault-F1529",
106+
"domain": "infra",
107+
"highestSeverity": "critical",
108+
"lastTransition": "2026-03-13T11:48:02.867+00:00",
109+
"lc": "raised",
110+
"occur": "1",
111+
"origSeverity": "critical",
112+
"prevSeverity": "critical",
113+
"rule": "eqpt-storage-full-critical",
114+
"severity": "critical",
115+
"status": "",
116+
"subject": "equipment-full",
117+
"title": "",
118+
"type": "operational"
119+
}
120+
}
121+
},
122+
123+
{
124+
"faultInst": {
125+
"attributes": {
126+
"ack": "no",
127+
"alert": "no",
128+
"cause": "equipment-full",
129+
"changeSet": "inodesFree (Old: 12167881, New: 12167880), inodesUsed (Old: 64, New: 65)",
130+
"childAction": "",
131+
"code": "F1528",
132+
"created": "2026-03-13T11:46:02.307+00:00",
133+
"delegated": "no",
134+
"descr": "Storage unit /tmp on node 1 with hostname fab3-apic1 mounted at /tmp is 89% full",
135+
"dn": "topology/pod-1/node-1/sys/ch/p-[/tmp]-f-[tmpfs]/fault-F1528",
136+
"domain": "infra",
137+
"highestSeverity": "major",
138+
"lastTransition": "2026-03-13T11:48:02.867+00:00",
139+
"lc": "raised",
140+
"occur": "1",
141+
"origSeverity": "major",
142+
"prevSeverity": "major",
143+
"rule": "eqpt-storage-full-major",
144+
"severity": "major",
145+
"status": "",
146+
"subject": "equipment-full",
147+
"title": "",
148+
"type": "operational"
149+
}
150+
}
151+
},
152+
153+
{
154+
"faultInst": {
155+
"attributes": {
156+
"ack": "no",
157+
"alert": "no",
158+
"cause": "equipment-full",
159+
"changeSet": "available (Old: 1501496, New: 240908), capUtilized (Old: 79, New: 82), inodesFree (Old: 12148991, New: 12148990), inodesUsed (Old: 721, New: 722), used (Old: 595656, New: 1856244)",
160+
"childAction": "",
161+
"code": "F1527",
162+
"created": "2026-03-13T11:46:02.307+00:00",
163+
"delegated": "no",
164+
"descr": "Storage unit /data/log on Node 1 with hostname fab3-apic1 mounted at /data/log is 82% full",
165+
"dn": "topology/pod-1/node-1/sys/ch/p-[/data/log]-f-[tmpfs]/fault-F1527",
166+
"domain": "infra",
167+
"highestSeverity": "warning",
168+
"lastTransition": "2026-03-13T11:48:02.867+00:00",
169+
"lc": "raised",
170+
"occur": "1",
171+
"origSeverity": "warning",
172+
"prevSeverity": "warning",
173+
"rule": "eqpt-storage-full-warning",
174+
"severity": "warning",
175+
"status": "",
176+
"subject": "equipment-full",
177+
"title": "",
178+
"type": "operational"
179+
}
180+
}
91181
}
92182
]

‎tests/checks/apic_disk_space_faults_check/Fault_exists_not_raised.json‎

Lines changed: 60 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -87,5 +87,65 @@
8787
"type": "operational"
8888
}
8989
}
90+
},
91+
92+
{
93+
"faultInst": {
94+
"attributes": {
95+
"ack": "no",
96+
"alert": "no",
97+
"cause": "equipment-full",
98+
"changeSet": "inodesFree (Old: 12167881, New: 12167880), inodesUsed (Old: 64, New: 65)",
99+
"childAction": "",
100+
"code": "F1529",
101+
"created": "2026-03-13T11:46:02.307+00:00",
102+
"delegated": "no",
103+
"descr": "Storage unit /tmp on node 1 with hostname fab3-apic1 mounted at /tmp is 100% full",
104+
"dn": "topology/pod-1/node-1/sys/ch/p-[/tmp]-f-[tmpfs]/fault-F1529",
105+
"domain": "infra",
106+
"highestSeverity": "critical",
107+
"lastTransition": "2026-03-13T11:48:02.867+00:00",
108+
"lc": "retaining",
109+
"occur": "1",
110+
"origSeverity": "critical",
111+
"prevSeverity": "critical",
112+
"rule": "eqpt-storage-full-critical",
113+
"severity": "cleared",
114+
"status": "",
115+
"subject": "equipment-full",
116+
"title": "",
117+
"type": "operational"
118+
}
119+
}
120+
},
121+
122+
{
123+
"faultInst": {
124+
"attributes": {
125+
"ack": "no",
126+
"alert": "no",
127+
"cause": "equipment-full",
128+
"changeSet": "inodesFree (Old: 12167881, New: 12167880), inodesUsed (Old: 64, New: 65)",
129+
"childAction": "",
130+
"code": "F1528",
131+
"created": "2026-03-13T11:46:02.307+00:00",
132+
"delegated": "no",
133+
"descr": "Storage unit /tmp on node 1 with hostname fab3-apic1 mounted at /tmp is 89% full",
134+
"dn": "topology/pod-1/node-1/sys/ch/p-[/tmp]-f-[tmpfs]/fault-F1528",
135+
"domain": "infra",
136+
"highestSeverity": "major",
137+
"lastTransition": "2026-03-13T11:48:02.867+00:00",
138+
"lc": "soaking-clearing",
139+
"occur": "1",
140+
"origSeverity": "major",
141+
"prevSeverity": "major",
142+
"rule": "eqpt-storage-full-major",
143+
"severity": "major",
144+
"status": "",
145+
"subject": "equipment-full",
146+
"title": "",
147+
"type": "operational"
148+
}
149+
}
90150
}
91151
]

‎tests/checks/apic_disk_space_faults_check/Fault_raised.json‎

Lines changed: 90 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -87,5 +87,95 @@
8787
"type": "operational"
8888
}
8989
}
90+
},
91+
92+
{
93+
"faultInst": {
94+
"attributes": {
95+
"ack": "no",
96+
"alert": "no",
97+
"cause": "equipment-full",
98+
"changeSet": "inodesFree (Old: 12167881, New: 12167880), inodesUsed (Old: 64, New: 65)",
99+
"childAction": "",
100+
"code": "F1529",
101+
"created": "2026-03-13T11:46:02.307+00:00",
102+
"delegated": "no",
103+
"descr": "Storage unit /tmp on node 1 with hostname fab3-apic1 mounted at /tmp is 100% full",
104+
"dn": "topology/pod-1/node-1/sys/ch/p-[/tmp]-f-[tmpfs]/fault-F1529",
105+
"domain": "infra",
106+
"highestSeverity": "critical",
107+
"lastTransition": "2026-03-13T11:48:02.867+00:00",
108+
"lc": "raised",
109+
"occur": "1",
110+
"origSeverity": "critical",
111+
"prevSeverity": "critical",
112+
"rule": "eqpt-storage-full-critical",
113+
"severity": "critical",
114+
"status": "",
115+
"subject": "equipment-full",
116+
"title": "",
117+
"type": "operational"
118+
}
119+
}
120+
},
121+
122+
{
123+
"faultInst": {
124+
"attributes": {
125+
"ack": "no",
126+
"alert": "no",
127+
"cause": "equipment-full",
128+
"changeSet": "inodesFree (Old: 12167881, New: 12167880), inodesUsed (Old: 64, New: 65)",
129+
"childAction": "",
130+
"code": "F1528",
131+
"created": "2026-03-13T11:46:02.307+00:00",
132+
"delegated": "no",
133+
"descr": "Storage unit /tmp on node 1 with hostname fab3-apic1 mounted at /tmp is 89% full",
134+
"dn": "topology/pod-1/node-1/sys/ch/p-[/tmp]-f-[tmpfs]/fault-F1528",
135+
"domain": "infra",
136+
"highestSeverity": "major",
137+
"lastTransition": "2026-03-13T11:48:02.867+00:00",
138+
"lc": "raised",
139+
"occur": "1",
140+
"origSeverity": "major",
141+
"prevSeverity": "major",
142+
"rule": "eqpt-storage-full-major",
143+
"severity": "major",
144+
"status": "",
145+
"subject": "equipment-full",
146+
"title": "",
147+
"type": "operational"
148+
}
149+
}
150+
},
151+
152+
{
153+
"faultInst": {
154+
"attributes": {
155+
"ack": "no",
156+
"alert": "no",
157+
"cause": "equipment-full",
158+
"changeSet": "inodesFree (Old: 12167881, New: 12167880), inodesUsed (Old: 64, New: 65)",
159+
"childAction": "",
160+
"code": "F1527",
161+
"created": "2026-03-13T11:46:02.307+00:00",
162+
"delegated": "no",
163+
"descr": "Storage unit /tmp on node 1 with hostname fab3-apic1 mounted at /tmp is 82% full",
164+
"dn": "topology/pod-1/node-1/sys/ch/p-[/tmp]-f-[tmpfs]/fault-F1527",
165+
"domain": "infra",
166+
"highestSeverity": "warning",
167+
"lastTransition": "2026-03-13T11:48:02.867+00:00",
168+
"lc": "raised",
169+
"occur": "1",
170+
"origSeverity": "warning",
171+
"prevSeverity": "warning",
172+
"rule": "eqpt-storage-full-warning",
173+
"severity": "warning",
174+
"status": "",
175+
"subject": "equipment-full",
176+
"title": "",
177+
"type": "operational"
178+
}
179+
}
90180
}
91181
]

0 commit comments

Comments
 (0)