11from collections import Counter
2- import json
32import logging
3+ import re
44import os
55import os .path
66import shlex
@@ -25,7 +25,7 @@ def _find_sonobuoy():
2525
2626
2727def _fmt_result (counter ):
28- return ', ' .join (f"{ counter .get (key , 0 )} { key } for key in ('passed' , 'failed' , 'skipped' ))
28+ return ', ' .join (f"{ counter .get (key , 0 )} { key } for key in ('passed' , 'failed' , 'failed_ok' , ' skipped'
2929
3030
3131class SonobuoyHandler :
@@ -42,6 +42,7 @@ def __init__(
4242 check_name = "sonobuoy_handler" ,
4343 kubeconfig = None ,
4444 result_dir_name = "sonobuoy_results" ,
45+ scs_sonobuoy_config_yaml = "kaas/sonobuoy-config.yaml" ,
4546 args = (),
4647 ):
4748 self .check_name = check_name
@@ -55,6 +56,9 @@ def __init__(
5556 logger .debug (f"working from { self .working_directory } )
5657 logger .debug (f"placing results at { self .result_dir_name } )
5758 logger .debug (f"sonobuoy executable at { self .sonobuoy } )
59+ if not os .path .exists (scs_sonobuoy_config_yaml ):
60+ raise RuntimeError (f"scs_sonobuoy_config_yaml { scs_sonobuoy_config_yaml } )
61+ self .scs_sonobuoy_config_yaml = scs_sonobuoy_config_yaml
5862 self .args = (arg0 for arg in args for arg0 in shlex .split (str (arg )))
5963
6064 def _invoke_sonobuoy (self , * args , ** kwargs ):
@@ -69,16 +73,6 @@ def _sonobuoy_run(self):
6973 def _sonobuoy_delete (self ):
7074 self ._invoke_sonobuoy ("delete" , "--wait" )
7175
72- def _sonobuoy_status_result (self ):
73- process = self ._invoke_sonobuoy ("status" , "--json" , capture_output = True )
74- json_data = json .loads (process .stdout )
75- counter = Counter ()
76- for entry in json_data ["plugins" ]:
77- logger .debug (f"plugin { entry ['plugin' ]} { _fmt_result (entry ['result-counts' ])} )
78- for key , value in entry ["result-counts" ].items ():
79- counter [key ] += value
80- return counter
81-
8276 def _eval_result (self , counter ):
8377 """evaluate test results and return return code"""
8478 result_message = f"sonobuoy reports { _fmt_result (counter )}
@@ -99,7 +93,7 @@ def _sonobuoy_retrieve_result(self, plugin='e2e'):
9993 """
10094 Invoke sonobuoy to retrieve results and to store them in a subdirectory of
10195 the working directory. Analyze the results yaml file for given `plugin` and
102- log each failure as ERROR. Return summary dict like `_sonobuoy_status_result` .
96+ log each failure as ERROR. Return summary dict.
10397 """
10498 logger .debug (f"retrieving results to { self .result_dir_name } )
10599 result_dir = os .path .join (self .working_directory , self .result_dir_name )
@@ -108,21 +102,9 @@ def _sonobuoy_retrieve_result(self, plugin='e2e'):
108102 self ._invoke_sonobuoy ("retrieve" , "-x" , result_dir )
109103 yaml_path = os .path .join (result_dir , 'plugins' , plugin , 'sonobuoy_results.yaml' )
110104 logger .debug (f"parsing results from { yaml_path } )
111- with open (yaml_path , "r" ) as fileobj :
112- result_obj = yaml .load (fileobj .read (), yaml .SafeLoader )
113- counter = Counter ()
114- for item1 in result_obj .get ('items' , ()):
115- # file ...
116- for item2 in item1 .get ('items' , ()):
117- # suite ...
118- for item in item2 .get ('items' , ()):
119- # testcase ... or so
120- status = item .get ('status' , 'skipped' )
121- counter [status ] += 1
122- if status == 'failed' :
123- logger .error (f"FAILED: { item ['name' ]} ) # <-- this is why this method exists!
124- logger .info (f"{ plugin } { _fmt_result (counter )} )
125- return counter
105+ ok_to_fail_regex_list = _load_ok_to_fail_regex_list (self .scs_sonobuoy_config_yaml )
106+
107+ return sonobuoy_parse_result (plugin , yaml_path , ok_to_fail_regex_list )
126108
127109 def run (self ):
128110 """
@@ -132,16 +114,91 @@ def run(self):
132114 self ._preflight_check ()
133115 try :
134116 self ._sonobuoy_run ()
135- return_code = self ._eval_result (self ._sonobuoy_status_result ())
117+ counter = self ._sonobuoy_retrieve_result ()
118+ return_code = self ._eval_result (counter )
136119 print (self .check_name + ": " + ("PASS" , "FAIL" )[min (1 , return_code )])
137- try :
138- self ._sonobuoy_retrieve_result ()
139- except Exception :
140- # swallow exception for the time being
141- logger .debug ('problem retrieving results' , exc_info = True )
142120 return return_code
143121 except BaseException :
144122 logger .exception ("something went wrong" )
145123 return 112
146124 finally :
147125 self ._sonobuoy_delete ()
126+
127+
128+ def sonobuoy_parse_result (plugin , sonobuoy_results_yaml_path , ok_to_fail_regex_list ):
129+ with open (sonobuoy_results_yaml_path , "r" ) as fileobj :
130+ result_obj = yaml .load (fileobj .read (), yaml .SafeLoader )
131+
132+ counter = Counter ()
133+ for item1 in result_obj .get ("items" , ()):
134+ # file ...
135+ for item2 in item1 .get ("items" , ()):
136+ # suite ...
137+ for item in item2 .get ("items" , ()):
138+ # testcase ... or so
139+ status = item .get ("status" , "skipped" )
140+ if status == "failed" :
141+ if ok_to_fail (ok_to_fail_regex_list , item ["name" ]):
142+ status = "failed_ok"
143+ else :
144+ logger .error (f"FAILED: { item ['name' ]} )
145+ counter [status ] += 1
146+
147+ logger .info (f"{ plugin } { _fmt_result (counter )} )
148+ return counter
149+
150+
151+ def _load_ok_to_fail_regex_list (scs_sonobuoy_config_yaml ):
152+ with open (scs_sonobuoy_config_yaml , "r" ) as fileobj :
153+ config_obj = yaml .load (fileobj .read (), yaml .SafeLoader ) or {}
154+ if not isinstance (config_obj , dict ):
155+ raise ValueError (f"Invalid sonobuoy config format in { scs_sonobuoy_config_yaml } )
156+ allowed_top_level_keys = {"okToFail" }
157+ unknown_top_level_keys = set (config_obj ) - allowed_top_level_keys
158+ if unknown_top_level_keys :
159+ raise ValueError (
160+ f"Invalid sonobuoy config format in { scs_sonobuoy_config_yaml } { sorted (unknown_top_level_keys )}
161+ )
162+ ok_to_fail_items = config_obj .get ("okToFail" , ())
163+ if not isinstance (ok_to_fail_items , list ):
164+ raise ValueError (f"Invalid sonobuoy config format in { scs_sonobuoy_config_yaml } )
165+
166+ ok_to_fail_regex_list = []
167+ for idx , entry in enumerate (ok_to_fail_items ):
168+ if not isinstance (entry , dict ):
169+ raise ValueError (
170+ f"Invalid sonobuoy config format in { scs_sonobuoy_config_yaml } { idx }
171+ )
172+ allowed_entry_keys = {"regex" , "reason" }
173+ unknown_entry_keys = set (entry ) - allowed_entry_keys
174+ if unknown_entry_keys :
175+ raise ValueError (
176+ f"Invalid sonobuoy config format in { scs_sonobuoy_config_yaml } { idx } { sorted (unknown_entry_keys )}
177+ )
178+ regex = entry .get ("regex" )
179+ if not isinstance (regex , str ) or not regex .strip ():
180+ raise ValueError (
181+ f"Invalid sonobuoy config format in { scs_sonobuoy_config_yaml } { idx }
182+ )
183+ reason = entry .get ("reason" )
184+ if not isinstance (reason , str ) or not reason .strip ():
185+ raise ValueError (
186+ f"Invalid sonobuoy config format in { scs_sonobuoy_config_yaml } { idx }
187+ )
188+ ok_to_fail_regex_list .append ((re .compile (regex ), reason ))
189+ return ok_to_fail_regex_list
190+
191+
192+ def ok_to_fail (ok_to_fail_regex_list , test_name ):
193+ name = test_name
194+ for regex , _ in ok_to_fail_regex_list :
195+ if re .search (regex , name ):
196+ return True
197+ return False
198+
199+
200+ def check_sonobuoy_result (scs_sonobuoy_config_yaml , result_yaml ):
201+ ok_to_fail_regex_list = _load_ok_to_fail_regex_list (scs_sonobuoy_config_yaml )
202+ counter = sonobuoy_parse_result ("" , result_yaml , ok_to_fail_regex_list )
203+ for key , value in counter .items ():
204+ print (f"{ key } { value } )
0 commit comments