11from collections import Counter
2- import json
32import logging
3+ import re
44import os
5+ import sys
56import os .path
67import shlex
78import shutil
@@ -25,7 +26,7 @@ def _find_sonobuoy():
2526
2627
2728def _fmt_result (counter ):
28- return ', ' .join (f"{ counter .get (key , 0 )} { key } for key in ('passed' , 'failed' , 'skipped' ))
29+ return ', ' .join (f"{ counter .get (key , 0 )} { key } for key in ('passed' , 'failed' , 'failed_ok' , ' skipped'
2930
3031
3132class SonobuoyHandler :
@@ -42,6 +43,7 @@ def __init__(
4243 check_name = "sonobuoy_handler" ,
4344 kubeconfig = None ,
4445 result_dir_name = "sonobuoy_results" ,
46+ scs_sonobuoy_config_yaml = "kaas/sonobuoy-config.yaml" ,
4547 args = (),
4648 ):
4749 self .check_name = check_name
@@ -55,6 +57,9 @@ def __init__(
5557 logger .debug (f"working from { self .working_directory } )
5658 logger .debug (f"placing results at { self .result_dir_name } )
5759 logger .debug (f"sonobuoy executable at { self .sonobuoy } )
60+ if not os .path .exists (scs_sonobuoy_config_yaml ):
61+ raise RuntimeError (f"scs_sonobuoy_config_yaml { scs_sonobuoy_config_yaml } )
62+ self .scs_sonobuoy_config_yaml = scs_sonobuoy_config_yaml
5863 self .args = (arg0 for arg in args for arg0 in shlex .split (str (arg )))
5964
6065 def _invoke_sonobuoy (self , * args , ** kwargs ):
@@ -69,16 +74,6 @@ def _sonobuoy_run(self):
6974 def _sonobuoy_delete (self ):
7075 self ._invoke_sonobuoy ("delete" , "--wait" )
7176
72- def _sonobuoy_status_result (self ):
73- process = self ._invoke_sonobuoy ("status" , "--json" , capture_output = True )
74- json_data = json .loads (process .stdout )
75- counter = Counter ()
76- for entry in json_data ["plugins" ]:
77- logger .debug (f"plugin { entry ['plugin' ]} { _fmt_result (entry ['result-counts' ])} )
78- for key , value in entry ["result-counts" ].items ():
79- counter [key ] += value
80- return counter
81-
8277 def _eval_result (self , counter ):
8378 """evaluate test results and return return code"""
8479 result_message = f"sonobuoy reports { _fmt_result (counter )}
@@ -99,7 +94,7 @@ def _sonobuoy_retrieve_result(self, plugin='e2e'):
9994 """
10095 Invoke sonobuoy to retrieve results and to store them in a subdirectory of
10196 the working directory. Analyze the results yaml file for given `plugin` and
102- log each failure as ERROR. Return summary dict like `_sonobuoy_status_result` .
97+ log each failure as ERROR. Return summary dict.
10398 """
10499 logger .debug (f"retrieving results to { self .result_dir_name } )
105100 result_dir = os .path .join (self .working_directory , self .result_dir_name )
@@ -108,21 +103,9 @@ def _sonobuoy_retrieve_result(self, plugin='e2e'):
108103 self ._invoke_sonobuoy ("retrieve" , "-x" , result_dir )
109104 yaml_path = os .path .join (result_dir , 'plugins' , plugin , 'sonobuoy_results.yaml' )
110105 logger .debug (f"parsing results from { yaml_path } )
111- with open (yaml_path , "r" ) as fileobj :
112- result_obj = yaml .load (fileobj .read (), yaml .SafeLoader )
113- counter = Counter ()
114- for item1 in result_obj .get ('items' , ()):
115- # file ...
116- for item2 in item1 .get ('items' , ()):
117- # suite ...
118- for item in item2 .get ('items' , ()):
119- # testcase ... or so
120- status = item .get ('status' , 'skipped' )
121- counter [status ] += 1
122- if status == 'failed' :
123- logger .error (f"FAILED: { item ['name' ]} ) # <-- this is why this method exists!
124- logger .info (f"{ plugin } { _fmt_result (counter )} )
125- return counter
106+ ok_to_fail_regex_list = _load_ok_to_fail_regex_list (self .scs_sonobuoy_config_yaml )
107+
108+ return sonobuoy_parse_result (plugin , yaml_path , ok_to_fail_regex_list )
126109
127110 def run (self ):
128111 """
@@ -132,16 +115,91 @@ def run(self):
132115 self ._preflight_check ()
133116 try :
134117 self ._sonobuoy_run ()
135- return_code = self ._eval_result (self ._sonobuoy_status_result ())
118+ counter = self ._sonobuoy_retrieve_result ()
119+ return_code = self ._eval_result (counter )
136120 print (self .check_name + ": " + ("PASS" , "FAIL" )[min (1 , return_code )])
137- try :
138- self ._sonobuoy_retrieve_result ()
139- except Exception :
140- # swallow exception for the time being
141- logger .debug ('problem retrieving results' , exc_info = True )
142121 return return_code
143122 except BaseException :
144123 logger .exception ("something went wrong" )
145124 return 112
146125 finally :
147126 self ._sonobuoy_delete ()
127+
128+
129+ def sonobuoy_parse_result (plugin , sonobuoy_results_yaml_path , ok_to_fail_regex_list ):
130+ with open (sonobuoy_results_yaml_path , "r" ) as fileobj :
131+ result_obj = yaml .load (fileobj .read (), yaml .SafeLoader )
132+
133+ counter = Counter ()
134+ for item1 in result_obj .get ("items" , ()):
135+ # file ...
136+ for item2 in item1 .get ("items" , ()):
137+ # suite ...
138+ for item in item2 .get ("items" , ()):
139+ # testcase ... or so
140+ status = item .get ("status" , "skipped" )
141+ if status == "failed" :
142+ if ok_to_fail (ok_to_fail_regex_list , item ["name" ]):
143+ status = "failed_ok"
144+ else :
145+ logger .error (f"FAILED: { item ['name' ]} )
146+ counter [status ] += 1
147+
148+ logger .info (f"{ plugin } { _fmt_result (counter )} )
149+ return counter
150+
151+
152+ def _load_ok_to_fail_regex_list (scs_sonobuoy_config_yaml ):
153+ with open (scs_sonobuoy_config_yaml , "r" ) as fileobj :
154+ config_obj = yaml .load (fileobj .read (), yaml .SafeLoader ) or {}
155+ if not isinstance (config_obj , dict ):
156+ raise ValueError (f"Invalid sonobuoy config format in { scs_sonobuoy_config_yaml } )
157+ allowed_top_level_keys = {"okToFail" }
158+ unknown_top_level_keys = set (config_obj ) - allowed_top_level_keys
159+ if unknown_top_level_keys :
160+ raise ValueError (
161+ f"Invalid sonobuoy config format in { scs_sonobuoy_config_yaml } { sorted (unknown_top_level_keys )}
162+ )
163+ ok_to_fail_items = config_obj .get ("okToFail" , ())
164+ if not isinstance (ok_to_fail_items , list ):
165+ raise ValueError (f"Invalid sonobuoy config format in { scs_sonobuoy_config_yaml } )
166+
167+ ok_to_fail_regex_list = []
168+ for idx , entry in enumerate (ok_to_fail_items ):
169+ if not isinstance (entry , dict ):
170+ raise ValueError (
171+ f"Invalid sonobuoy config format in { scs_sonobuoy_config_yaml } { idx }
172+ )
173+ allowed_entry_keys = {"regex" , "reason" }
174+ unknown_entry_keys = set (entry ) - allowed_entry_keys
175+ if unknown_entry_keys :
176+ raise ValueError (
177+ f"Invalid sonobuoy config format in { scs_sonobuoy_config_yaml } { idx } { sorted (unknown_entry_keys )}
178+ )
179+ regex = entry .get ("regex" )
180+ if not isinstance (regex , str ) or not regex .strip ():
181+ raise ValueError (
182+ f"Invalid sonobuoy config format in { scs_sonobuoy_config_yaml } { idx }
183+ )
184+ reason = entry .get ("reason" )
185+ if not isinstance (reason , str ) or not reason .strip ():
186+ raise ValueError (
187+ f"Invalid sonobuoy config format in { scs_sonobuoy_config_yaml } { idx }
188+ )
189+ ok_to_fail_regex_list .append ((re .compile (regex ), reason ))
190+ return ok_to_fail_regex_list
191+
192+
193+ def ok_to_fail (ok_to_fail_regex_list , test_name ):
194+ name = test_name
195+ for regex , _ in ok_to_fail_regex_list :
196+ if re .search (regex , name ):
197+ return True
198+ return False
199+
200+
201+ def check_sonobuoy_result (scs_sonobuoy_config_yaml , result_yaml ):
202+ ok_to_fail_regex_list = _load_ok_to_fail_regex_list (scs_sonobuoy_config_yaml )
203+ counter = sonobuoy_parse_result ("" , result_yaml , ok_to_fail_regex_list )
204+ for key , value in counter .items ():
205+ print (f"{ key } { value } )
0 commit comments