1717
1818import argparse
1919import json
20+ import logging
2021import matplotlib .pyplot as plt
2122import numpy as np
2223
24+ logging .basicConfig (level = logging .INFO )
25+ logger = logging .getLogger (__name__ )
26+
2327def geomean (data ):
2428 return np .prod (data ) ** (1 / len (data ))
2529
26- def generate_query_rel_speedup_chart (baseline , comparison , label1 : str , label2 : str , benchmark : str , title : str ):
30+ def get_durations (result , query_key ):
31+ """Extract durations from a query result, supporting both old (list) and new (dict) formats."""
32+ value = result [query_key ]
33+ if isinstance (value , dict ):
34+ return value ["durations" ]
35+ return value
36+
37+ def get_all_queries (results ):
38+ """Return the sorted union of all query keys across all result sets."""
39+ all_keys = set ()
40+ for result in results :
41+ all_keys .update (result .keys ())
42+ # Filter to numeric query keys and sort numerically
43+ numeric_keys = []
44+ for k in all_keys :
45+ try :
46+ numeric_keys .append (int (k ))
47+ except ValueError :
48+ pass
49+ return sorted (numeric_keys )
50+
51+ def get_common_queries (results , labels ):
52+ """Return queries present in ALL result sets, warning about queries missing from some files."""
53+ all_queries = get_all_queries (results )
54+ common = []
55+ for query in all_queries :
56+ key = str (query )
57+ present = [labels [i ] for i , r in enumerate (results ) if key in r ]
58+ missing = [labels [i ] for i , r in enumerate (results ) if key not in r ]
59+ if missing :
60+ logger .warning (f"Query { query } { ', ' .join (present )} { ', ' .join (missing )} )
61+ if not missing :
62+ common .append (query )
63+ return common
64+
65+ def check_result_consistency (results , labels , benchmark ):
66+ """Log warnings if row counts or result hashes differ across result sets."""
67+ all_queries = get_all_queries (results )
68+ for query in all_queries :
69+ key = str (query )
70+ row_counts = []
71+ hashes = []
72+ for i , result in enumerate (results ):
73+ if key not in result :
74+ continue
75+ value = result [key ]
76+ if not isinstance (value , dict ):
77+ continue
78+ if "row_count" in value :
79+ row_counts .append ((labels [i ], value ["row_count" ]))
80+ if "result_hash" in value :
81+ hashes .append ((labels [i ], value ["result_hash" ]))
82+
83+ if len (row_counts ) > 1 :
84+ counts = set (rc for _ , rc in row_counts )
85+ if len (counts ) > 1 :
86+ details = ", " .join (f"{ label } { rc } for label , rc in row_counts )
87+ logger .warning (f"Query { query } { details } )
88+
89+ if len (hashes ) > 1 :
90+ hash_values = set (h for _ , h in hashes )
91+ if len (hash_values ) > 1 :
92+ details = ", " .join (f"{ label } { h } for label , h in hashes )
93+ logger .warning (f"Query { query } { details } )
94+
95+ def generate_query_rel_speedup_chart (baseline , comparison , label1 : str , label2 : str , benchmark : str , title : str , common_queries = None ):
96+ if common_queries is None :
97+ common_queries = range (1 , query_count (benchmark )+ 1 )
2798 results = []
28- for query in range (1 , query_count (benchmark )+ 1 ):
29- if query == 999 :
30- continue
31- a = np .median (np .array (baseline [str (query )]))
32- b = np .median (np .array (comparison [str (query )]))
99+ for query in common_queries :
100+ a = np .median (np .array (get_durations (baseline , str (query ))))
101+ b = np .median (np .array (get_durations (comparison , str (query ))))
33102 if a > b :
34103 speedup = a / b - 1
35104 else :
@@ -80,13 +149,13 @@ def generate_query_rel_speedup_chart(baseline, comparison, label1: str, label2:
80149 # Save the plot as an image file
81150 plt .savefig (f'{ benchmark } , format = 'png' )
82151
83- def generate_query_abs_speedup_chart (baseline , comparison , label1 : str , label2 : str , benchmark : str , title : str ):
152+ def generate_query_abs_speedup_chart (baseline , comparison , label1 : str , label2 : str , benchmark : str , title : str , common_queries = None ):
153+ if common_queries is None :
154+ common_queries = range (1 , query_count (benchmark )+ 1 )
84155 results = []
85- for query in range (1 , query_count (benchmark )+ 1 ):
86- if query == 999 :
87- continue
88- a = np .median (np .array (baseline [str (query )]))
89- b = np .median (np .array (comparison [str (query )]))
156+ for query in common_queries :
157+ a = np .median (np .array (get_durations (baseline , str (query ))))
158+ b = np .median (np .array (get_durations (comparison , str (query ))))
90159 speedup = a - b
91160 results .append (("q" + str (query ), round (speedup , 1 )))
92161
@@ -130,17 +199,17 @@ def generate_query_abs_speedup_chart(baseline, comparison, label1: str, label2:
130199 # Save the plot as an image file
131200 plt .savefig (f'{ benchmark } , format = 'png' )
132201
133- def generate_query_comparison_chart (results , labels , benchmark : str , title : str ):
202+ def generate_query_comparison_chart (results , labels , benchmark : str , title : str , common_queries = None ):
203+ if common_queries is None :
204+ common_queries = range (1 , query_count (benchmark )+ 1 )
134205 queries = []
135206 benches = []
136207 for _ in results :
137208 benches .append ([])
138- for query in range (1 , query_count (benchmark )+ 1 ):
139- if query == 999 :
140- continue
209+ for query in common_queries :
141210 queries .append ("q" + str (query ))
142211 for i in range (0 , len (results )):
143- benches [i ].append (np .median (np .array (results [i ][ str (query )] )))
212+ benches [i ].append (np .median (np .array (get_durations ( results [i ], str (query )) )))
144213
145214 # Define the width of the bars
146215 bar_width = 0.3
@@ -168,25 +237,25 @@ def generate_query_comparison_chart(results, labels, benchmark: str, title: str)
168237 # Save the plot as an image file
169238 plt .savefig (f'{ benchmark } , format = 'png' )
170239
171- def generate_summary (results , labels , benchmark : str , title : str ):
240+ def generate_summary (results , labels , benchmark : str , title : str , common_queries = None ):
241+ if common_queries is None :
242+ common_queries = range (1 , query_count (benchmark )+ 1 )
172243 timings = []
173244 for _ in results :
174245 timings .append (0 )
175246
176- num_queries = query_count (benchmark )
177- for query in range (1 , num_queries + 1 ):
178- if query == 999 :
179- continue
247+ num_queries = len ([q for q in common_queries ])
248+ for query in common_queries :
180249 for i in range (0 , len (results )):
181- timings [i ] += np .median (np .array (results [i ][ str (query )] ))
250+ timings [i ] += np .median (np .array (get_durations ( results [i ], str (query )) ))
182251
183252 # Create figure and axis
184253 fig , ax = plt .subplots ()
185254 fig .set_size_inches (10 , 6 )
186255
187256 # Add title and labels
188257 ax .set_title (title )
189- ax .set_ylabel (f'Time in seconds to run all { num_queries } { benchmark } )
258+ ax .set_ylabel (f'Time in seconds to run { num_queries } { benchmark } )
190259
191260 times = [round (x ,0 ) for x in timings ]
192261
@@ -213,11 +282,16 @@ def main(files, labels, benchmark: str, title: str):
213282 for filename in files :
214283 with open (filename ) as f :
215284 results .append (json .load (f ))
216- generate_summary (results , labels , benchmark , title )
217- generate_query_comparison_chart (results , labels , benchmark , title )
285+ check_result_consistency (results , labels , benchmark )
286+ common_queries = get_common_queries (results , labels )
287+ if not common_queries :
288+ logger .error ("No queries found in common across all result files" )
289+ return
290+ generate_summary (results , labels , benchmark , title , common_queries )
291+ generate_query_comparison_chart (results , labels , benchmark , title , common_queries )
218292 if len (files ) == 2 :
219- generate_query_abs_speedup_chart (results [0 ], results [1 ], labels [0 ], labels [1 ], benchmark , title )
220- generate_query_rel_speedup_chart (results [0 ], results [1 ], labels [0 ], labels [1 ], benchmark , title )
293+ generate_query_abs_speedup_chart (results [0 ], results [1 ], labels [0 ], labels [1 ], benchmark , title , common_queries )
294+ generate_query_rel_speedup_chart (results [0 ], results [1 ], labels [0 ], labels [1 ], benchmark , title , common_queries )
221295
222296if __name__ == '__main__' :
223297 argparse = argparse .ArgumentParser (description = 'Generate comparison' )
0 commit comments