|
| 1 | +import os |
| 2 | +import re |
| 3 | + |
| 4 | +available_benchmarks = [ |
| 5 | + "benchmark-aiohttp", |
| 6 | + "benchmark-aiozmq", |
| 7 | + "benchmark-blacksheep", |
| 8 | + "benchmark-fastapi", |
| 9 | + "benchmark-sanic", |
| 10 | + "benchmark-zero", |
| 11 | +] |
| 12 | + |
| 13 | + |
| 14 | +def get_latest_history(): |
| 15 | + # file format is `<available_benchmark>__<date>.log` |
| 16 | + history_dir = "dockerize/history" |
| 17 | + if not os.path.exists(history_dir): |
| 18 | + return [] |
| 19 | + files = os.listdir(history_dir) |
| 20 | + files = [f for f in files if f.endswith(".log")] |
| 21 | + |
| 22 | + latest_files = {} |
| 23 | + for f in files: |
| 24 | + benchmark_name = f.split("__")[0] |
| 25 | + if benchmark_name in available_benchmarks: |
| 26 | + date_str = f.split("__")[1].replace(".log", "") |
| 27 | + if ( |
| 28 | + benchmark_name not in latest_files |
| 29 | + or date_str > latest_files[benchmark_name] |
| 30 | + ): |
| 31 | + latest_files[benchmark_name] = date_str |
| 32 | + |
| 33 | + return [f"{benchmark}__{date}.log" for benchmark, date in latest_files.items()] |
| 34 | + |
| 35 | + |
| 36 | +def parse_benchmark_log(filepath): |
| 37 | + results = [] |
| 38 | + current_test = None |
| 39 | + latency_99 = None |
| 40 | + requests_sec = None |
| 41 | + |
| 42 | + # Patterns to identify test type and metrics |
| 43 | + test_type_pattern = re.compile(r"Running 30s test @ http://gateway:8000/(\w+_?\w*)") |
| 44 | + latency_pattern = re.compile(r"99%\s+([\d.]+)ms") |
| 45 | + req_sec_pattern = re.compile(r"Requests/sec:\s+([\d.]+)") |
| 46 | + |
| 47 | + with open(filepath, "r") as f: |
| 48 | + for line in f: |
| 49 | + # Identify the test type |
| 50 | + match_test = test_type_pattern.search(line) |
| 51 | + if match_test: |
| 52 | + current_test = match_test.group(1) |
| 53 | + continue |
| 54 | + |
| 55 | + # Find 99% latency |
| 56 | + match_99 = latency_pattern.search(line) |
| 57 | + if match_99: |
| 58 | + latency_99 = float(match_99.group(1)) |
| 59 | + continue |
| 60 | + |
| 61 | + # Find Requests/sec |
| 62 | + match_req = req_sec_pattern.search(line) |
| 63 | + if match_req and current_test: |
| 64 | + requests_sec = float(match_req.group(1)) |
| 65 | + results.append( |
| 66 | + { |
| 67 | + "test": current_test, |
| 68 | + "99%_latency_ms": latency_99, |
| 69 | + "requests_per_sec": requests_sec, |
| 70 | + } |
| 71 | + ) |
| 72 | + # Reset for next run |
| 73 | + current_test = None |
| 74 | + latency_99 = None |
| 75 | + requests_sec = None |
| 76 | + |
| 77 | + return results |
| 78 | + |
| 79 | + |
| 80 | +def make_comparison_table(all_results): |
| 81 | + # Map: framework -> { "hello": {...}, "order": {...}, "async_hello": {...}, "async_order": {...} } |
| 82 | + framework_map = { |
| 83 | + "aiohttp": "aiohttp", |
| 84 | + "aiozmq": "aiozmq", |
| 85 | + "blacksheep": "blacksheep", |
| 86 | + "fastapi": "fastApi", |
| 87 | + "sanic": "sanic", |
| 88 | + "zero": "zero(sync)", |
| 89 | + "zero-async": "zero(async)", |
| 90 | + } |
| 91 | + # Prepare a dict to collect results |
| 92 | + table_data = {fw: {} for fw in framework_map.values()} |
| 93 | + |
| 94 | + # Helper to match test names to columns |
| 95 | + def classify(framework, test): |
| 96 | + if framework == "zero": |
| 97 | + if test == "hello": |
| 98 | + return "zero(sync)", "hello" |
| 99 | + elif test == "order": |
| 100 | + return "zero(sync)", "order" |
| 101 | + elif test == "async_hello": |
| 102 | + return "zero(async)", "hello" |
| 103 | + elif test == "async_order": |
| 104 | + return "zero(async)", "order" |
| 105 | + else: |
| 106 | + if test == "hello": |
| 107 | + return framework_map[framework], "hello" |
| 108 | + elif test == "order": |
| 109 | + return framework_map[framework], "order" |
| 110 | + return None, None |
| 111 | + |
| 112 | + for benchmark, results in all_results.items(): |
| 113 | + framework = benchmark.replace("benchmark-", "") |
| 114 | + for result in results: |
| 115 | + fw, col = classify(framework, result["test"]) |
| 116 | + if fw and col: |
| 117 | + table_data[fw][col] = result |
| 118 | + |
| 119 | + header = ( |
| 120 | + '| Framework | "hello world" (req/s) | 99% latency (ms) | redis save (req/s) | 99% latency (ms) |\n' |
| 121 | + "| ----------- | --------------------- | ---------------- | ------------------ | ---------------- |" |
| 122 | + ) |
| 123 | + rows = [] |
| 124 | + for fw in [ |
| 125 | + "aiohttp", |
| 126 | + "aiozmq", |
| 127 | + "blacksheep", |
| 128 | + "fastApi", |
| 129 | + "sanic", |
| 130 | + "zero(sync)", |
| 131 | + "zero(async)", |
| 132 | + ]: |
| 133 | + hello = table_data[fw].get("hello", {}) |
| 134 | + order = table_data[fw].get("order", {}) |
| 135 | + row = ( |
| 136 | + f"| {fw:<11} | " |
| 137 | + f"{hello.get('requests_per_sec', ''):<21} | {hello.get('99%_latency_ms', ''):<16} | " |
| 138 | + f"{order.get('requests_per_sec', ''):<18} | {order.get('99%_latency_ms', ''):<16} |" |
| 139 | + ) |
| 140 | + rows.append(row) |
| 141 | + return header + "\n" + "\n".join(rows) |
| 142 | + |
| 143 | + |
| 144 | +if __name__ == "__main__": |
| 145 | + history_files = get_latest_history() |
| 146 | + all_results = {} |
| 147 | + for history_file in history_files: |
| 148 | + filepath = os.path.join("dockerize/history", history_file) |
| 149 | + if os.path.exists(filepath): |
| 150 | + benchmark_name = history_file.split("__")[0] |
| 151 | + results = parse_benchmark_log(filepath) |
| 152 | + all_results[benchmark_name] = results |
| 153 | + comparison_table = make_comparison_table(all_results) |
| 154 | + print(comparison_table) |
0 commit comments