Skip to content

Commit 3c42ffe

Browse files
alwayslove2013XuanYang-cn
alwayslove2013
authored and
XuanYang-cn
committed
update leaderboard: add s3vectors results; add streaming results
Signed-off-by: min.tian <min.tian.cn@gmail.com>
1 parent 899c070 commit 3c42ffe

3 files changed

Lines changed: 389 additions & 2 deletions

File tree

‎vectordb_bench/results/getLeaderboardDataV2.py‎

Lines changed: 33 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -2,10 +2,11 @@
22
import logging
33

44

5-
from vectordb_bench.backend.cases import CaseType
5+
from vectordb_bench.backend.cases import CaseType, StreamingPerformanceCase
66
from vectordb_bench.backend.clients import DB
77
from vectordb_bench.models import CaseResult
88
from vectordb_bench import config
9+
import numpy as np
910

1011
logging.basicConfig(level=logging.DEBUG, format="%(asctime)s - %(levelname)s - %(message)s")
1112

@@ -29,9 +30,11 @@ def save_to_json(data: list[dict], file_name: str):
2930
def main():
3031
standard_2025_case_results = get_standard_2025_results()
3132
data = []
33+
streaming_data = []
3234
for case_result in standard_2025_case_results:
3335
db = case_result.task_config.db
3436
label = case_result.task_config.db_config.db_label
37+
db_name = f"{db.value}{f'-{label}' if label else ''}"
3538
metrics = case_result.metrics
3639
qps = metrics.qps
3740
latency = metrics.serial_latency_p99
@@ -45,14 +48,42 @@ def main():
4548
"dataset": dataset,
4649
"db": db.value,
4750
"label": label,
48-
"db_name": f"{db.value}-{label}",
51+
"db_name": db_name,
4952
"qps": round(qps, 4),
5053
"latency": round(latency, 4),
5154
"recall": round(recall, 4),
5255
"filter_ratio": round(filter_ratio, 3),
5356
}
5457
)
58+
else:
59+
case: StreamingPerformanceCase = case
60+
# only use streaming-results (reading while inserting), exclude streaming-final or streaming-optimized results.
61+
qps_list = [
62+
qps
63+
for i, qps in enumerate(case_result.metrics.st_max_qps_list_list)
64+
if case_result.metrics.st_search_stage_list[i] < 100
65+
]
66+
latency_list = [
67+
latency
68+
for i, latency in enumerate(case_result.metrics.st_serial_latency_p99_list)
69+
if case_result.metrics.st_search_stage_list[i] < 100
70+
]
71+
insert_rate = case.insert_rate
72+
streaming_data.append(
73+
{
74+
"dataset": dataset,
75+
"db": db.value,
76+
"label": label,
77+
"db_name": db_name,
78+
"insert_rate": insert_rate,
79+
"qps_avg": round(np.mean(qps_list), 4),
80+
"qps_std": round(np.std(qps_list), 4),
81+
"latency_avg": round(np.mean(latency_list), 4),
82+
"latency_std": round(np.std(latency_list), 4),
83+
}
84+
)
5585
save_to_json(data, config.RESULTS_LOCAL_DIR / "leaderboard_v2.json")
86+
save_to_json(streaming_data, config.RESULTS_LOCAL_DIR / "leaderboard_v2_streaming.json")
5687

5788

5889
if __name__ == "__main__":

‎vectordb_bench/results/leaderboard_v2.json‎

Lines changed: 200 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2658,5 +2658,205 @@
26582658
"latency": 3.3,
26592659
"recall": 0.9147,
26602660
"filter_ratio": 0.5
2661+
},
2662+
{
2663+
"dataset": "Cohere (Medium)",
2664+
"db": "S3Vectors",
2665+
"label": "",
2666+
"db_name": "S3Vectors",
2667+
"qps": 199.4972,
2668+
"latency": 337.1,
2669+
"recall": 0.8717,
2670+
"filter_ratio": 0.0
2671+
},
2672+
{
2673+
"dataset": "Cohere (Medium)",
2674+
"db": "S3Vectors",
2675+
"label": "",
2676+
"db_name": "S3Vectors",
2677+
"qps": 192.1164,
2678+
"latency": 345.6,
2679+
"recall": 0.4276,
2680+
"filter_ratio": 0.999
2681+
},
2682+
{
2683+
"dataset": "Cohere (Medium)",
2684+
"db": "S3Vectors",
2685+
"label": "",
2686+
"db_name": "S3Vectors",
2687+
"qps": 197.4455,
2688+
"latency": 349.3,
2689+
"recall": 0.5314,
2690+
"filter_ratio": 0.998
2691+
},
2692+
{
2693+
"dataset": "Cohere (Medium)",
2694+
"db": "S3Vectors",
2695+
"label": "",
2696+
"db_name": "S3Vectors",
2697+
"qps": 196.9391,
2698+
"latency": 263.4,
2699+
"recall": 0.6549,
2700+
"filter_ratio": 0.995
2701+
},
2702+
{
2703+
"dataset": "Cohere (Medium)",
2704+
"db": "S3Vectors",
2705+
"label": "",
2706+
"db_name": "S3Vectors",
2707+
"qps": 201.5401,
2708+
"latency": 282.4,
2709+
"recall": 0.7086,
2710+
"filter_ratio": 0.99
2711+
},
2712+
{
2713+
"dataset": "Cohere (Medium)",
2714+
"db": "S3Vectors",
2715+
"label": "",
2716+
"db_name": "S3Vectors",
2717+
"qps": 202.2424,
2718+
"latency": 301.7,
2719+
"recall": 0.7592,
2720+
"filter_ratio": 0.98
2721+
},
2722+
{
2723+
"dataset": "Cohere (Medium)",
2724+
"db": "S3Vectors",
2725+
"label": "",
2726+
"db_name": "S3Vectors",
2727+
"qps": 198.599,
2728+
"latency": 358.8,
2729+
"recall": 0.8085,
2730+
"filter_ratio": 0.95
2731+
},
2732+
{
2733+
"dataset": "Cohere (Medium)",
2734+
"db": "S3Vectors",
2735+
"label": "",
2736+
"db_name": "S3Vectors",
2737+
"qps": 199.0349,
2738+
"latency": 275.3,
2739+
"recall": 0.8325,
2740+
"filter_ratio": 0.9
2741+
},
2742+
{
2743+
"dataset": "Cohere (Medium)",
2744+
"db": "S3Vectors",
2745+
"label": "",
2746+
"db_name": "S3Vectors",
2747+
"qps": 202.1405,
2748+
"latency": 282.6,
2749+
"recall": 0.8492,
2750+
"filter_ratio": 0.8
2751+
},
2752+
{
2753+
"dataset": "Cohere (Medium)",
2754+
"db": "S3Vectors",
2755+
"label": "",
2756+
"db_name": "S3Vectors",
2757+
"qps": 201.1282,
2758+
"latency": 269.2,
2759+
"recall": 0.8637,
2760+
"filter_ratio": 0.5
2761+
},
2762+
{
2763+
"dataset": "Cohere (Large)",
2764+
"db": "S3Vectors",
2765+
"label": "",
2766+
"db_name": "S3Vectors",
2767+
"qps": 194.8021,
2768+
"latency": 559.8,
2769+
"recall": 0.86,
2770+
"filter_ratio": 0.0
2771+
},
2772+
{
2773+
"dataset": "Cohere (Large)",
2774+
"db": "S3Vectors",
2775+
"label": "",
2776+
"db_name": "S3Vectors",
2777+
"qps": 187.4268,
2778+
"latency": 453.7,
2779+
"recall": 0.4692,
2780+
"filter_ratio": 0.999
2781+
},
2782+
{
2783+
"dataset": "Cohere (Large)",
2784+
"db": "S3Vectors",
2785+
"label": "",
2786+
"db_name": "S3Vectors",
2787+
"qps": 198.397,
2788+
"latency": 506.9,
2789+
"recall": 0.5409,
2790+
"filter_ratio": 0.998
2791+
},
2792+
{
2793+
"dataset": "Cohere (Large)",
2794+
"db": "S3Vectors",
2795+
"label": "",
2796+
"db_name": "S3Vectors",
2797+
"qps": 174.3549,
2798+
"latency": 496.9,
2799+
"recall": 0.6279,
2800+
"filter_ratio": 0.995
2801+
},
2802+
{
2803+
"dataset": "Cohere (Large)",
2804+
"db": "S3Vectors",
2805+
"label": "",
2806+
"db_name": "S3Vectors",
2807+
"qps": 172.95,
2808+
"latency": 515.6,
2809+
"recall": 0.7004,
2810+
"filter_ratio": 0.99
2811+
},
2812+
{
2813+
"dataset": "Cohere (Large)",
2814+
"db": "S3Vectors",
2815+
"label": "",
2816+
"db_name": "S3Vectors",
2817+
"qps": 190.9747,
2818+
"latency": 517.4,
2819+
"recall": 0.7398,
2820+
"filter_ratio": 0.98
2821+
},
2822+
{
2823+
"dataset": "Cohere (Large)",
2824+
"db": "S3Vectors",
2825+
"label": "",
2826+
"db_name": "S3Vectors",
2827+
"qps": 186.0237,
2828+
"latency": 474.0,
2829+
"recall": 0.7847,
2830+
"filter_ratio": 0.95
2831+
},
2832+
{
2833+
"dataset": "Cohere (Large)",
2834+
"db": "S3Vectors",
2835+
"label": "",
2836+
"db_name": "S3Vectors",
2837+
"qps": 192.1458,
2838+
"latency": 480.5,
2839+
"recall": 0.8103,
2840+
"filter_ratio": 0.9
2841+
},
2842+
{
2843+
"dataset": "Cohere (Large)",
2844+
"db": "S3Vectors",
2845+
"label": "",
2846+
"db_name": "S3Vectors",
2847+
"qps": 179.4203,
2848+
"latency": 497.5,
2849+
"recall": 0.8273,
2850+
"filter_ratio": 0.8
2851+
},
2852+
{
2853+
"dataset": "Cohere (Large)",
2854+
"db": "S3Vectors",
2855+
"label": "",
2856+
"db_name": "S3Vectors",
2857+
"qps": 199.5444,
2858+
"latency": 463.9,
2859+
"recall": 0.8478,
2860+
"filter_ratio": 0.5
26612861
}
26622862
]

0 commit comments

Comments
 (0)