Skip to content

update leaderboard: add s3vectors results; add streaming results #580

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
XuanYang-cn merged 1 commit into from
Aug 6, 2025
Merged

update leaderboard: add s3vectors results; add streaming results #580

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
35 changes: 33 additions & 2 deletions vectordb_bench/results/getLeaderboardDataV2.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,10 +2,11 @@
import logging


from vectordb_bench.backend.cases import CaseType
from vectordb_bench.backend.cases import CaseType, StreamingPerformanceCase
from vectordb_bench.backend.clients import DB
from vectordb_bench.models import CaseResult
from vectordb_bench import config
import numpy as np

logging.basicConfig(level=logging.DEBUG, format="%(asctime)s - %(levelname)s - %(message)s")

Expand All @@ -29,9 +30,11 @@ def save_to_json(data: list[dict], file_name: str):
def main():
standard_2025_case_results = get_standard_2025_results()
data = []
streaming_data = []
for case_result in standard_2025_case_results:
db = case_result.task_config.db
label = case_result.task_config.db_config.db_label
db_name = f"{db.value}{f'-{label}' if label else ''}"
metrics = case_result.metrics
qps = metrics.qps
latency = metrics.serial_latency_p99
Expand All @@ -45,14 +48,42 @@ def main():
"dataset": dataset,
"db": db.value,
"label": label,
"db_name": f"{db.value}-{label}",
"db_name": db_name,
"qps": round(qps, 4),
"latency": round(latency, 4),
"recall": round(recall, 4),
"filter_ratio": round(filter_ratio, 3),
}
)
else:
case: StreamingPerformanceCase = case
# only use streaming-results (reading while inserting), exclude streaming-final or streaming-optimized results.
qps_list = [
qps
for i, qps in enumerate(case_result.metrics.st_max_qps_list_list)
if case_result.metrics.st_search_stage_list[i] < 100
]
latency_list = [
latency
for i, latency in enumerate(case_result.metrics.st_serial_latency_p99_list)
if case_result.metrics.st_search_stage_list[i] < 100
]
insert_rate = case.insert_rate
streaming_data.append(
{
"dataset": dataset,
"db": db.value,
"label": label,
"db_name": db_name,
"insert_rate": insert_rate,
"qps_avg": round(np.mean(qps_list), 4),
"qps_std": round(np.std(qps_list), 4),
"latency_avg": round(np.mean(latency_list), 4),
"latency_std": round(np.std(latency_list), 4),
}
)
save_to_json(data, config.RESULTS_LOCAL_DIR / "leaderboard_v2.json")
save_to_json(streaming_data, config.RESULTS_LOCAL_DIR / "leaderboard_v2_streaming.json")


if __name__ == "__main__":
Expand Down
200 changes: 200 additions & 0 deletions vectordb_bench/results/leaderboard_v2.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2658,5 +2658,205 @@
"latency": 3.3,
"recall": 0.9147,
"filter_ratio": 0.5
},
{
"dataset": "Cohere (Medium)",
"db": "S3Vectors",
"label": "",
"db_name": "S3Vectors",
"qps": 199.4972,
"latency": 337.1,
"recall": 0.8717,
"filter_ratio": 0.0
},
{
"dataset": "Cohere (Medium)",
"db": "S3Vectors",
"label": "",
"db_name": "S3Vectors",
"qps": 192.1164,
"latency": 345.6,
"recall": 0.4276,
"filter_ratio": 0.999
},
{
"dataset": "Cohere (Medium)",
"db": "S3Vectors",
"label": "",
"db_name": "S3Vectors",
"qps": 197.4455,
"latency": 349.3,
"recall": 0.5314,
"filter_ratio": 0.998
},
{
"dataset": "Cohere (Medium)",
"db": "S3Vectors",
"label": "",
"db_name": "S3Vectors",
"qps": 196.9391,
"latency": 263.4,
"recall": 0.6549,
"filter_ratio": 0.995
},
{
"dataset": "Cohere (Medium)",
"db": "S3Vectors",
"label": "",
"db_name": "S3Vectors",
"qps": 201.5401,
"latency": 282.4,
"recall": 0.7086,
"filter_ratio": 0.99
},
{
"dataset": "Cohere (Medium)",
"db": "S3Vectors",
"label": "",
"db_name": "S3Vectors",
"qps": 202.2424,
"latency": 301.7,
"recall": 0.7592,
"filter_ratio": 0.98
},
{
"dataset": "Cohere (Medium)",
"db": "S3Vectors",
"label": "",
"db_name": "S3Vectors",
"qps": 198.599,
"latency": 358.8,
"recall": 0.8085,
"filter_ratio": 0.95
},
{
"dataset": "Cohere (Medium)",
"db": "S3Vectors",
"label": "",
"db_name": "S3Vectors",
"qps": 199.0349,
"latency": 275.3,
"recall": 0.8325,
"filter_ratio": 0.9
},
{
"dataset": "Cohere (Medium)",
"db": "S3Vectors",
"label": "",
"db_name": "S3Vectors",
"qps": 202.1405,
"latency": 282.6,
"recall": 0.8492,
"filter_ratio": 0.8
},
{
"dataset": "Cohere (Medium)",
"db": "S3Vectors",
"label": "",
"db_name": "S3Vectors",
"qps": 201.1282,
"latency": 269.2,
"recall": 0.8637,
"filter_ratio": 0.5
},
{
"dataset": "Cohere (Large)",
"db": "S3Vectors",
"label": "",
"db_name": "S3Vectors",
"qps": 194.8021,
"latency": 559.8,
"recall": 0.86,
"filter_ratio": 0.0
},
{
"dataset": "Cohere (Large)",
"db": "S3Vectors",
"label": "",
"db_name": "S3Vectors",
"qps": 187.4268,
"latency": 453.7,
"recall": 0.4692,
"filter_ratio": 0.999
},
{
"dataset": "Cohere (Large)",
"db": "S3Vectors",
"label": "",
"db_name": "S3Vectors",
"qps": 198.397,
"latency": 506.9,
"recall": 0.5409,
"filter_ratio": 0.998
},
{
"dataset": "Cohere (Large)",
"db": "S3Vectors",
"label": "",
"db_name": "S3Vectors",
"qps": 174.3549,
"latency": 496.9,
"recall": 0.6279,
"filter_ratio": 0.995
},
{
"dataset": "Cohere (Large)",
"db": "S3Vectors",
"label": "",
"db_name": "S3Vectors",
"qps": 172.95,
"latency": 515.6,
"recall": 0.7004,
"filter_ratio": 0.99
},
{
"dataset": "Cohere (Large)",
"db": "S3Vectors",
"label": "",
"db_name": "S3Vectors",
"qps": 190.9747,
"latency": 517.4,
"recall": 0.7398,
"filter_ratio": 0.98
},
{
"dataset": "Cohere (Large)",
"db": "S3Vectors",
"label": "",
"db_name": "S3Vectors",
"qps": 186.0237,
"latency": 474.0,
"recall": 0.7847,
"filter_ratio": 0.95
},
{
"dataset": "Cohere (Large)",
"db": "S3Vectors",
"label": "",
"db_name": "S3Vectors",
"qps": 192.1458,
"latency": 480.5,
"recall": 0.8103,
"filter_ratio": 0.9
},
{
"dataset": "Cohere (Large)",
"db": "S3Vectors",
"label": "",
"db_name": "S3Vectors",
"qps": 179.4203,
"latency": 497.5,
"recall": 0.8273,
"filter_ratio": 0.8
},
{
"dataset": "Cohere (Large)",
"db": "S3Vectors",
"label": "",
"db_name": "S3Vectors",
"qps": 199.5444,
"latency": 463.9,
"recall": 0.8478,
"filter_ratio": 0.5
}
]
Loading
Loading