From 2de3ee1053bf8b4d3bf43a1526edf538e3559640 Mon Sep 17 00:00:00 2001 From: "siqi.an" Date: Tue, 8 Apr 2025 14:20:45 +0800 Subject: [PATCH] set navigator --- .../components/check_results/headerIcon.py | 29 ++-- .../frontend/components/check_results/nav.py | 20 +++ .../components/welcome/explainPrams.py | 66 ++++++++ .../frontend/components/welcome/pagestyle.py | 105 +++++++++++++ .../components/welcome/welcomePrams.py | 147 ++++++++++++++++++ vectordb_bench/frontend/pages/concurrent.py | 4 + vectordb_bench/frontend/pages/custom.py | 4 + vectordb_bench/frontend/pages/label_filter.py | 4 + .../frontend/pages/quries_per_dollar.py | 6 +- vectordb_bench/frontend/pages/results.py | 60 +++++++ vectordb_bench/frontend/pages/run_test.py | 6 +- vectordb_bench/frontend/pages/streaming.py | 4 + vectordb_bench/frontend/pages/tables.py | 4 + vectordb_bench/frontend/vdb_benchmark.py | 57 ++----- 14 files changed, 458 insertions(+), 58 deletions(-) create mode 100644 vectordb_bench/frontend/components/welcome/explainPrams.py create mode 100644 vectordb_bench/frontend/components/welcome/pagestyle.py create mode 100644 vectordb_bench/frontend/components/welcome/welcomePrams.py create mode 100644 vectordb_bench/frontend/pages/results.py diff --git a/vectordb_bench/frontend/components/check_results/headerIcon.py b/vectordb_bench/frontend/components/check_results/headerIcon.py index cd18251e8..27e748e18 100644 --- a/vectordb_bench/frontend/components/check_results/headerIcon.py +++ b/vectordb_bench/frontend/components/check_results/headerIcon.py @@ -4,19 +4,22 @@ def drawHeaderIcon(st): st.markdown( f""" -
+ +
+ - + .headerIconContainer {{ + position: relative; + top: 0px; + height: 50px; + width: 100%; + border-bottom: 2px solid #E8EAEE; + background-image: url({HEADER_ICON}); + background-repeat: no-repeat; + cursor: pointer; + }} + + """, unsafe_allow_html=True, ) diff --git a/vectordb_bench/frontend/components/check_results/nav.py b/vectordb_bench/frontend/components/check_results/nav.py index f95e43d7a..ca9e9d997 100644 --- a/vectordb_bench/frontend/components/check_results/nav.py +++ b/vectordb_bench/frontend/components/check_results/nav.py @@ -20,3 +20,23 @@ def NavToResults(st, key="nav-to-results"): navClick = st.button("<   Back to Results", key=key) if navClick: switch_page("vdb benchmark") + + +def NavToPages(st): + options = [ + {"name": "Run Test", "link": "run_test"}, + {"name": "Results", "link": "results"}, + {"name": "Concurrent", "link": "concurrent"}, + {"name": "Label Filter", "link": "label_filter"}, + {"name": "Quries Per Dollar", "link": "quries_per_dollar"}, + {"name": "Tables", "link": "tables"}, + {"name": "Custom", "link": "custom"}, + {"name": "Streaming", "link": "streaming"}, + ] + + html = "" + for i, option in enumerate(options): + html += f'{option["name"]}' + if i < len(options) - 1: + html += '|' + st.markdown(html, unsafe_allow_html=True) diff --git a/vectordb_bench/frontend/components/welcome/explainPrams.py b/vectordb_bench/frontend/components/welcome/explainPrams.py new file mode 100644 index 000000000..966277232 --- /dev/null +++ b/vectordb_bench/frontend/components/welcome/explainPrams.py @@ -0,0 +1,66 @@ +def explainPrams(st): + st.markdown("## descriptions") + st.markdown("### 1.Overview") + st.markdown( + """ +- **VectorDBBench** is an open-source benchmarking tool designed specifically for vector databases. Its main features include: + - (1) An easy-to-use **web UI** for configuration of tests and visual analysis of results. + - (2) A comprehensive set of **standards for testing and metric collection**. + - (3) Support for **various scenarios**, including additional support for **Filter** and **Streaming** based on standard tests. +- VectorDBBench embraces open-source and welcome contributions of code and test result submissions. The testing process and extended scenarios of VectorDBBench, as well as the intention behind our design will be introduced as follows. +""" + ) + st.markdown("### 2.Dataset") + st.markdown( + """ +- We provide two embedding datasets: + - (1)*[Cohere 768dim](https://huggingface.co/datasets/Cohere/wikipedia-22-12)*, generated using the **Cohere** model based on the Wikipedia corpus. + - (2)*[Cohere 1024dim](https://huggingface.co/datasets/Cohere/beir-embed-english-v3)*, generated using the **Cohere** embed-english-v3.0 model based on the bioasq corpus. + - (3)*OpenAI 1536dim*, generated using the **OpenAI** model based on the [C4 corpus](https://huggingface.co/datasets/legacy-datasets/c4). +""" + ) + st.markdown("### 3.Standard Test") + st.markdown( + """ +The test is actually divided into 3 sub-processes +- **3.1 Test Part 1 - Load (Insert + Optimize)** + - (1) Use a single process to perform serial inserts until all data is inserted, and record the time taken as **insert_duration**. + - (2) For most vector databases, index construction requires additional time to optimize to achieve an optimal state, and record the time taken as **optimize_duration**. + - (3) **Load_duration (insert_duration + optimize_duration)** can be understood as the time from the start of insertion until the database is ready to query. + - load_duration can serve as a reference for the insert capability of a vector database to some extent. However, it should be noted that some vector databases may perform better under **concurrent insert operations**. +- **3.2 Test Part 2 - Serial Search Test** + - (1) Use a single process to perform serial searches, record the results and time taken for each search, and calculate **recall** and **latency**. + - (2) **Recall**: For vector databases, most searches are approximately nearest neighbor(ANN) searches rather than perfectly accurate results. In production environments, commonly targeted recall rates are 0.9 or 0.95. + - Note that there is a **trade-off** between **accuracy** and **search performance**. By adjusting parameters, it is possible to sacrifice some accuracy in exchange for better performance. We recommend comparing performance while ensuring that the recall rates remain reasonably close. + - (3) **Latency**:**p99** rather than average. **latency_p99** focuses on **the slowest 1% of requests**. In many high-demand applications, ensuring that most user requests stay within acceptable latency limits is critical, whereas **latency_avg** can be skewed by faster requests. + - **serial_latency** can serve as a reference for a database's search capability to some extent. However, serial_latency is significantly affected by network conditions. We recommend running the test client and database server within the same local network. +- **3.3 Test Part 3 - Concurrent Search Test** + - (1) Create multiple processes, each perform serial searches independently to test the database's **maximum throughput(max-qps)**. + - (2) Since different databases may reach peak throughput under different conditions, we conduct multiple test rounds. The number of processes **starts at 1 by default and gradually increases up to 80**, with each test group running for **30 seconds**. + - Detailed latency and QPS metrics at different concurrency levels can be viewed on the *concurrent* page. + - The highest recorded QPS value from these tests will be selected as the final max-qps. +""", + unsafe_allow_html=True, + ) + st.markdown("### 4.Filter Search Test") + st.markdown( + """ +- Compared to the Standard Test, the **Filter Search** introduces additional scalar constraints (e.g. **color == red**) during the Search Test. Different **filter_ratios** present varying levels of challenge to the VectorDB's search performance. +- We provide an additional **string column** containing 10 labels with different distribution ratios (50%,20%,10%,5%,2%,1%,0.5%,0.2%,0.1%). For each label, we conduct both a **Serial Test** and a **Concurrency Test** to observe the VectorDB's performance in terms of **QPS, latency, and recall** under different filtering conditions. +""" + ) + st.markdown("### 5.Streaming Search Test") + st.markdown( + """ +Different from Standard's load and search separation, Streaming Search Test primarily focuses on **search performance during the insertion process**. +Different **base dataset sizes** and varying **insertion rates** set distinct challenges to the VectorDB's search capabilities. +VectorDBBench will send insert requests at a **fixed rate**, maintaining consistent insertion pressure. The search test consists of three steps as follows: +- 1.**Streaming Search** + - Users can configure **multiple search stages**. When the inserted data volume reaches a specified stage, a **Serial Test** and a **Concurrent Test** will be conducted, recording qps, latency, and recall performance. +- 2.**Streaming Final Search** + - After all of the data is inserted, a Serial Test and a Concurrent Test are immediately performed, recording qps, latency, and recall performance. + - Note: at this time, the insertion pressure drops to zero since data insertion is complete. +- 3.**Optimized Search (Optional)** + - Users can optionally perform an additional optimization step followed by a Serial Test and a Concurrent Test, recording qps, latency, and recall performance. This step **compares performance in Streaming section with the theoretically optimal performance**. +""" + ) diff --git a/vectordb_bench/frontend/components/welcome/pagestyle.py b/vectordb_bench/frontend/components/welcome/pagestyle.py new file mode 100644 index 000000000..0a628797e --- /dev/null +++ b/vectordb_bench/frontend/components/welcome/pagestyle.py @@ -0,0 +1,105 @@ +def pagestyle(): + html_content = """ + + +
+ """ + return html_content diff --git a/vectordb_bench/frontend/components/welcome/welcomePrams.py b/vectordb_bench/frontend/components/welcome/welcomePrams.py new file mode 100644 index 000000000..eade277fb --- /dev/null +++ b/vectordb_bench/frontend/components/welcome/welcomePrams.py @@ -0,0 +1,147 @@ +import base64 +from PIL import Image +from io import BytesIO +import os + +from vectordb_bench.frontend.components.welcome.pagestyle import pagestyle + + +def get_image_as_base64(image_path): + try: + if image_path.startswith("http"): + return image_path + + path = os.path.expanduser(image_path) + img = Image.open(path) + buffered = BytesIO() + img.save(buffered, format="PNG") + return f"data:image/png;base64,{base64.b64encode(buffered.getvalue()).decode()}" + except Exception as e: + raise (f"wrong loading: {e}") + + +def welcomePrams(st): + st.title("Welcome to VectorDB Benchmark!") + options = [ + { + "title": "Results", + "description": ( + "" + "Select a specific run or compare all results side by side to view the results of previous tests." + "" + ), + "image": "/Users/zilliz/static/results.png", + "link": "results", + }, + { + "title": "Quries Per Dollar", + "description": ( + "" + "To view the results of quries per dollar.
" + "(similar to qps in Results) " + "" + ), + "image": "/Users/zilliz/static/qpd.png", + "link": "quries_per_dollar", + }, + { + "title": "Tables", + "description": ( + "" "To view the results of differnt datasets in tables." "" + ), + "image": "/Users/zilliz/static/tables.png", + "link": "tables", + }, + { + "title": "Concurrent Performance", + "description": ( + "" + "To view the variation of qps with latency under different concurrent." + "" + ), + "image": "/Users/zilliz/static/concurrent.png", + "link": "concurrent", + }, + { + "title": "Label Filter", + "description": ( + "" + "To view the perfomance of datasets under different filter ratios " + "" + ), + "image": "/Users/zilliz/static/label_filter.png", + "link": "label_filter", + }, + { + "title": "Streaming Performance", + "description": ( + "" + "To view the perfomance of datasets under different search stages and insertion rates. " + "" + ), + "image": "/Users/zilliz/static/streaming.png", + "link": "streaming", + }, + { + "title": "Run Test", + "description": ( + "" + "Select the databases and cases to test.
" + "The test results will be displayed in Results." + "" + ), + "image": "/Users/zilliz/static/run_test.png", + "link": "run_test", + }, + { + "title": "Custom Dataset", + "description": ( + "" + "Define users' own datasets with detailed descriptions of setting each parameter." + "" + ), + "image": "/Users/zilliz/static/custom.png", + "link": "custom", + }, + ] + + html_content = pagestyle() + + for option in options: + option["image"] = get_image_as_base64(option["image"]) + + for i, option in enumerate(options[:6]): + html_content += f""" + +
+ {option['title']} +
{option['title']}
+
{option['description']}
+
+ + """ + + html_content += """ +
+
+

Set And Run

+
+
+ """ + + for option in options[6:8]: + html_content += f""" + +
+ {option['title']} +
{option['title']}
+
{option['description']}
+
+ + """ + + html_content += """ +
+ """ + + st.html(html_content) diff --git a/vectordb_bench/frontend/pages/concurrent.py b/vectordb_bench/frontend/pages/concurrent.py index 852f31651..74a536edc 100644 --- a/vectordb_bench/frontend/pages/concurrent.py +++ b/vectordb_bench/frontend/pages/concurrent.py @@ -4,6 +4,7 @@ from vectordb_bench.frontend.components.check_results.nav import ( NavToResults, NavToRunTest, + NavToPages, ) from vectordb_bench.frontend.components.check_results.filters import getshownData from vectordb_bench.frontend.components.concurrent.charts import drawChartsByCase @@ -25,6 +26,9 @@ def main(): # header drawHeaderIcon(st) + # navigate + NavToPages(st) + allResults = benchmark_runner.get_results() def check_conc_data(res: TestResult): diff --git a/vectordb_bench/frontend/pages/custom.py b/vectordb_bench/frontend/pages/custom.py index 4f6beed91..b316d2094 100644 --- a/vectordb_bench/frontend/pages/custom.py +++ b/vectordb_bench/frontend/pages/custom.py @@ -1,6 +1,7 @@ from functools import partial import streamlit as st from vectordb_bench.frontend.components.check_results.headerIcon import drawHeaderIcon +from vectordb_bench.frontend.components.check_results.nav import NavToPages from vectordb_bench.frontend.components.custom.displayCustomCase import ( displayCustomCase, ) @@ -49,6 +50,9 @@ def main(): # init style initStyle(st) + # navigate + NavToPages(st) + st.title("Custom Dataset") displayParams(st) customCaseManager = CustomCaseManager() diff --git a/vectordb_bench/frontend/pages/label_filter.py b/vectordb_bench/frontend/pages/label_filter.py index 9e39c8874..deab8b73a 100644 --- a/vectordb_bench/frontend/pages/label_filter.py +++ b/vectordb_bench/frontend/pages/label_filter.py @@ -5,6 +5,7 @@ from vectordb_bench.frontend.components.check_results.nav import ( NavToQuriesPerDollar, NavToRunTest, + NavToPages, ) from vectordb_bench.frontend.components.label_filter.charts import drawCharts from vectordb_bench.frontend.components.check_results.filters import getshownData @@ -24,6 +25,9 @@ def main(): # header drawHeaderIcon(st) + # navigate + NavToPages(st) + allResults = benchmark_runner.get_results() st.title("Vector Database Benchmark (Label Filter)") diff --git a/vectordb_bench/frontend/pages/quries_per_dollar.py b/vectordb_bench/frontend/pages/quries_per_dollar.py index 2822f2864..078d8490d 100644 --- a/vectordb_bench/frontend/pages/quries_per_dollar.py +++ b/vectordb_bench/frontend/pages/quries_per_dollar.py @@ -9,6 +9,7 @@ ) from vectordb_bench.frontend.components.check_results.headerIcon import drawHeaderIcon from vectordb_bench.frontend.components.check_results.nav import ( + NavToPages, NavToResults, NavToRunTest, ) @@ -27,13 +28,16 @@ def main(): # header drawHeaderIcon(st) + # navigate + NavToPages(st) + allResults = benchmark_runner.get_results() st.title("Vector DB Benchmark (QP$)") # results selector resultSelectorContainer = st.sidebar.container() - shownData, _, showCaseNames = getshownData(allResults, resultSelectorContainer) + shownData, _, showCaseNames = getshownData(resultSelectorContainer, allResults) resultSelectorContainer.divider() diff --git a/vectordb_bench/frontend/pages/results.py b/vectordb_bench/frontend/pages/results.py new file mode 100644 index 000000000..933b9b29f --- /dev/null +++ b/vectordb_bench/frontend/pages/results.py @@ -0,0 +1,60 @@ +import streamlit as st +from vectordb_bench.frontend.components.check_results.footer import footer +from vectordb_bench.frontend.components.check_results.stPageConfig import ( + initResultsPageConfig, +) +from vectordb_bench.frontend.components.check_results.headerIcon import drawHeaderIcon +from vectordb_bench.frontend.components.check_results.nav import ( + NavToQuriesPerDollar, + NavToRunTest, + NavToPages, +) +from vectordb_bench.frontend.components.check_results.charts import drawCharts +from vectordb_bench.frontend.components.check_results.filters import getshownData +from vectordb_bench.frontend.components.get_results.saveAsImage import getResults + +from vectordb_bench.interface import benchmark_runner + + +def main(): + # set page config + initResultsPageConfig(st) + + # header + drawHeaderIcon(st) + + # navigate + NavToPages(st) + + allResults = benchmark_runner.get_results() + + st.title("Vector Database Benchmark") + st.caption( + "Except for zillizcloud-v2024.1, which was tested in _January 2024_, all other tests were completed before _August 2023_." + ) + st.caption("All tested milvus are in _standalone_ mode.") + + # results selector and filter + resultSelectorContainer = st.sidebar.container() + shownData, failedTasks, showCaseNames = getshownData(resultSelectorContainer, allResults) + + resultSelectorContainer.divider() + + # nav + navContainer = st.sidebar.container() + NavToRunTest(navContainer) + NavToQuriesPerDollar(navContainer) + + # save or share + resultesContainer = st.sidebar.container() + getResults(resultesContainer, "vectordb_bench") + + # charts + drawCharts(st, shownData, failedTasks, showCaseNames) + + # footer + footer(st.container()) + + +if __name__ == "__main__": + main() diff --git a/vectordb_bench/frontend/pages/run_test.py b/vectordb_bench/frontend/pages/run_test.py index 3da8ea2c0..e4472e767 100644 --- a/vectordb_bench/frontend/pages/run_test.py +++ b/vectordb_bench/frontend/pages/run_test.py @@ -7,7 +7,7 @@ from vectordb_bench.frontend.components.run_test.hideSidebar import hideSidebar from vectordb_bench.frontend.components.run_test.initStyle import initStyle from vectordb_bench.frontend.components.run_test.submitTask import submitTask -from vectordb_bench.frontend.components.check_results.nav import NavToResults +from vectordb_bench.frontend.components.check_results.nav import NavToResults, NavToPages from vectordb_bench.frontend.components.check_results.headerIcon import drawHeaderIcon from vectordb_bench.frontend.components.check_results.stPageConfig import initRunTestPageConfig @@ -25,8 +25,8 @@ def main(): # hide sidebar hideSidebar(st) - # nav to results - NavToResults(st) + # navigate + NavToPages(st) # header st.title("Run Your Test") diff --git a/vectordb_bench/frontend/pages/streaming.py b/vectordb_bench/frontend/pages/streaming.py index 53e894630..d3faf1a5a 100644 --- a/vectordb_bench/frontend/pages/streaming.py +++ b/vectordb_bench/frontend/pages/streaming.py @@ -6,6 +6,7 @@ from vectordb_bench.frontend.components.check_results.nav import ( NavToResults, NavToRunTest, + NavToPages, ) from vectordb_bench.frontend.components.check_results.filters import getshownData from vectordb_bench.frontend.components.streaming.charts import drawChartsByCase @@ -30,6 +31,9 @@ def main(): # header drawHeaderIcon(st) + # navigate + NavToPages(st) + allResults = benchmark_runner.get_results() def check_conc_data(res: TestResult): diff --git a/vectordb_bench/frontend/pages/tables.py b/vectordb_bench/frontend/pages/tables.py index c088dc930..7240e2375 100644 --- a/vectordb_bench/frontend/pages/tables.py +++ b/vectordb_bench/frontend/pages/tables.py @@ -1,5 +1,6 @@ import streamlit as st from vectordb_bench.frontend.components.check_results.headerIcon import drawHeaderIcon +from vectordb_bench.frontend.components.check_results.nav import NavToPages from vectordb_bench.frontend.components.tables.data import getNewResults from vectordb_bench.frontend.config.styles import FAVICON @@ -16,6 +17,9 @@ def main(): # header drawHeaderIcon(st) + # navigate + NavToPages(st) + df = getNewResults() st.dataframe(df, height=800) diff --git a/vectordb_bench/frontend/vdb_benchmark.py b/vectordb_bench/frontend/vdb_benchmark.py index ba77edd45..860467734 100644 --- a/vectordb_bench/frontend/vdb_benchmark.py +++ b/vectordb_bench/frontend/vdb_benchmark.py @@ -1,55 +1,30 @@ import streamlit as st -from vectordb_bench.frontend.components.check_results.footer import footer -from vectordb_bench.frontend.components.check_results.stPageConfig import ( - initResultsPageConfig, -) from vectordb_bench.frontend.components.check_results.headerIcon import drawHeaderIcon -from vectordb_bench.frontend.components.check_results.nav import ( - NavToQuriesPerDollar, - NavToRunTest, -) -from vectordb_bench.frontend.components.check_results.charts import drawCharts -from vectordb_bench.frontend.components.check_results.filters import getshownData -from vectordb_bench.frontend.components.get_results.saveAsImage import getResults - -from vectordb_bench.interface import benchmark_runner +from vectordb_bench.frontend.components.custom.initStyle import initStyle +from vectordb_bench.frontend.components.welcome.explainPrams import explainPrams +from vectordb_bench.frontend.components.welcome.welcomePrams import welcomePrams +from vectordb_bench.frontend.config.styles import FAVICON, PAGE_TITLE def main(): - # set page config - initResultsPageConfig(st) + st.set_page_config( + page_title=PAGE_TITLE, + page_icon=FAVICON, + layout="wide", + initial_sidebar_state="collapsed", + ) # header drawHeaderIcon(st) - allResults = benchmark_runner.get_results() - - st.title("Vector Database Benchmark") - st.caption( - "Except for zillizcloud-v2024.1, which was tested in _January 2024_, all other tests were completed before _August 2023_." - ) - st.caption("All tested milvus are in _standalone_ mode.") - - # results selector and filter - resultSelectorContainer = st.sidebar.container() - shownData, failedTasks, showCaseNames = getshownData(resultSelectorContainer, allResults) - - resultSelectorContainer.divider() - - # nav - navContainer = st.sidebar.container() - NavToRunTest(navContainer) - NavToQuriesPerDollar(navContainer) - - # save or share - resultesContainer = st.sidebar.container() - getResults(resultesContainer, "vectordb_bench") + # init style + initStyle(st) - # charts - drawCharts(st, shownData, failedTasks, showCaseNames) + # page + welcomePrams(st) - # footer - footer(st.container()) + # description + explainPrams(st) if __name__ == "__main__":