Skip to content

Commit 4fc1161

Browse files
Merge pull request #11 from VectifyAI/v0.3
revert bench
2 parents fb90c52 + 3d5a480 commit 4fc1161

14 files changed

Lines changed: 2025 additions & 0 deletions

bench/benchmark_retrievers.py

Lines changed: 645 additions & 0 deletions
Large diffs are not rendered by default.

bench/build_context7_fs.py

Lines changed: 104 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,104 @@
1+
#!/usr/bin/env python3
2+
"""Build Context7 filesystem corpus as one JSON tree file."""
3+
4+
from __future__ import annotations
5+
6+
import json
7+
import os
8+
import re
9+
import urllib.request
10+
from datetime import datetime, timezone
11+
from pathlib import Path
12+
from urllib.parse import urlparse
13+
14+
BASE_DIR = Path(__file__).resolve().parent / "filesystem" / "context7"
15+
LLMS_TXT_URL = "https://context7.com/docs/llms.txt"
16+
TREE_FILENAME = "tree.json"
17+
# 0 means "all docs found in llms.txt".
18+
MAX_DOCS = int(os.environ.get("CONTEXT7_MAX_DOCS", "0"))
19+
TIMEOUT = 15
20+
21+
22+
def fetch_text(url: str) -> str:
23+
req = urllib.request.Request(
24+
url, headers={"User-Agent": "ConDB-context7-builder/1.0"}
25+
)
26+
with urllib.request.urlopen(req, timeout=TIMEOUT) as resp:
27+
return resp.read().decode("utf-8", errors="replace")
28+
29+
30+
def parse_doc_urls(llms_txt: str) -> list[str]:
31+
urls = re.findall(r"\((https://context7\.com/docs/[^)]+)\)", llms_txt)
32+
dedup: list[str] = []
33+
seen: set[str] = set()
34+
for url in urls:
35+
if url in seen:
36+
continue
37+
seen.add(url)
38+
dedup.append(url)
39+
return dedup
40+
41+
42+
def logical_path_for_url(url: str) -> str:
43+
path = urlparse(url).path.lstrip("/")
44+
if not path:
45+
path = "index.md"
46+
if path.endswith("/"):
47+
path += "index.md"
48+
if "." not in Path(path).name:
49+
path += ".md"
50+
return path
51+
52+
53+
def write_tree_file(out_path: Path, docs: list[dict[str, str]]) -> None:
54+
payload = {
55+
"generated_at": datetime.now(timezone.utc).isoformat(),
56+
"source_index": LLMS_TXT_URL,
57+
"docs_count": len(docs),
58+
"docs": docs,
59+
}
60+
out_path.write_text(json.dumps(payload, ensure_ascii=False, indent=2), encoding="utf-8")
61+
62+
63+
def main() -> None:
64+
BASE_DIR.mkdir(parents=True, exist_ok=True)
65+
66+
llms_txt = fetch_text(LLMS_TXT_URL)
67+
all_doc_urls = parse_doc_urls(llms_txt)
68+
doc_urls = [url for url in all_doc_urls if url.endswith(".md")]
69+
if not doc_urls:
70+
raise RuntimeError(f"no markdown doc urls found from {LLMS_TXT_URL}")
71+
if MAX_DOCS > 0:
72+
doc_urls = doc_urls[:MAX_DOCS]
73+
74+
docs: list[dict[str, str]] = []
75+
errors: list[dict[str, str]] = []
76+
77+
for url in doc_urls:
78+
try:
79+
content = fetch_text(url)
80+
path = logical_path_for_url(url)
81+
docs.append({"url": url, "path": path, "content": content})
82+
print(f"+ {url} -> {TREE_FILENAME}::{path}")
83+
except Exception as e: # noqa: BLE001
84+
errors.append({"url": url, "error": str(e)})
85+
print(f"! failed: {url} ({e})")
86+
87+
if not docs:
88+
raise RuntimeError(f"failed to fetch any context7 docs; errors={len(errors)}")
89+
90+
tree_path = BASE_DIR / TREE_FILENAME
91+
write_tree_file(tree_path, docs)
92+
93+
print(
94+
f"done: docs={len(docs)} errors={len(errors)} "
95+
f"tree={tree_path}"
96+
)
97+
if errors:
98+
print("error_summary:")
99+
for err in errors:
100+
print(f"- {err['url']} :: {err['error']}")
101+
102+
103+
if __name__ == "__main__":
104+
main()
Lines changed: 24 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,24 @@
1+
{
2+
"queries": [
3+
{
4+
"query": "Where is the API guide for Context7?",
5+
"ground_truth": ["docs/api-guide.md"]
6+
},
7+
{
8+
"query": "Where is the overview page for Context7 agentic tools?",
9+
"ground_truth": ["docs/agentic-tools/overview.md"]
10+
},
11+
{
12+
"query": "Where is the query-docs tool documented in AI SDK tools?",
13+
"ground_truth": ["docs/agentic-tools/ai-sdk/tools/query-docs.md"]
14+
},
15+
{
16+
"query": "Where is the resolve-library-id tool documented?",
17+
"ground_truth": ["docs/agentic-tools/ai-sdk/tools/resolve-library-id.md"]
18+
},
19+
{
20+
"query": "Where is the API reference for adding a GitHub repository?",
21+
"ground_truth": ["docs/api-reference/add-library/add-a-github-repository.md"]
22+
}
23+
]
24+
}

bench/filesystem/context7/tree.json

Lines changed: 242 additions & 0 deletions
Large diffs are not rendered by default.

bench/fs_block_beam_vertical.json

Lines changed: 67 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,67 @@
1+
{
2+
"generated_at_utc": "2026-03-18T11:28:35.747027+00:00",
3+
"rows": [
4+
{
5+
"scenario": "context7",
6+
"retriever": "Block",
7+
"queries_ok": 5,
8+
"queries_total": 5,
9+
"avg_time_s": 5.46975998878479,
10+
"avg_llm_calls": 1.0,
11+
"total_cost": 0.07618710000000001,
12+
"hit1": 1.0,
13+
"hit10": 1.0
14+
},
15+
{
16+
"scenario": "context7",
17+
"retriever": "Beam",
18+
"queries_ok": 5,
19+
"queries_total": 5,
20+
"avg_time_s": 20.17977318763733,
21+
"avg_llm_calls": 4.6,
22+
"total_cost": 0.132762,
23+
"hit1": 0.6,
24+
"hit10": 0.8
25+
},
26+
{
27+
"scenario": "context7",
28+
"retriever": "Vertical",
29+
"queries_ok": 5,
30+
"queries_total": 5,
31+
"avg_time_s": 7.308028888702393,
32+
"avg_llm_calls": 1.6,
33+
"total_cost": 0.14863275,
34+
"hit1": 1.0,
35+
"hit10": 1.0
36+
}
37+
],
38+
"aggregate": [
39+
{
40+
"retriever": "Block",
41+
"queries_total": 5,
42+
"avg_time_s": 5.46975998878479,
43+
"avg_llm_calls": 1.0,
44+
"hit1": 1.0,
45+
"hit10": 1.0,
46+
"total_cost_usd": 0.07618710000000001
47+
},
48+
{
49+
"retriever": "Beam",
50+
"queries_total": 5,
51+
"avg_time_s": 20.17977318763733,
52+
"avg_llm_calls": 4.6,
53+
"hit1": 0.6,
54+
"hit10": 0.8,
55+
"total_cost_usd": 0.132762
56+
},
57+
{
58+
"retriever": "Vertical",
59+
"queries_total": 5,
60+
"avg_time_s": 7.308028888702393,
61+
"avg_llm_calls": 1.6,
62+
"hit1": 1.0,
63+
"hit10": 1.0,
64+
"total_cost_usd": 0.14863275
65+
}
66+
]
67+
}

bench/fs_block_beam_vertical.md

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,16 @@
1+
# Filesystem Retriever Comparison (Block vs Beam vs Vertical)
2+
3+
Data sources:
4+
- `bench/fs_block_beam_vertical_context7.json`
5+
6+
Run setup: `fs_query_order=prefix`, `beam_size=3`, `max_turns=10`, total `5` queries on `context7` only.
7+
8+
## Overall (5 queries: context7)
9+
10+
| Retriever | Avg Time (s) | Avg LLM Calls | Total Cost (USD) | Hit@1 | Hit@10 |
11+
|---|---:|---:|---:|---:|---:|
12+
| Block | 5.4698 | 1.0000 | 0.0762 | 1.0000 | 1.0000 |
13+
| Vertical | 7.3080 | 1.6000 | 0.1486 | 1.0000 | 1.0000 |
14+
| Beam | 20.1798 | 4.6000 | 0.1328 | 0.6000 | 0.8000 |
15+
16+
Conclusion: on the current `context7` filesystem set, `Block` is still the best default. It keeps perfect `Hit@1` / `Hit@10`, while staying faster and cheaper than `Vertical`. `Beam` remains cheaper than `Vertical`, but it still loses clearly on accuracy.

0 commit comments

Comments
 (0)