@@ -6,6 +6,14 @@ Hit@N = expected symbol found in top N results.
66
77| Version | Model | Symbols | Hit@1 | Hit@3 | Hit@5 | Misses | Embed Time |
88| ---------| -------| --------:| ------:| ------:| ------:| -------:| -----------:|
9+ | 3.11.1 | minilm | 1500 | 65.4% ~ | 85.7% ↓0.6pp | 91.1% ↓0.7pp | 72 | 149.6s |
10+ | 3.11.1 | jina-small | 1500 | 76.2% ↓0.9pp | 92.3% ~ | 94.9% ↓0.9pp | 35 | 296.2s |
11+ | 3.11.1 | jina-base | 1500 | 70.5% ↓0.9pp | 89.9% ↓1.5pp | 93.3% ↓2.1pp | 49 | 1525.9s |
12+ | 3.11.1 | jina-code | 1500 | 66.6% | 85.7% | 90.8% | 68 | 1310.7s |
13+ | 3.11.1 | nomic | 1500 | 79.0% ↓2.3pp | 95.0% ~ | 97.2% ~ | 15 | 1536.3s |
14+ | 3.11.1 | nomic-v1.5 | 1500 | 76.9% ↓2.5pp | 93.1% ↓0.9pp | 95.9% ↓0.8pp | 21 | 1522.4s |
15+ | 3.11.1 | mxbai-xsmall | 1500 | 49.9% | 70.7% | 76.9% | 242 | 213.9s |
16+ | 3.11.1 | modernbert | 1500 | 74.0% | 91.6% | 94.7% | 40 | 1415.6s |
917| 3.10.0 | minilm | 1500 | 65.9% ↓1.5pp | 86.3% ↓0.9pp | 91.8% ↓0.6pp | 58 | 128.0s |
1018| 3.10.0 | jina-small | 1500 | 77.1% ↑1.3pp | 92.8% ↓0.8pp | 95.8% ↓0.7pp | 29 | 262.7s |
1119| 3.10.0 | jina-base | 1500 | 71.4% ↓0.7pp | 91.3% ↑1.0pp | 95.4% ~ | 35 | 1346.3s |
@@ -19,18 +27,125 @@ Hit@N = expected symbol found in top N results.
1927
2028### Latest results
2129
22- ** Version:** 3.10.0 | ** Strategy:** structured | ** Symbols:** 1500 | ** Date:** 2026-05-11
30+ ** Version:** 3.11.1 | ** Strategy:** structured | ** Symbols:** 1500 | ** Date:** 2026-05-31
2331
2432| Model | Dim | Context | Hit@1 | Hit@3 | Hit@5 | Hit@10 | Misses | Embed | Search |
2533| -------| ----:| --------:| ------:| ------:| ------:| -------:| -------:| ------:| -------:|
26- | minilm | 384 | 256 | 65.9% | 86.3% | 91.8% | 96.1% | 58 | 128.0s | 133.9s |
27- | jina-small | 512 | 8192 | 77.1% | 92.8% | 95.8% | 98.1% | 29 | 262.7s | 161.9s |
28- | jina-base | 768 | 8192 | 71.4% | 91.3% | 95.4% | 97.7% | 35 | 1346.3s | 209.5s |
29- | nomic | 768 | 8192 | 81.3% | 94.7% | 97.5% | 98.8% | 18 | 1340.3s | 210.5s |
30- | nomic-v1.5 | 768 | 8192 | 79.3% | 94.0% | 96.7% | 98.4% | 24 | 1335.2s | 208.1s |
34+ | minilm | 384 | 256 | 65.4% | 85.7% | 91.1% | 95.2% | 72 | 149.6s | 153.1s |
35+ | jina-small | 512 | 8192 | 76.2% | 92.3% | 94.9% | 97.7% | 35 | 296.2s | 180.7s |
36+ | jina-base | 768 | 8192 | 70.5% | 89.9% | 93.3% | 96.7% | 49 | 1525.9s | 232.8s |
37+ | jina-code | 768 | 8192 | 66.6% | 85.7% | 90.8% | 95.5% | 68 | 1310.7s | 231.2s |
38+ | nomic | 768 | 8192 | 79.0% | 95.0% | 97.2% | 99.0% | 15 | 1536.3s | 231.7s |
39+ | nomic-v1.5 | 768 | 8192 | 76.9% | 93.1% | 95.9% | 98.6% | 21 | 1522.4s | 230.0s |
40+ | mxbai-xsmall | 384 | 4096 | 49.9% | 70.7% | 76.9% | 83.9% | 242 | 213.9s | 153.1s |
41+ | modernbert | 768 | 8192 | 74.0% | 91.6% | 94.7% | 97.3% | 40 | 1415.6s | 231.2s |
3142
3243<!-- EMBEDDING_BENCHMARK_DATA
3344[
45+ {
46+ "version": "3.11.1",
47+ "date": "2026-05-31",
48+ "strategy": "structured",
49+ "symbols": 1500,
50+ "models": {
51+ "minilm": {
52+ "dim": 384,
53+ "contextWindow": 256,
54+ "hits1": 981,
55+ "hits3": 1285,
56+ "hits5": 1367,
57+ "hits10": 1428,
58+ "misses": 72,
59+ "total": 1500,
60+ "embedTimeMs": 149582,
61+ "searchTimeMs": 153149
62+ },
63+ "jina-small": {
64+ "dim": 512,
65+ "contextWindow": 8192,
66+ "hits1": 1143,
67+ "hits3": 1385,
68+ "hits5": 1424,
69+ "hits10": 1465,
70+ "misses": 35,
71+ "total": 1500,
72+ "embedTimeMs": 296235,
73+ "searchTimeMs": 180693
74+ },
75+ "jina-base": {
76+ "dim": 768,
77+ "contextWindow": 8192,
78+ "hits1": 1057,
79+ "hits3": 1348,
80+ "hits5": 1399,
81+ "hits10": 1451,
82+ "misses": 49,
83+ "total": 1500,
84+ "embedTimeMs": 1525891,
85+ "searchTimeMs": 232757
86+ },
87+ "jina-code": {
88+ "dim": 768,
89+ "contextWindow": 8192,
90+ "hits1": 999,
91+ "hits3": 1285,
92+ "hits5": 1362,
93+ "hits10": 1432,
94+ "misses": 68,
95+ "total": 1500,
96+ "embedTimeMs": 1310746,
97+ "searchTimeMs": 231202
98+ },
99+ "nomic": {
100+ "dim": 768,
101+ "contextWindow": 8192,
102+ "hits1": 1185,
103+ "hits3": 1425,
104+ "hits5": 1458,
105+ "hits10": 1485,
106+ "misses": 15,
107+ "total": 1500,
108+ "embedTimeMs": 1536284,
109+ "searchTimeMs": 231696
110+ },
111+ "nomic-v1.5": {
112+ "dim": 768,
113+ "contextWindow": 8192,
114+ "hits1": 1153,
115+ "hits3": 1397,
116+ "hits5": 1439,
117+ "hits10": 1479,
118+ "misses": 21,
119+ "total": 1500,
120+ "embedTimeMs": 1522404,
121+ "searchTimeMs": 229968
122+ },
123+ "mxbai-xsmall": {
124+ "dim": 384,
125+ "contextWindow": 4096,
126+ "hits1": 748,
127+ "hits3": 1061,
128+ "hits5": 1154,
129+ "hits10": 1258,
130+ "misses": 242,
131+ "total": 1500,
132+ "embedTimeMs": 213870,
133+ "searchTimeMs": 153090
134+ },
135+ "modernbert": {
136+ "dim": 768,
137+ "contextWindow": 8192,
138+ "hits1": 1110,
139+ "hits3": 1374,
140+ "hits5": 1421,
141+ "hits10": 1460,
142+ "misses": 40,
143+ "total": 1500,
144+ "embedTimeMs": 1415646,
145+ "searchTimeMs": 231199
146+ }
147+ }
148+ },
34149 {
35150 "version": "3.10.0",
36151 "date": "2026-05-11",
0 commit comments