Skip to content

Commit 860e2ad

Browse files
hh-space-invaderjoein
hh-space-invader
and
joein
authored
Tsk 375 add jina rerankers (#379)
* feat: Added jina reranker models * chore: Added jina reranker canonical score values * chore: added rounding of the output for easier reproducability * chore: Added jina reranker models in batch test * chore: remove redundant np.round * chore: test only <1gb files in local * chore: Updated docs to add rerankers * fix: recompute canonical values with fp16 * new: extend NOTICE with jina reranker v2 --------- Co-authored-by: George Panchuk <george.panchuk@qdrant.tech>
1 parent d141e29 commit 860e2ad

4 files changed

Lines changed: 110 additions & 39 deletions

File tree

‎NOTICE‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,8 @@ This product includes software developed by Qdrant
55
This distribution includes the following Jina AI models, each with its respective license:
66
- jinaai/jina-colbert-v2
77
- License: cc-by-nc-4.0
8+
- jinaai/jina-reranker-v2-base-multilingual
9+
- License: cc-by-nc-4.0
810

911
These models are developed by Jina (https://jina.ai/) and are subject to Jina AI's licensing terms.
1012

‎docs/examples/Supported_Models.ipynb‎

Lines changed: 70 additions & 38 deletions
Original file line numberDiff line numberDiff line change
@@ -4,8 +4,8 @@
44
"cell_type": "code",
55
"metadata": {
66
"ExecuteTime": {
7-
"end_time": "2024-11-13T08:52:46.651285Z",
8-
"start_time": "2024-11-13T08:52:46.633741Z"
7+
"end_time": "2024-11-13T09:01:03.324551Z",
8+
"start_time": "2024-11-13T09:01:03.234711Z"
99
}
1010
},
1111
"source": [
@@ -22,14 +22,14 @@
2222
]
2323
}
2424
],
25-
"execution_count": 3
25+
"execution_count": 10
2626
},
2727
{
2828
"cell_type": "code",
2929
"metadata": {
3030
"ExecuteTime": {
31-
"end_time": "2024-11-13T08:52:46.973001Z",
32-
"start_time": "2024-11-13T08:52:46.962113Z"
31+
"end_time": "2024-11-13T09:01:04.505772Z",
32+
"start_time": "2024-11-13T09:01:04.493296Z"
3333
}
3434
},
3535
"source": [
@@ -44,7 +44,7 @@
4444
"from fastembed.rerank.cross_encoder import TextCrossEncoder"
4545
],
4646
"outputs": [],
47-
"execution_count": 4
47+
"execution_count": 11
4848
},
4949
{
5050
"cell_type": "markdown",
@@ -57,8 +57,8 @@
5757
"cell_type": "code",
5858
"metadata": {
5959
"ExecuteTime": {
60-
"end_time": "2024-11-13T08:52:49.705020Z",
61-
"start_time": "2024-11-13T08:52:49.665915Z"
60+
"end_time": "2024-11-13T09:01:05.812271Z",
61+
"start_time": "2024-11-13T09:01:05.795846Z"
6262
}
6363
},
6464
"source": [
@@ -360,12 +360,12 @@
360360
"</div>"
361361
]
362362
},
363-
"execution_count": 5,
363+
"execution_count": 12,
364364
"metadata": {},
365365
"output_type": "execute_result"
366366
}
367367
],
368-
"execution_count": 5
368+
"execution_count": 12
369369
},
370370
{
371371
"cell_type": "markdown",
@@ -378,8 +378,8 @@
378378
"cell_type": "code",
379379
"metadata": {
380380
"ExecuteTime": {
381-
"end_time": "2024-11-13T08:52:51.690851Z",
382-
"start_time": "2024-11-13T08:52:51.677404Z"
381+
"end_time": "2024-11-13T09:01:07.038954Z",
382+
"start_time": "2024-11-13T09:01:07.019656Z"
383383
}
384384
},
385385
"source": [
@@ -481,12 +481,12 @@
481481
"</div>"
482482
]
483483
},
484-
"execution_count": 6,
484+
"execution_count": 13,
485485
"metadata": {},
486486
"output_type": "execute_result"
487487
}
488488
],
489-
"execution_count": 6
489+
"execution_count": 13
490490
},
491491
{
492492
"cell_type": "markdown",
@@ -502,8 +502,8 @@
502502
"metadata": {
503503
"collapsed": false,
504504
"ExecuteTime": {
505-
"end_time": "2024-11-13T08:52:52.866135Z",
506-
"start_time": "2024-11-13T08:52:52.852411Z"
505+
"end_time": "2024-11-13T09:01:08.074442Z",
506+
"start_time": "2024-11-13T09:01:08.056138Z"
507507
}
508508
},
509509
"source": [
@@ -593,12 +593,12 @@
593593
"</div>"
594594
]
595595
},
596-
"execution_count": 7,
596+
"execution_count": 14,
597597
"metadata": {},
598598
"output_type": "execute_result"
599599
}
600600
],
601-
"execution_count": 7
601+
"execution_count": 14
602602
},
603603
{
604604
"cell_type": "markdown",
@@ -614,8 +614,8 @@
614614
"metadata": {
615615
"collapsed": false,
616616
"ExecuteTime": {
617-
"end_time": "2024-11-13T08:52:54.060642Z",
618-
"start_time": "2024-11-13T08:52:54.043437Z"
617+
"end_time": "2024-11-13T09:01:09.171647Z",
618+
"start_time": "2024-11-13T09:01:09.150940Z"
619619
}
620620
},
621621
"source": [
@@ -706,26 +706,26 @@
706706
"</div>"
707707
]
708708
},
709-
"execution_count": 8,
709+
"execution_count": 15,
710710
"metadata": {},
711711
"output_type": "execute_result"
712712
}
713713
],
714-
"execution_count": 8
714+
"execution_count": 15
715715
},
716716
{
717717
"cell_type": "markdown",
718718
"metadata": {},
719719
"source": [
720-
"## Supported Rerankers Models"
720+
"## Supported Rerank Cross Encoder Models"
721721
]
722722
},
723723
{
724724
"cell_type": "code",
725725
"metadata": {
726726
"ExecuteTime": {
727-
"end_time": "2024-11-13T08:52:55.933033Z",
728-
"start_time": "2024-11-13T08:52:55.915572Z"
727+
"end_time": "2024-11-13T09:01:10.313943Z",
728+
"start_time": "2024-11-13T09:01:10.298428Z"
729729
}
730730
},
731731
"source": [
@@ -740,15 +740,21 @@
740740
{
741741
"data": {
742742
"text/plain": [
743-
" model size_in_GB \\\n",
744-
"0 Xenova/ms-marco-MiniLM-L-6-v2 0.08 \n",
745-
"1 Xenova/ms-marco-MiniLM-L-12-v2 0.12 \n",
746-
"2 BAAI/bge-reranker-base 1.04 \n",
743+
" model size_in_GB \\\n",
744+
"0 Xenova/ms-marco-MiniLM-L-6-v2 0.08 \n",
745+
"1 Xenova/ms-marco-MiniLM-L-12-v2 0.12 \n",
746+
"2 jinaai/jina-reranker-v1-tiny-en 0.13 \n",
747+
"3 jinaai/jina-reranker-v1-turbo-en 0.15 \n",
748+
"4 BAAI/bge-reranker-base 1.04 \n",
749+
"5 jinaai/jina-reranker-v2-base-multilingual 1.11 \n",
747750
"\n",
748-
" description license \n",
749-
"0 MiniLM-L-6-v2 model optimized for re-ranking t... apache-2.0 \n",
750-
"1 MiniLM-L-12-v2 model optimized for re-ranking ... apache-2.0 \n",
751-
"2 BGE reranker base model for cross-encoder re-r... mit "
751+
" description license \n",
752+
"0 MiniLM-L-6-v2 model optimized for re-ranking t... apache-2.0 \n",
753+
"1 MiniLM-L-12-v2 model optimized for re-ranking ... apache-2.0 \n",
754+
"2 Designed for blazing-fast re-ranking with 8K c... apache-2.0 \n",
755+
"3 Designed for blazing-fast re-ranking with 8K c... apache-2.0 \n",
756+
"4 BGE reranker base model for cross-encoder re-r... mit \n",
757+
"5 A multi-lingual reranker model for cross-encod... cc-by-nc-4.0 "
752758
],
753759
"text/html": [
754760
"<div>\n",
@@ -792,22 +798,43 @@
792798
" </tr>\n",
793799
" <tr>\n",
794800
" <th>2</th>\n",
801+
" <td>jinaai/jina-reranker-v1-tiny-en</td>\n",
802+
" <td>0.13</td>\n",
803+
" <td>Designed for blazing-fast re-ranking with 8K c...</td>\n",
804+
" <td>apache-2.0</td>\n",
805+
" </tr>\n",
806+
" <tr>\n",
807+
" <th>3</th>\n",
808+
" <td>jinaai/jina-reranker-v1-turbo-en</td>\n",
809+
" <td>0.15</td>\n",
810+
" <td>Designed for blazing-fast re-ranking with 8K c...</td>\n",
811+
" <td>apache-2.0</td>\n",
812+
" </tr>\n",
813+
" <tr>\n",
814+
" <th>4</th>\n",
795815
" <td>BAAI/bge-reranker-base</td>\n",
796816
" <td>1.04</td>\n",
797817
" <td>BGE reranker base model for cross-encoder re-r...</td>\n",
798818
" <td>mit</td>\n",
799819
" </tr>\n",
820+
" <tr>\n",
821+
" <th>5</th>\n",
822+
" <td>jinaai/jina-reranker-v2-base-multilingual</td>\n",
823+
" <td>1.11</td>\n",
824+
" <td>A multi-lingual reranker model for cross-encod...</td>\n",
825+
" <td>cc-by-nc-4.0</td>\n",
826+
" </tr>\n",
800827
" </tbody>\n",
801828
"</table>\n",
802829
"</div>"
803830
]
804831
},
805-
"execution_count": 9,
832+
"execution_count": 16,
806833
"metadata": {},
807834
"output_type": "execute_result"
808835
}
809836
],
810-
"execution_count": 9
837+
"execution_count": 16
811838
},
812839
{
813840
"metadata": {},
@@ -819,7 +846,7 @@
819846
],
820847
"metadata": {
821848
"kernelspec": {
822-
"display_name": ".venv",
849+
"display_name": "Python 3.8.18 ('base')",
823850
"language": "python",
824851
"name": "python3"
825852
},
@@ -833,9 +860,14 @@
833860
"name": "python",
834861
"nbconvert_exporter": "python",
835862
"pygments_lexer": "ipython3",
836-
"version": "3.11.2"
863+
"version": "3.11.8"
837864
},
838-
"orig_nbformat": 4
865+
"orig_nbformat": 4,
866+
"vscode": {
867+
"interpreter": {
868+
"hash": "c4a27af61e455bc18dcf16f5867a2ff0402fa12b01dd0f6ce3a79ae73ad15e91"
869+
}
870+
}
839871
},
840872
"nbformat": 4,
841873
"nbformat_minor": 2

‎fastembed/rerank/cross_encoder/onnx_text_cross_encoder.py‎

Lines changed: 30 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -38,6 +38,36 @@
3838
"description": "BGE reranker base model for cross-encoder re-ranking.",
3939
"license": "mit",
4040
},
41+
{
42+
"model": "jinaai/jina-reranker-v1-tiny-en",
43+
"size_in_GB": 0.13,
44+
"sources": {
45+
"hf": "jinaai/jina-reranker-v1-tiny-en",
46+
},
47+
"model_file": "onnx/model.onnx",
48+
"description": "Designed for blazing-fast re-ranking with 8K context length and fewer parameters than jina-reranker-v1-turbo-en.",
49+
"license": "apache-2.0",
50+
},
51+
{
52+
"model": "jinaai/jina-reranker-v1-turbo-en",
53+
"size_in_GB": 0.15,
54+
"sources": {
55+
"hf": "jinaai/jina-reranker-v1-turbo-en",
56+
},
57+
"model_file": "onnx/model.onnx",
58+
"description": "Designed for blazing-fast re-ranking with 8K context length.",
59+
"license": "apache-2.0",
60+
},
61+
{
62+
"model": "jinaai/jina-reranker-v2-base-multilingual",
63+
"size_in_GB": 1.11,
64+
"sources": {
65+
"hf": "jinaai/jina-reranker-v2-base-multilingual",
66+
},
67+
"model_file": "onnx/model.onnx",
68+
"description": "A multi-lingual reranker model for cross-encoder re-ranking with 1K context length and sliding window",
69+
"license": "cc-by-nc-4.0",
70+
},
4171
]
4272

4373

‎tests/test_text_cross_encoder.py‎

Lines changed: 8 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,9 @@
1010
"Xenova/ms-marco-MiniLM-L-6-v2": np.array([8.500708, -2.541011]),
1111
"Xenova/ms-marco-MiniLM-L-12-v2": np.array([9.330912, -2.0380247]),
1212
"BAAI/bge-reranker-base": np.array([6.15733337, -3.65939403]),
13+
"jinaai/jina-reranker-v1-tiny-en": np.array([2.5911, 0.1122]),
14+
"jinaai/jina-reranker-v1-turbo-en": np.array([1.8295, -2.8908]),
15+
"jinaai/jina-reranker-v2-base-multilingual": np.array([1.6533, -1.6455]),
1316
}
1417

1518

@@ -37,7 +40,11 @@ def test_rerank():
3740

3841
@pytest.mark.parametrize(
3942
"model_name",
40-
["Xenova/ms-marco-MiniLM-L-6-v2", "Xenova/ms-marco-MiniLM-L-12-v2", "BAAI/bge-reranker-base"],
43+
[
44+
model_desc["model"]
45+
for model_desc in TextCrossEncoder.list_supported_models()
46+
if model_desc["size_in_GB"] < 1 and model_desc["model"] in CANONICAL_SCORE_VALUES.keys()
47+
],
4148
)
4249
def test_batch_rerank(model_name):
4350
is_ci = os.getenv("CI")

0 commit comments

Comments
 (0)