semantic-search-eval/configs/example.yaml at main · machinelearningZH/semantic-search-eval · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
# Output directory to save results and plots
folder: results

# Testset settings
name: example # What the testset will be named
docs: example.docs.parquet # Can be a path to a CSV file or a parquet file with column "text"
queries: null # Can be a path to a CSV file or a parquet file with columns "search_query" and "idx" (pointing to document id related to query), if set to "null" queries will automatically be generated (see next setting!)
is-public-data: false # Whether queries can be generated with OpenAI (only matters if no queries are provided)
queries-per-doc: 10 # Number of queries to generate per document if queries is set to null
spacy-model: de_core_news_sm # Used to create random keyword queries if no queries are provided
openai-model: gpt-4.1-mini # Used to generate queries if is-public-data is set to true

# Metrics for the evaluation
metrics:
  - accuracy@1 # Top-1 accuracy: checks if gold doc is rank 1
  - accuracy@5 # Top-5 accuracy: checks if gold doc is in top 5
  - accuracy@10 # Top-10 accuracy: same logic for top 10
  - ndcg@10 # NDCG@10: rewards higher-ranked gold docs (position-aware)
  - latency # Time taken for full inference per model

# Global max token length for truncation (needs to be based on smallest model max len for fair comparison)
max-len: 512

# Configs for all models to evaluate
# Add new models by specifying the corresponding HugginFace or OpenAI model path.
models:
  lexical:
    bm25: de_core_news_sm # uses spacy model to lemmatize before indexing

  huggingface:
    jina-v2: jinaai/jina-embeddings-v2-base-de # max-len 8192
    all-MiniLM-v2: sentence-transformers/all-MiniLM-L6-v2 # max-len 512
    granite: ibm-granite/granite-embedding-278m-multilingual # max-len 512
    nomic:
      model: nomic-ai/nomic-embed-text-v2-moe # max-len 512
      set_builtin_query_prompt: query
      set_builtin_passage_prompt: passage
    snowflake:
      model: Snowflake/snowflake-arctic-embed-l-v2.0 # max-len 8192
      set_builtin_query_prompt: query
    jina-v3:
      model: jinaai/jina-embeddings-v3 # max-len 8192
      set_builtin_query_prompt: retrieval.query
      set_builtin_passage_prompt: retrieval.passage
      set_passage_task_prompt: retrieval.passage
      set_query_task_prompt: retrieval.query
    jina-v5-small:
      model: jinaai/jina-embeddings-v5-text-small # max-len 32768
      set_query_task_prompt: retrieval
      set_passage_task_prompt: retrieval
      set_builtin_passage_prompt: document
      set_builtin_query_prompt: query
    intfloat-small:
      model: intfloat/multilingual-e5-small # max-len 512
      set_custom_query_prefix: "query: "
      set_custom_passage_prefix: "passage: "
    intfloat-base:
      model: intfloat/multilingual-e5-base # max-len 512
      set_custom_query_prefix: "query: "
      set_custom_passage_prefix: "passage: "
    intfloat-instruct:
      model: intfloat/multilingual-e5-large-instruct # max-len 512
      set_custom_query_prefix: "Instruct: Given a web search query, retrieve relevant passages that answer the query\nQuery: "

  # open-ai:
  #   open-ai-3-small: text-embedding-3-small # max-len 8191