Skip to content

Commit 7b22043

Browse files
committed
add minimal case-normalization for BM25Retriever
1 parent b3c6cf1 commit 7b22043

1 file changed

Lines changed: 3 additions & 1 deletion

File tree

src/retrievers/csv_chroma.py

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -37,7 +37,9 @@ def create_bm25_chroma_ensemble_retriever(
3737
reactome_csvs_dir: Path = embeddings_directory / "csv_files"
3838
loader = CSVLoader(file_path=reactome_csvs_dir / csv_file_name)
3939
data = loader.load()
40-
bm25_retriever = BM25Retriever.from_documents(data)
40+
bm25_retriever = BM25Retriever.from_documents(
41+
data, preprocess_func=lambda text: text.casefold().split()
42+
)
4143
bm25_retriever.k = 10
4244

4345
# set up vectorstore SelfQuery retriever

0 commit comments

Comments
 (0)