From cec80c27121aef78aad5125eb7cecd7c26a0f4be Mon Sep 17 00:00:00 2001 From: hxaxd Date: Mon, 13 Jul 2026 23:21:53 +0800 Subject: [PATCH] fix: prune deleted BM25 vocabulary entries --- .../in_memory/document_store.py | 3 +++ ...ency-bm25-vocabulary-2669497a2350057b.yaml | 5 +++++ test/document_stores/test_in_memory.py | 20 +++++++++++++++++++ 3 files changed, 28 insertions(+) create mode 100644 releasenotes/notes/prune-zero-frequency-bm25-vocabulary-2669497a2350057b.yaml diff --git a/haystack/document_stores/in_memory/document_store.py b/haystack/document_stores/in_memory/document_store.py index 05b8875dac0..48e7275d322 100644 --- a/haystack/document_stores/in_memory/document_store.py +++ b/haystack/document_stores/in_memory/document_store.py @@ -507,6 +507,9 @@ def delete_documents(self, document_ids: list[str]) -> None: doc_len = doc_stats.doc_len self._freq_vocab_for_idf.subtract(Counter(freq.keys())) + for token in freq: + if self._freq_vocab_for_idf[token] <= 0: + del self._freq_vocab_for_idf[token] try: self._avg_doc_len = (self._avg_doc_len * (len(self._bm25_attr) + 1) - doc_len) / len(self._bm25_attr) except ZeroDivisionError: diff --git a/releasenotes/notes/prune-zero-frequency-bm25-vocabulary-2669497a2350057b.yaml b/releasenotes/notes/prune-zero-frequency-bm25-vocabulary-2669497a2350057b.yaml new file mode 100644 index 00000000000..656bb86259a --- /dev/null +++ b/releasenotes/notes/prune-zero-frequency-bm25-vocabulary-2669497a2350057b.yaml @@ -0,0 +1,5 @@ +fixes: + - | + Remove zero-frequency vocabulary entries after deleting documents from + ``InMemoryDocumentStore`` so previous document history no longer changes + BM25Okapi scores for the active corpus. diff --git a/test/document_stores/test_in_memory.py b/test/document_stores/test_in_memory.py index 40968b4186d..b5219178916 100644 --- a/test/document_stores/test_in_memory.py +++ b/test/document_stores/test_in_memory.py @@ -763,6 +763,26 @@ def test_bm25_avg_doc_len_after_delete(self, in_memory_doc_store): # After removing "hello world" (2 tokens), only "foo bar baz" (3 tokens) remains assert in_memory_doc_store._avg_doc_len == pytest.approx(3.0) + def test_bm25_okapi_scores_do_not_depend_on_deleted_documents(self): + active_documents = [ + Document(id="d1", content="common alpha"), + Document(id="d2", content="common beta"), + Document(id="d3", content="common gamma"), + ] + fresh_store = InMemoryDocumentStore(bm25_algorithm="BM25Okapi", shared=False) + reused_store = InMemoryDocumentStore(bm25_algorithm="BM25Okapi", shared=False) + deleted_document = Document(content="one two three four five six seven eight nine ten") + + reused_store.write_documents([deleted_document]) + reused_store.delete_documents([deleted_document.id]) + fresh_store.write_documents(active_documents) + reused_store.write_documents(active_documents) + + fresh_scores = {doc.id: doc.score for doc in fresh_store.bm25_retrieval(query="common", top_k=3)} + reused_scores = {doc.id: doc.score for doc in reused_store.bm25_retrieval(query="common", top_k=3)} + + assert reused_scores == pytest.approx(fresh_scores) + class TestMemoryDocumentStoreNotShared(TestMemoryDocumentStore): """