From dce382a21b868d1e99ebfd76ba7bdfde59c20927 Mon Sep 17 00:00:00 2001 From: Aarkin7 Date: Sun, 19 Jul 2026 19:54:33 +0530 Subject: [PATCH 1/4] fix: apply FilterPolicy.MERGE correctly in in-memory retrievers --- .../retrievers/in_memory/bm25_retriever.py | 12 +++--------- .../retrievers/in_memory/embedding_retriever.py | 12 +++--------- 2 files changed, 6 insertions(+), 18 deletions(-) diff --git a/haystack/components/retrievers/in_memory/bm25_retriever.py b/haystack/components/retrievers/in_memory/bm25_retriever.py index 05618f7d810..450f70cc561 100644 --- a/haystack/components/retrievers/in_memory/bm25_retriever.py +++ b/haystack/components/retrievers/in_memory/bm25_retriever.py @@ -6,7 +6,7 @@ from haystack import Document, component, default_from_dict, default_to_dict from haystack.document_stores.in_memory import InMemoryDocumentStore -from haystack.document_stores.types import FilterPolicy +from haystack.document_stores.types import FilterPolicy, apply_filter_policy @component @@ -143,10 +143,7 @@ def run( :raises ValueError: If the specified DocumentStore is not found or is not a InMemoryDocumentStore instance. """ - if self.filter_policy == FilterPolicy.MERGE and filters: - filters = {**(self.filters or {}), **filters} - else: - filters = filters or self.filters + filters = apply_filter_policy(self.filter_policy, self.filters, filters) if top_k is None: top_k = self.top_k if scale_score is None: @@ -181,10 +178,7 @@ async def run_async( :raises ValueError: If the specified DocumentStore is not found or is not a InMemoryDocumentStore instance. """ - if self.filter_policy == FilterPolicy.MERGE and filters: - filters = {**(self.filters or {}), **filters} - else: - filters = filters or self.filters + filters = apply_filter_policy(self.filter_policy, self.filters, filters) if top_k is None: top_k = self.top_k if scale_score is None: diff --git a/haystack/components/retrievers/in_memory/embedding_retriever.py b/haystack/components/retrievers/in_memory/embedding_retriever.py index f185d33dc2f..5c2bf0dcf8f 100644 --- a/haystack/components/retrievers/in_memory/embedding_retriever.py +++ b/haystack/components/retrievers/in_memory/embedding_retriever.py @@ -6,7 +6,7 @@ from haystack import Document, component, default_from_dict, default_to_dict from haystack.document_stores.in_memory import InMemoryDocumentStore -from haystack.document_stores.types import FilterPolicy +from haystack.document_stores.types import FilterPolicy, apply_filter_policy @component @@ -163,10 +163,7 @@ def run( :raises ValueError: If the specified DocumentStore is not found or is not an InMemoryDocumentStore instance. """ - if self.filter_policy == FilterPolicy.MERGE and filters: - filters = {**(self.filters or {}), **filters} - else: - filters = filters or self.filters + filters = apply_filter_policy(self.filter_policy, self.filters, filters) if top_k is None: top_k = self.top_k if scale_score is None: @@ -214,10 +211,7 @@ async def run_async( :raises ValueError: If the specified DocumentStore is not found or is not an InMemoryDocumentStore instance. """ - if self.filter_policy == FilterPolicy.MERGE and filters: - filters = {**(self.filters or {}), **filters} - else: - filters = filters or self.filters + filters = apply_filter_policy(self.filter_policy, self.filters, filters) if top_k is None: top_k = self.top_k if scale_score is None: From 35d4f1a347df981fd00f15add3a5ec9843e347e1 Mon Sep 17 00:00:00 2001 From: Aarkin7 Date: Sun, 19 Jul 2026 20:02:48 +0530 Subject: [PATCH 2/4] test: add regression coverage for merged init and runtime filters --- .../test_in_memory_bm25_retriever.py | 45 ++++++++++++ .../test_in_memory_embedding_retriever.py | 71 +++++++++++++++++++ 2 files changed, 116 insertions(+) diff --git a/test/components/retrievers/test_in_memory_bm25_retriever.py b/test/components/retrievers/test_in_memory_bm25_retriever.py index 42cbcb56ce8..89bfa30df1c 100644 --- a/test/components/retrievers/test_in_memory_bm25_retriever.py +++ b/test/components/retrievers/test_in_memory_bm25_retriever.py @@ -139,6 +139,51 @@ def test_retriever_valid_run(self, in_memory_doc_store, mock_docs): assert len(result["documents"]) == 5 assert result["documents"][0].content == "PHP is a popular programming language" + def test_run_with_filter_policy_merge_combines_init_and_runtime_filters(self, in_memory_doc_store): + in_memory_doc_store.write_documents( + [ + Document(content="python article current", meta={"type": "article", "year": 2020}), + Document(content="python blog current", meta={"type": "blog", "year": 2021}), + Document(content="python article archived", meta={"type": "article", "year": 2019}), + ] + ) + + retriever = InMemoryBM25Retriever( + in_memory_doc_store, + filters={"field": "meta.type", "operator": "==", "value": "article"}, + filter_policy=FilterPolicy.MERGE, + ) + + result = retriever.run( + query="python", + filters={"field": "meta.year", "operator": ">=", "value": 2020}, + ) + + assert [doc.content for doc in result["documents"]] == ["python article current"] + + @pytest.mark.asyncio + async def test_run_async_with_filter_policy_merge_combines_init_and_runtime_filters(self, in_memory_doc_store): + in_memory_doc_store.write_documents( + [ + Document(content="python article current", meta={"type": "article", "year": 2020}), + Document(content="python blog current", meta={"type": "blog", "year": 2021}), + Document(content="python article archived", meta={"type": "article", "year": 2019}), + ] + ) + + retriever = InMemoryBM25Retriever( + in_memory_doc_store, + filters={"field": "meta.type", "operator": "==", "value": "article"}, + filter_policy=FilterPolicy.MERGE, + ) + + result = await retriever.run_async( + query="python", + filters={"field": "meta.year", "operator": ">=", "value": 2020}, + ) + + assert [doc.content for doc in result["documents"]] == ["python article current"] + def test_invalid_run_wrong_store_type(self): SomeOtherDocumentStore = document_store_class("SomeOtherDocumentStore") with pytest.raises(TypeError, match="document_store must be an instance of InMemoryDocumentStore"): diff --git a/test/components/retrievers/test_in_memory_embedding_retriever.py b/test/components/retrievers/test_in_memory_embedding_retriever.py index 024495c312b..3b6927b11a9 100644 --- a/test/components/retrievers/test_in_memory_embedding_retriever.py +++ b/test/components/retrievers/test_in_memory_embedding_retriever.py @@ -141,6 +141,77 @@ def test_valid_run(self): assert len(result["documents"]) == top_k assert result["documents"][0].embedding == [1.0, 1.0, 1.0, 1.0] + def test_run_with_filter_policy_merge_combines_init_and_runtime_filters(self): + ds = InMemoryDocumentStore(embedding_similarity_function="cosine") + ds.write_documents( + [ + Document( + content="python article current", + embedding=[1.0, 0.0, 0.0, 0.0], + meta={"type": "article", "year": 2020}, + ), + Document( + content="python blog current", + embedding=[1.0, 0.0, 0.0, 0.0], + meta={"type": "blog", "year": 2021}, + ), + Document( + content="python article archived", + embedding=[1.0, 0.0, 0.0, 0.0], + meta={"type": "article", "year": 2019}, + ), + ] + ) + + retriever = InMemoryEmbeddingRetriever( + ds, + filters={"field": "meta.type", "operator": "==", "value": "article"}, + filter_policy=FilterPolicy.MERGE, + ) + + result = retriever.run( + query_embedding=[1.0, 0.0, 0.0, 0.0], + filters={"field": "meta.year", "operator": ">=", "value": 2020}, + ) + + assert [doc.content for doc in result["documents"]] == ["python article current"] + + @pytest.mark.asyncio + async def test_run_async_with_filter_policy_merge_combines_init_and_runtime_filters(self): + ds = InMemoryDocumentStore(embedding_similarity_function="cosine") + ds.write_documents( + [ + Document( + content="python article current", + embedding=[1.0, 0.0, 0.0, 0.0], + meta={"type": "article", "year": 2020}, + ), + Document( + content="python blog current", + embedding=[1.0, 0.0, 0.0, 0.0], + meta={"type": "blog", "year": 2021}, + ), + Document( + content="python article archived", + embedding=[1.0, 0.0, 0.0, 0.0], + meta={"type": "article", "year": 2019}, + ), + ] + ) + + retriever = InMemoryEmbeddingRetriever( + ds, + filters={"field": "meta.type", "operator": "==", "value": "article"}, + filter_policy=FilterPolicy.MERGE, + ) + + result = await retriever.run_async( + query_embedding=[1.0, 0.0, 0.0, 0.0], + filters={"field": "meta.year", "operator": ">=", "value": 2020}, + ) + + assert [doc.content for doc in result["documents"]] == ["python article current"] + def test_invalid_run_wrong_store_type(self): SomeOtherDocumentStore = document_store_class("SomeOtherDocumentStore") with pytest.raises(TypeError, match="document_store must be an instance of InMemoryDocumentStore"): From 9cec235c78f9666315c8cf6ff38f1361d026d104 Mon Sep 17 00:00:00 2001 From: Aarkin7 Date: Sun, 19 Jul 2026 20:04:24 +0530 Subject: [PATCH 3/4] docs: add release note for the in-memory filter merge fix --- ...-policy-merge-in-memory-retrievers-f49dfcb2a1332e8b.yaml | 6 ++++++ 1 file changed, 6 insertions(+) create mode 100644 releasenotes/notes/filter-policy-merge-in-memory-retrievers-f49dfcb2a1332e8b.yaml diff --git a/releasenotes/notes/filter-policy-merge-in-memory-retrievers-f49dfcb2a1332e8b.yaml b/releasenotes/notes/filter-policy-merge-in-memory-retrievers-f49dfcb2a1332e8b.yaml new file mode 100644 index 00000000000..c4eb313961c --- /dev/null +++ b/releasenotes/notes/filter-policy-merge-in-memory-retrievers-f49dfcb2a1332e8b.yaml @@ -0,0 +1,6 @@ +--- +fixes: + - | + Fix ``FilterPolicy.MERGE`` in ``InMemoryBM25Retriever`` and + ``InMemoryEmbeddingRetriever`` so initialization filters are combined with + runtime comparison filters instead of being silently overwritten. \ No newline at end of file From 8dec27f7ec7ccdede63fec3c0447173d40282c0a Mon Sep 17 00:00:00 2001 From: Aarkin7 Date: Sun, 19 Jul 2026 20:15:35 +0530 Subject: [PATCH 4/4] style: format in-memory retriever regression tests for Ruff --- .../test_in_memory_bm25_retriever.py | 8 ++----- .../test_in_memory_embedding_retriever.py | 22 +++++-------------- 2 files changed, 8 insertions(+), 22 deletions(-) diff --git a/test/components/retrievers/test_in_memory_bm25_retriever.py b/test/components/retrievers/test_in_memory_bm25_retriever.py index 89bfa30df1c..3452667d95d 100644 --- a/test/components/retrievers/test_in_memory_bm25_retriever.py +++ b/test/components/retrievers/test_in_memory_bm25_retriever.py @@ -154,10 +154,7 @@ def test_run_with_filter_policy_merge_combines_init_and_runtime_filters(self, in filter_policy=FilterPolicy.MERGE, ) - result = retriever.run( - query="python", - filters={"field": "meta.year", "operator": ">=", "value": 2020}, - ) + result = retriever.run(query="python", filters={"field": "meta.year", "operator": ">=", "value": 2020}) assert [doc.content for doc in result["documents"]] == ["python article current"] @@ -178,8 +175,7 @@ async def test_run_async_with_filter_policy_merge_combines_init_and_runtime_filt ) result = await retriever.run_async( - query="python", - filters={"field": "meta.year", "operator": ">=", "value": 2020}, + query="python", filters={"field": "meta.year", "operator": ">=", "value": 2020} ) assert [doc.content for doc in result["documents"]] == ["python article current"] diff --git a/test/components/retrievers/test_in_memory_embedding_retriever.py b/test/components/retrievers/test_in_memory_embedding_retriever.py index 3b6927b11a9..4cf6d9dceef 100644 --- a/test/components/retrievers/test_in_memory_embedding_retriever.py +++ b/test/components/retrievers/test_in_memory_embedding_retriever.py @@ -151,9 +151,7 @@ def test_run_with_filter_policy_merge_combines_init_and_runtime_filters(self): meta={"type": "article", "year": 2020}, ), Document( - content="python blog current", - embedding=[1.0, 0.0, 0.0, 0.0], - meta={"type": "blog", "year": 2021}, + content="python blog current", embedding=[1.0, 0.0, 0.0, 0.0], meta={"type": "blog", "year": 2021} ), Document( content="python article archived", @@ -164,14 +162,11 @@ def test_run_with_filter_policy_merge_combines_init_and_runtime_filters(self): ) retriever = InMemoryEmbeddingRetriever( - ds, - filters={"field": "meta.type", "operator": "==", "value": "article"}, - filter_policy=FilterPolicy.MERGE, + ds, filters={"field": "meta.type", "operator": "==", "value": "article"}, filter_policy=FilterPolicy.MERGE ) result = retriever.run( - query_embedding=[1.0, 0.0, 0.0, 0.0], - filters={"field": "meta.year", "operator": ">=", "value": 2020}, + query_embedding=[1.0, 0.0, 0.0, 0.0], filters={"field": "meta.year", "operator": ">=", "value": 2020} ) assert [doc.content for doc in result["documents"]] == ["python article current"] @@ -187,9 +182,7 @@ async def test_run_async_with_filter_policy_merge_combines_init_and_runtime_filt meta={"type": "article", "year": 2020}, ), Document( - content="python blog current", - embedding=[1.0, 0.0, 0.0, 0.0], - meta={"type": "blog", "year": 2021}, + content="python blog current", embedding=[1.0, 0.0, 0.0, 0.0], meta={"type": "blog", "year": 2021} ), Document( content="python article archived", @@ -200,14 +193,11 @@ async def test_run_async_with_filter_policy_merge_combines_init_and_runtime_filt ) retriever = InMemoryEmbeddingRetriever( - ds, - filters={"field": "meta.type", "operator": "==", "value": "article"}, - filter_policy=FilterPolicy.MERGE, + ds, filters={"field": "meta.type", "operator": "==", "value": "article"}, filter_policy=FilterPolicy.MERGE ) result = await retriever.run_async( - query_embedding=[1.0, 0.0, 0.0, 0.0], - filters={"field": "meta.year", "operator": ">=", "value": 2020}, + query_embedding=[1.0, 0.0, 0.0, 0.0], filters={"field": "meta.year", "operator": ">=", "value": 2020} ) assert [doc.content for doc in result["documents"]] == ["python article current"]