From 2cbb05d6d773215db35e124f9a32d9afcc923733 Mon Sep 17 00:00:00 2001 From: "David S. Batista" Date: Mon, 27 Jul 2026 17:08:01 +0200 Subject: [PATCH 1/4] adding offset pagination to get_metadata_field_unique_values + tests --- .../in_memory/document_store.py | 40 +++++++++++-------- test/document_stores/test_in_memory.py | 24 +++++++++++ 2 files changed, 48 insertions(+), 16 deletions(-) diff --git a/haystack/document_stores/in_memory/document_store.py b/haystack/document_stores/in_memory/document_store.py index 48e7275d322..b73b9222ff8 100644 --- a/haystack/document_stores/in_memory/document_store.py +++ b/haystack/document_stores/in_memory/document_store.py @@ -653,23 +653,27 @@ def get_metadata_field_min_max(self, metadata_field: str) -> dict[str, Any]: return {"min": None, "max": None} def get_metadata_field_unique_values( - self, metadata_field: str, search_term: str | None = None + self, metadata_field: str, search_term: str | None = None, from_: int = 0, size: int = 10 ) -> tuple[list[str], int]: """ - Returns unique values for a metadata field, optionally filtered by a search term in content. + Returns unique values for a metadata field, optionally filtered by a search term, with pagination. :param metadata_field: The metadata field name. Can include or omit the "meta." prefix. - :param search_term: If set, only documents whose content contains this term (case-insensitive) - are considered. - :returns: A tuple of (list of unique values, total count of unique values). + :param search_term: Optional search term to filter values, matched as a case-insensitive substring + against the metadata field's value. + :param from_: The offset to start returning values from (for pagination). + :param size: The maximum number of unique values to return. + :returns: A tuple of (paginated list of unique values, total count of unique values). """ key = metadata_field.removeprefix("meta.") if metadata_field.startswith("meta.") else metadata_field + values = {str(doc.meta[key]) for doc in self.storage.values() if key in doc.meta and doc.meta[key] is not None} + if search_term: - docs = [doc for doc in self.storage.values() if doc.content and search_term.lower() in doc.content.lower()] - else: - docs = list(self.storage.values()) - values = sorted({str(doc.meta[key]) for doc in docs if key in doc.meta and doc.meta[key] is not None}, key=str) - return values, len(values) + search_term_lower = search_term.lower() + values = {value for value in values if search_term_lower in value.lower()} + + sorted_values = sorted(values) + return sorted_values[from_ : from_ + size], len(sorted_values) def bm25_retrieval( self, query: str, filters: dict[str, Any] | None = None, top_k: int = 10, scale_score: bool = False @@ -963,19 +967,23 @@ async def get_metadata_field_min_max_async(self, metadata_field: str) -> dict[st ) async def get_metadata_field_unique_values_async( - self, metadata_field: str, search_term: str | None = None + self, metadata_field: str, search_term: str | None = None, from_: int = 0, size: int = 10 ) -> tuple[list[str], int]: """ - Returns unique values for a metadata field, optionally filtered by a search term in content. + Returns unique values for a metadata field, optionally filtered by a search term, with pagination. :param metadata_field: The metadata field name. Can include or omit the "meta." prefix. - :param search_term: If set, only documents whose content contains this term (case-insensitive) - are considered. - :returns: A tuple of (list of unique values, total count of unique values). + :param search_term: Optional search term to filter values, matched as a case-insensitive substring + against the metadata field's value. + :param from_: The offset to start returning values from (for pagination). + :param size: The maximum number of unique values to return. + :returns: A tuple of (paginated list of unique values, total count of unique values). """ return await asyncio.get_running_loop().run_in_executor( self.executor, - lambda: self.get_metadata_field_unique_values(metadata_field=metadata_field, search_term=search_term), + lambda: self.get_metadata_field_unique_values( + metadata_field=metadata_field, search_term=search_term, from_=from_, size=size + ), ) async def delete_all_documents_async(self) -> None: diff --git a/test/document_stores/test_in_memory.py b/test/document_stores/test_in_memory.py index b5219178916..0452cff7db1 100644 --- a/test/document_stores/test_in_memory.py +++ b/test/document_stores/test_in_memory.py @@ -343,6 +343,30 @@ def test_bm25_retrieval_default_filter(self, document_store: InMemoryDocumentSto results = document_store.bm25_retrieval(query="doesn't matter, top_k is 10", top_k=10) assert len(results) == 0 + def test_get_metadata_field_unique_values_pagination_and_search_term( + self, document_store: InMemoryDocumentStore + ) -> None: + docs = [Document(content="Doc 1", meta={"category": f"category_{i}"}) for i in range(5)] + # this must NOT be picked up, since search_term matches the metadata value, not content + docs.append(Document(content="mentions category_0 in its content", meta={"category": "other"})) + document_store.write_documents(docs) + + # pagination: total count reflects all unique values, but the returned page is limited to `size` + page, total = document_store.get_metadata_field_unique_values("category", from_=0, size=2) + assert total == 6 + assert page == sorted({f"category_{i}" for i in range(5)} | {"other"})[:2] + + next_page, total = document_store.get_metadata_field_unique_values("category", from_=2, size=2) + assert total == 6 + assert next_page == sorted({f"category_{i}" for i in range(5)} | {"other"})[2:4] + + # search_term matches against the metadata value itself, not document content + filtered, filtered_total = document_store.get_metadata_field_unique_values( + "category", search_term="category_", from_=0, size=10 + ) + assert filtered_total == 5 + assert set(filtered) == {f"category_{i}" for i in range(5)} + def test_embedding_retrieval_return_embedding_false_on_store(self): # Initialize InMemoryDocumentStore with return_embedding=False docstore = InMemoryDocumentStore(embedding_similarity_function="cosine", return_embedding=False) From 170bffa6068525f4f4e4ca785d4b82fdb48027b1 Mon Sep 17 00:00:00 2001 From: "David S. Batista" Date: Mon, 27 Jul 2026 17:26:16 +0200 Subject: [PATCH 2/4] updating Mixin document store tests --- haystack/testing/document_store.py | 100 ++++++++++++++++---- haystack/testing/document_store_async.py | 114 +++++++++++++++++++---- 2 files changed, 174 insertions(+), 40 deletions(-) diff --git a/haystack/testing/document_store.py b/haystack/testing/document_store.py index 0139ca0f480..7e1c7aa7c82 100644 --- a/haystack/testing/document_store.py +++ b/haystack/testing/document_store.py @@ -1204,8 +1204,8 @@ class GetMetadataFieldUniqueValuesTest: """ Tests for Document Store get_metadata_field_unique_values(). - Only mix in for stores that implement get_metadata_field_unique_values. - Expects the method to return (values_list, total_count) or (values_list, pagination_key). + Only mix in for stores that implement get_metadata_field_unique_values() with the standardized + signature: get_metadata_field_unique_values(field, search_term=None, from_=0, size=10) -> (values, total_count). """ @staticmethod @@ -1221,27 +1221,87 @@ def test_get_metadata_field_unique_values_basic(document_store: DocumentStore): document_store.write_documents(docs) assert document_store.count_documents() == 5 - sig = inspect.signature(document_store.get_metadata_field_unique_values) # type:ignore[attr-defined] - params: dict = {} - if "search_term" in sig.parameters: - params["search_term"] = None - if "from_" in sig.parameters: - params["from_"] = 0 - elif "offset" in sig.parameters: - params["offset"] = 0 - if "size" in sig.parameters: - params["size"] = 10 - elif "limit" in sig.parameters: - params["limit"] = 10 - - result = document_store.get_metadata_field_unique_values("category", **params) # type:ignore[attr-defined] - - values = result[0] if isinstance(result, tuple) else result + values, total_count = document_store.get_metadata_field_unique_values("category") # type:ignore[attr-defined] + assert isinstance(values, list) assert len(values) == 3 # the returned values must not contain duplicates assert set(values) == {"A", "B", "C"} - if isinstance(result, tuple) and len(result) >= 2 and isinstance(result[1], int): - assert result[1] == 3 + assert total_count == 3 + + @staticmethod + def test_get_metadata_field_unique_values_meta_prefix(document_store: DocumentStore): + """Test get_metadata_field_unique_values() with a field name that includes the 'meta.' prefix.""" + docs = [ + Document(content="Doc 1", meta={"category": "A"}), + Document(content="Doc 2", meta={"category": "B"}), + Document(content="Doc 3", meta={"category": "A"}), + ] + document_store.write_documents(docs) + + values, total_count = document_store.get_metadata_field_unique_values("category") # type:ignore[attr-defined] + prefixed_values, prefixed_total_count = document_store.get_metadata_field_unique_values( # type:ignore[attr-defined] + "meta.category" + ) + + assert set(prefixed_values) == set(values) + assert prefixed_total_count == total_count + + @staticmethod + def test_get_metadata_field_unique_values_search_term(document_store: DocumentStore): + """Test get_metadata_field_unique_values() filters by a case-insensitive substring of the value.""" + docs = [ + Document(content="Doc 1", meta={"category": "All_Beauty"}), + Document(content="Doc 2", meta={"category": "Electronics"}), + # the content mentions "beaut" but the value doesn't - must not match search_term="beaut" + Document(content="something about beauty products", meta={"category": "Home"}), + ] + document_store.write_documents(docs) + + values, total_count = document_store.get_metadata_field_unique_values( # type:ignore[attr-defined] + "category", search_term="beaut" + ) + + assert values == ["All_Beauty"] + assert total_count == 1 + + @staticmethod + def test_get_metadata_field_unique_values_pagination(document_store: DocumentStore): + """Test get_metadata_field_unique_values() paginates a stable ordering via from_/size.""" + docs = [Document(content=f"Doc {i}", meta={"category": f"category_{i}"}) for i in range(5)] + document_store.write_documents(docs) + + all_values = sorted(f"category_{i}" for i in range(5)) + + first_page, total_count = document_store.get_metadata_field_unique_values( # type:ignore[attr-defined] + "category", from_=0, size=2 + ) + assert first_page == all_values[:2] + assert total_count == 5 + + second_page, total_count = document_store.get_metadata_field_unique_values( # type:ignore[attr-defined] + "category", from_=2, size=2 + ) + assert second_page == all_values[2:4] + assert total_count == 5 + + @staticmethod + def test_get_metadata_field_unique_values_empty_store(document_store: DocumentStore): + """Test get_metadata_field_unique_values() on an empty store.""" + assert document_store.count_documents() == 0 + + values, total_count = document_store.get_metadata_field_unique_values("category") # type:ignore[attr-defined] + assert values == [] + assert total_count == 0 + + @staticmethod + def test_get_metadata_field_unique_values_missing_field(document_store: DocumentStore): + """Test get_metadata_field_unique_values() for a field that no document has.""" + docs = [Document(content="Doc 1", meta={"category": "A"})] + document_store.write_documents(docs) + + values, total_count = document_store.get_metadata_field_unique_values("missing_field") # type:ignore[attr-defined] + assert values == [] + assert total_count == 0 class DocumentStoreBaseTests(CountDocumentsTest, DeleteDocumentsTest, FilterDocumentsTest, WriteDocumentsTest): diff --git a/haystack/testing/document_store_async.py b/haystack/testing/document_store_async.py index ebe6f22eaf4..f977b3adcdb 100644 --- a/haystack/testing/document_store_async.py +++ b/haystack/testing/document_store_async.py @@ -709,8 +709,9 @@ class GetMetadataFieldUniqueValuesAsyncTest: """ Tests for Document Store get_metadata_field_unique_values_async(). - Only mix in for stores that implement get_metadata_field_unique_values_async. - Expects the method to return (values_list, total_count) or (values_list, pagination_key). + Only mix in for stores that implement get_metadata_field_unique_values_async() with the standardized + signature: get_metadata_field_unique_values_async(field, search_term=None, from_=0, size=10) + -> (values, total_count). """ @staticmethod @@ -727,27 +728,100 @@ async def test_get_metadata_field_unique_values_basic_async(document_store: Asyn await document_store.write_documents_async(docs) assert await document_store.count_documents_async() == 5 - sig = inspect.signature(document_store.get_metadata_field_unique_values_async) # type:ignore[attr-defined] - params: dict = {} - if "search_term" in sig.parameters: - params["search_term"] = None - if "from_" in sig.parameters: - params["from_"] = 0 - elif "offset" in sig.parameters: - params["offset"] = 0 - if "size" in sig.parameters: - params["size"] = 10 - elif "limit" in sig.parameters: - params["limit"] = 10 - - result = await document_store.get_metadata_field_unique_values_async("category", **params) # type:ignore[attr-defined] - - values = result[0] if isinstance(result, tuple) else result + values, total_count = await document_store.get_metadata_field_unique_values_async( # type:ignore[attr-defined] + "category" + ) + assert isinstance(values, list) assert len(values) == 3 # the returned values must not contain duplicates assert set(values) == {"A", "B", "C"} - if isinstance(result, tuple) and len(result) >= 2 and isinstance(result[1], int): - assert result[1] == 3 + assert total_count == 3 + + @staticmethod + @pytest.mark.asyncio + async def test_get_metadata_field_unique_values_meta_prefix_async(document_store: AsyncDocumentStore): + """Test get_metadata_field_unique_values_async() with a field name that includes the 'meta.' prefix.""" + docs = [ + Document(content="Doc 1", meta={"category": "A"}), + Document(content="Doc 2", meta={"category": "B"}), + Document(content="Doc 3", meta={"category": "A"}), + ] + await document_store.write_documents_async(docs) + + values, total_count = await document_store.get_metadata_field_unique_values_async( # type:ignore[attr-defined] + "category" + ) + prefixed_values, prefixed_total_count = await document_store.get_metadata_field_unique_values_async( # type:ignore[attr-defined] + "meta.category" + ) + + assert set(prefixed_values) == set(values) + assert prefixed_total_count == total_count + + @staticmethod + @pytest.mark.asyncio + async def test_get_metadata_field_unique_values_search_term_async(document_store: AsyncDocumentStore): + """Test get_metadata_field_unique_values_async() filters by a case-insensitive substring of the value.""" + docs = [ + Document(content="Doc 1", meta={"category": "All_Beauty"}), + Document(content="Doc 2", meta={"category": "Electronics"}), + # the content mentions "beaut" but the value doesn't - must not match search_term="beaut" + Document(content="something about beauty products", meta={"category": "Home"}), + ] + await document_store.write_documents_async(docs) + + values, total_count = await document_store.get_metadata_field_unique_values_async( # type:ignore[attr-defined] + "category", search_term="beaut" + ) + + assert values == ["All_Beauty"] + assert total_count == 1 + + @staticmethod + @pytest.mark.asyncio + async def test_get_metadata_field_unique_values_pagination_async(document_store: AsyncDocumentStore): + """Test get_metadata_field_unique_values_async() paginates a stable ordering via from_/size.""" + docs = [Document(content=f"Doc {i}", meta={"category": f"category_{i}"}) for i in range(5)] + await document_store.write_documents_async(docs) + + all_values = sorted(f"category_{i}" for i in range(5)) + + first_page, total_count = await document_store.get_metadata_field_unique_values_async( # type:ignore[attr-defined] + "category", from_=0, size=2 + ) + assert first_page == all_values[:2] + assert total_count == 5 + + second_page, total_count = await document_store.get_metadata_field_unique_values_async( # type:ignore[attr-defined] + "category", from_=2, size=2 + ) + assert second_page == all_values[2:4] + assert total_count == 5 + + @staticmethod + @pytest.mark.asyncio + async def test_get_metadata_field_unique_values_empty_store_async(document_store: AsyncDocumentStore): + """Test get_metadata_field_unique_values_async() on an empty store.""" + assert await document_store.count_documents_async() == 0 + + values, total_count = await document_store.get_metadata_field_unique_values_async( # type:ignore[attr-defined] + "category" + ) + assert values == [] + assert total_count == 0 + + @staticmethod + @pytest.mark.asyncio + async def test_get_metadata_field_unique_values_missing_field_async(document_store: AsyncDocumentStore): + """Test get_metadata_field_unique_values_async() for a field that no document has.""" + docs = [Document(content="Doc 1", meta={"category": "A"})] + await document_store.write_documents_async(docs) + + values, total_count = await document_store.get_metadata_field_unique_values_async( # type:ignore[attr-defined] + "missing_field" + ) + assert values == [] + assert total_count == 0 class FilterDocumentsAsyncTest(AssertDocumentsEqualMixin, FilterableDocsFixtureMixin): From 00be6e6f3aa60f9e5162160e02e7edab51805844 Mon Sep 17 00:00:00 2001 From: "David S. Batista" Date: Mon, 27 Jul 2026 17:26:52 +0200 Subject: [PATCH 3/4] updating tests --- test/document_stores/test_in_memory.py | 24 ------------------------ 1 file changed, 24 deletions(-) diff --git a/test/document_stores/test_in_memory.py b/test/document_stores/test_in_memory.py index 0452cff7db1..b5219178916 100644 --- a/test/document_stores/test_in_memory.py +++ b/test/document_stores/test_in_memory.py @@ -343,30 +343,6 @@ def test_bm25_retrieval_default_filter(self, document_store: InMemoryDocumentSto results = document_store.bm25_retrieval(query="doesn't matter, top_k is 10", top_k=10) assert len(results) == 0 - def test_get_metadata_field_unique_values_pagination_and_search_term( - self, document_store: InMemoryDocumentStore - ) -> None: - docs = [Document(content="Doc 1", meta={"category": f"category_{i}"}) for i in range(5)] - # this must NOT be picked up, since search_term matches the metadata value, not content - docs.append(Document(content="mentions category_0 in its content", meta={"category": "other"})) - document_store.write_documents(docs) - - # pagination: total count reflects all unique values, but the returned page is limited to `size` - page, total = document_store.get_metadata_field_unique_values("category", from_=0, size=2) - assert total == 6 - assert page == sorted({f"category_{i}" for i in range(5)} | {"other"})[:2] - - next_page, total = document_store.get_metadata_field_unique_values("category", from_=2, size=2) - assert total == 6 - assert next_page == sorted({f"category_{i}" for i in range(5)} | {"other"})[2:4] - - # search_term matches against the metadata value itself, not document content - filtered, filtered_total = document_store.get_metadata_field_unique_values( - "category", search_term="category_", from_=0, size=10 - ) - assert filtered_total == 5 - assert set(filtered) == {f"category_{i}" for i in range(5)} - def test_embedding_retrieval_return_embedding_false_on_store(self): # Initialize InMemoryDocumentStore with return_embedding=False docstore = InMemoryDocumentStore(embedding_similarity_function="cosine", return_embedding=False) From 958422c97df3d894efd67c4598c1fc25e5242d4e Mon Sep 17 00:00:00 2001 From: "David S. Batista" Date: Mon, 27 Jul 2026 17:42:32 +0200 Subject: [PATCH 4/4] adding release notes --- ...unique-metadata-values-pagination-a3f6c9d21b7e4081.yaml | 7 +++++++ 1 file changed, 7 insertions(+) create mode 100644 releasenotes/notes/inmemory-unique-metadata-values-pagination-a3f6c9d21b7e4081.yaml diff --git a/releasenotes/notes/inmemory-unique-metadata-values-pagination-a3f6c9d21b7e4081.yaml b/releasenotes/notes/inmemory-unique-metadata-values-pagination-a3f6c9d21b7e4081.yaml new file mode 100644 index 00000000000..eb2c4eef1e4 --- /dev/null +++ b/releasenotes/notes/inmemory-unique-metadata-values-pagination-a3f6c9d21b7e4081.yaml @@ -0,0 +1,7 @@ +--- +enhancements: + - | + ``InMemoryDocumentStore.get_metadata_field_unique_values`` and its async counterpart now support + pagination via ``from_`` and ``size`` parameters, matching the behavior of other Document Stores + (e.g. Chroma). ``search_term`` now matches against the metadata field's own value (case-insensitive + substring) instead of the document's content.