fix: in memory doc store get_unique_metadata_values() pagination and case-insensitive + Mixin tests updated

haystack/document_stores/in_memory/document_store.py

@@ -653,23 +653,27 @@ def get_metadata_field_min_max(self, metadata_field: str) -> dict[str, Any]:
             return {"min": None, "max": None}
 
     def get_metadata_field_unique_values(
-        self, metadata_field: str, search_term: str | None = None
+        self, metadata_field: str, search_term: str | None = None, from_: int = 0, size: int = 10
     ) -> tuple[list[str], int]:
         """
-        Returns unique values for a metadata field, optionally filtered by a search term in content.
+        Returns unique values for a metadata field, optionally filtered by a search term, with pagination.
 
         :param metadata_field: The metadata field name. Can include or omit the "meta." prefix.
-        :param search_term: If set, only documents whose content contains this term (case-insensitive)
-            are considered.
-        :returns: A tuple of (list of unique values, total count of unique values).
+        :param search_term: Optional search term to filter values, matched as a case-insensitive substring
+            against the metadata field's value.
+        :param from_: The offset to start returning values from (for pagination).
+        :param size: The maximum number of unique values to return.
+        :returns: A tuple of (paginated list of unique values, total count of unique values).
         """
         key = metadata_field.removeprefix("meta.") if metadata_field.startswith("meta.") else metadata_field
+        values = {str(doc.meta[key]) for doc in self.storage.values() if key in doc.meta and doc.meta[key] is not None}
+
         if search_term:
-            docs = [doc for doc in self.storage.values() if doc.content and search_term.lower() in doc.content.lower()]
-        else:
-            docs = list(self.storage.values())
-        values = sorted({str(doc.meta[key]) for doc in docs if key in doc.meta and doc.meta[key] is not None}, key=str)
-        return values, len(values)
+            search_term_lower = search_term.lower()
+            values = {value for value in values if search_term_lower in value.lower()}
+
+        sorted_values = sorted(values)
+        return sorted_values[from_ : from_ + size], len(sorted_values)
 
     def bm25_retrieval(
         self, query: str, filters: dict[str, Any] | None = None, top_k: int = 10, scale_score: bool = False
@@ -963,19 +967,23 @@ async def get_metadata_field_min_max_async(self, metadata_field: str) -> dict[st
         )
 
     async def get_metadata_field_unique_values_async(
-        self, metadata_field: str, search_term: str | None = None
+        self, metadata_field: str, search_term: str | None = None, from_: int = 0, size: int = 10
     ) -> tuple[list[str], int]:
         """
-        Returns unique values for a metadata field, optionally filtered by a search term in content.
+        Returns unique values for a metadata field, optionally filtered by a search term, with pagination.
 
         :param metadata_field: The metadata field name. Can include or omit the "meta." prefix.
-        :param search_term: If set, only documents whose content contains this term (case-insensitive)
-            are considered.
-        :returns: A tuple of (list of unique values, total count of unique values).
+        :param search_term: Optional search term to filter values, matched as a case-insensitive substring
+            against the metadata field's value.
+        :param from_: The offset to start returning values from (for pagination).
+        :param size: The maximum number of unique values to return.
+        :returns: A tuple of (paginated list of unique values, total count of unique values).
         """
         return await asyncio.get_running_loop().run_in_executor(
             self.executor,
-            lambda: self.get_metadata_field_unique_values(metadata_field=metadata_field, search_term=search_term),
+            lambda: self.get_metadata_field_unique_values(
+                metadata_field=metadata_field, search_term=search_term, from_=from_, size=size
+            ),
         )
 
     async def delete_all_documents_async(self) -> None:

haystack/testing/document_store.py

@@ -1204,8 +1204,8 @@ class GetMetadataFieldUniqueValuesTest:
     """
     Tests for Document Store get_metadata_field_unique_values().
 
-    Only mix in for stores that implement get_metadata_field_unique_values.
-    Expects the method to return (values_list, total_count) or (values_list, pagination_key).
+    Only mix in for stores that implement get_metadata_field_unique_values() with the standardized
+    signature: get_metadata_field_unique_values(field, search_term=None, from_=0, size=10) -> (values, total_count).
     """
 
     @staticmethod
@@ -1221,27 +1221,87 @@ def test_get_metadata_field_unique_values_basic(document_store: DocumentStore):
         document_store.write_documents(docs)
         assert document_store.count_documents() == 5
 
-        sig = inspect.signature(document_store.get_metadata_field_unique_values)  # type:ignore[attr-defined]
-        params: dict = {}
-        if "search_term" in sig.parameters:
-            params["search_term"] = None
-        if "from_" in sig.parameters:
-            params["from_"] = 0
-        elif "offset" in sig.parameters:
-            params["offset"] = 0
-        if "size" in sig.parameters:
-            params["size"] = 10
-        elif "limit" in sig.parameters:
-            params["limit"] = 10
-
-        result = document_store.get_metadata_field_unique_values("category", **params)  # type:ignore[attr-defined]
-
-        values = result[0] if isinstance(result, tuple) else result
+        values, total_count = document_store.get_metadata_field_unique_values("category")  # type:ignore[attr-defined]
+
         assert isinstance(values, list)
         assert len(values) == 3  # the returned values must not contain duplicates
         assert set(values) == {"A", "B", "C"}
-        if isinstance(result, tuple) and len(result) >= 2 and isinstance(result[1], int):
-            assert result[1] == 3
+        assert total_count == 3
+
+    @staticmethod
+    def test_get_metadata_field_unique_values_meta_prefix(document_store: DocumentStore):
+        """Test get_metadata_field_unique_values() with a field name that includes the 'meta.' prefix."""
+        docs = [
+            Document(content="Doc 1", meta={"category": "A"}),
+            Document(content="Doc 2", meta={"category": "B"}),
+            Document(content="Doc 3", meta={"category": "A"}),
+        ]
+        document_store.write_documents(docs)
+
+        values, total_count = document_store.get_metadata_field_unique_values("category")  # type:ignore[attr-defined]
+        prefixed_values, prefixed_total_count = document_store.get_metadata_field_unique_values(  # type:ignore[attr-defined]
+            "meta.category"
+        )
+
+        assert set(prefixed_values) == set(values)
+        assert prefixed_total_count == total_count
+
+    @staticmethod
+    def test_get_metadata_field_unique_values_search_term(document_store: DocumentStore):
+        """Test get_metadata_field_unique_values() filters by a case-insensitive substring of the value."""
+        docs = [
+            Document(content="Doc 1", meta={"category": "All_Beauty"}),
+            Document(content="Doc 2", meta={"category": "Electronics"}),
+            # the content mentions "beaut" but the value doesn't - must not match search_term="beaut"
+            Document(content="something about beauty products", meta={"category": "Home"}),
+        ]
+        document_store.write_documents(docs)
+
+        values, total_count = document_store.get_metadata_field_unique_values(  # type:ignore[attr-defined]
+            "category", search_term="beaut"
+        )
+
+        assert values == ["All_Beauty"]
+        assert total_count == 1
+
+    @staticmethod
+    def test_get_metadata_field_unique_values_pagination(document_store: DocumentStore):
+        """Test get_metadata_field_unique_values() paginates a stable ordering via from_/size."""
+        docs = [Document(content=f"Doc {i}", meta={"category": f"category_{i}"}) for i in range(5)]
+        document_store.write_documents(docs)
+
+        all_values = sorted(f"category_{i}" for i in range(5))
+
+        first_page, total_count = document_store.get_metadata_field_unique_values(  # type:ignore[attr-defined]
+            "category", from_=0, size=2
+        )
+        assert first_page == all_values[:2]
+        assert total_count == 5
+
+        second_page, total_count = document_store.get_metadata_field_unique_values(  # type:ignore[attr-defined]
+            "category", from_=2, size=2
+        )
+        assert second_page == all_values[2:4]
+        assert total_count == 5
+
+    @staticmethod
+    def test_get_metadata_field_unique_values_empty_store(document_store: DocumentStore):
+        """Test get_metadata_field_unique_values() on an empty store."""
+        assert document_store.count_documents() == 0
+
+        values, total_count = document_store.get_metadata_field_unique_values("category")  # type:ignore[attr-defined]
+        assert values == []
+        assert total_count == 0
+
+    @staticmethod
+    def test_get_metadata_field_unique_values_missing_field(document_store: DocumentStore):
+        """Test get_metadata_field_unique_values() for a field that no document has."""
+        docs = [Document(content="Doc 1", meta={"category": "A"})]
+        document_store.write_documents(docs)
+
+        values, total_count = document_store.get_metadata_field_unique_values("missing_field")  # type:ignore[attr-defined]
+        assert values == []
+        assert total_count == 0
 
 
 class DocumentStoreBaseTests(CountDocumentsTest, DeleteDocumentsTest, FilterDocumentsTest, WriteDocumentsTest):

haystack/testing/document_store_async.py

@@ -709,8 +709,9 @@ class GetMetadataFieldUniqueValuesAsyncTest:
     """
     Tests for Document Store get_metadata_field_unique_values_async().
 
-    Only mix in for stores that implement get_metadata_field_unique_values_async.
-    Expects the method to return (values_list, total_count) or (values_list, pagination_key).
+    Only mix in for stores that implement get_metadata_field_unique_values_async() with the standardized
+    signature: get_metadata_field_unique_values_async(field, search_term=None, from_=0, size=10)
+    -> (values, total_count).
     """
 
     @staticmethod
@@ -727,27 +728,100 @@ async def test_get_metadata_field_unique_values_basic_async(document_store: Asyn
         await document_store.write_documents_async(docs)
         assert await document_store.count_documents_async() == 5
 
-        sig = inspect.signature(document_store.get_metadata_field_unique_values_async)  # type:ignore[attr-defined]
-        params: dict = {}
-        if "search_term" in sig.parameters:
-            params["search_term"] = None
-        if "from_" in sig.parameters:
-            params["from_"] = 0
-        elif "offset" in sig.parameters:
-            params["offset"] = 0
-        if "size" in sig.parameters:
-            params["size"] = 10
-        elif "limit" in sig.parameters:
-            params["limit"] = 10
-
-        result = await document_store.get_metadata_field_unique_values_async("category", **params)  # type:ignore[attr-defined]
-
-        values = result[0] if isinstance(result, tuple) else result
+        values, total_count = await document_store.get_metadata_field_unique_values_async(  # type:ignore[attr-defined]
+            "category"
+        )
+
         assert isinstance(values, list)
         assert len(values) == 3  # the returned values must not contain duplicates
         assert set(values) == {"A", "B", "C"}
-        if isinstance(result, tuple) and len(result) >= 2 and isinstance(result[1], int):
-            assert result[1] == 3
+        assert total_count == 3
+
+    @staticmethod
+    @pytest.mark.asyncio
+    async def test_get_metadata_field_unique_values_meta_prefix_async(document_store: AsyncDocumentStore):
+        """Test get_metadata_field_unique_values_async() with a field name that includes the 'meta.' prefix."""
+        docs = [
+            Document(content="Doc 1", meta={"category": "A"}),
+            Document(content="Doc 2", meta={"category": "B"}),
+            Document(content="Doc 3", meta={"category": "A"}),
+        ]
+        await document_store.write_documents_async(docs)
+
+        values, total_count = await document_store.get_metadata_field_unique_values_async(  # type:ignore[attr-defined]
+            "category"
+        )
+        prefixed_values, prefixed_total_count = await document_store.get_metadata_field_unique_values_async(  # type:ignore[attr-defined]
+            "meta.category"
+        )
+
+        assert set(prefixed_values) == set(values)
+        assert prefixed_total_count == total_count
+
+    @staticmethod
+    @pytest.mark.asyncio
+    async def test_get_metadata_field_unique_values_search_term_async(document_store: AsyncDocumentStore):
+        """Test get_metadata_field_unique_values_async() filters by a case-insensitive substring of the value."""
+        docs = [
+            Document(content="Doc 1", meta={"category": "All_Beauty"}),
+            Document(content="Doc 2", meta={"category": "Electronics"}),
+            # the content mentions "beaut" but the value doesn't - must not match search_term="beaut"
+            Document(content="something about beauty products", meta={"category": "Home"}),
+        ]
+        await document_store.write_documents_async(docs)
+
+        values, total_count = await document_store.get_metadata_field_unique_values_async(  # type:ignore[attr-defined]
+            "category", search_term="beaut"
+        )
+
+        assert values == ["All_Beauty"]
+        assert total_count == 1
+
+    @staticmethod
+    @pytest.mark.asyncio
+    async def test_get_metadata_field_unique_values_pagination_async(document_store: AsyncDocumentStore):
+        """Test get_metadata_field_unique_values_async() paginates a stable ordering via from_/size."""
+        docs = [Document(content=f"Doc {i}", meta={"category": f"category_{i}"}) for i in range(5)]
+        await document_store.write_documents_async(docs)
+
+        all_values = sorted(f"category_{i}" for i in range(5))
+
+        first_page, total_count = await document_store.get_metadata_field_unique_values_async(  # type:ignore[attr-defined]
+            "category", from_=0, size=2
+        )
+        assert first_page == all_values[:2]
+        assert total_count == 5
+
+        second_page, total_count = await document_store.get_metadata_field_unique_values_async(  # type:ignore[attr-defined]
+            "category", from_=2, size=2
+        )
+        assert second_page == all_values[2:4]
+        assert total_count == 5
+
+    @staticmethod
+    @pytest.mark.asyncio
+    async def test_get_metadata_field_unique_values_empty_store_async(document_store: AsyncDocumentStore):
+        """Test get_metadata_field_unique_values_async() on an empty store."""
+        assert await document_store.count_documents_async() == 0
+
+        values, total_count = await document_store.get_metadata_field_unique_values_async(  # type:ignore[attr-defined]
+            "category"
+        )
+        assert values == []
+        assert total_count == 0
+
+    @staticmethod
+    @pytest.mark.asyncio
+    async def test_get_metadata_field_unique_values_missing_field_async(document_store: AsyncDocumentStore):
+        """Test get_metadata_field_unique_values_async() for a field that no document has."""
+        docs = [Document(content="Doc 1", meta={"category": "A"})]
+        await document_store.write_documents_async(docs)
+
+        values, total_count = await document_store.get_metadata_field_unique_values_async(  # type:ignore[attr-defined]
+            "missing_field"
+        )
+        assert values == []
+        assert total_count == 0
 
 
 class FilterDocumentsAsyncTest(AssertDocumentsEqualMixin, FilterableDocsFixtureMixin):

releasenotes/notes/inmemory-unique-metadata-values-pagination-a3f6c9d21b7e4081.yaml

@@ -0,0 +1,7 @@
+---
+enhancements:
+  - |
+    ``InMemoryDocumentStore.get_metadata_field_unique_values`` and its async counterpart now support
+    pagination via ``from_`` and ``size`` parameters, matching the behavior of other Document Stores
+    (e.g. Chroma). ``search_term`` now matches against the metadata field's own value (case-insensitive
+    substring) instead of the document's content.