diff --git a/haystack/document_stores/in_memory/document_store.py b/haystack/document_stores/in_memory/document_store.py index 48e7275d322..b73b9222ff8 100644 --- a/haystack/document_stores/in_memory/document_store.py +++ b/haystack/document_stores/in_memory/document_store.py @@ -653,23 +653,27 @@ def get_metadata_field_min_max(self, metadata_field: str) -> dict[str, Any]: return {"min": None, "max": None} def get_metadata_field_unique_values( - self, metadata_field: str, search_term: str | None = None + self, metadata_field: str, search_term: str | None = None, from_: int = 0, size: int = 10 ) -> tuple[list[str], int]: """ - Returns unique values for a metadata field, optionally filtered by a search term in content. + Returns unique values for a metadata field, optionally filtered by a search term, with pagination. :param metadata_field: The metadata field name. Can include or omit the "meta." prefix. - :param search_term: If set, only documents whose content contains this term (case-insensitive) - are considered. - :returns: A tuple of (list of unique values, total count of unique values). + :param search_term: Optional search term to filter values, matched as a case-insensitive substring + against the metadata field's value. + :param from_: The offset to start returning values from (for pagination). + :param size: The maximum number of unique values to return. + :returns: A tuple of (paginated list of unique values, total count of unique values). """ key = metadata_field.removeprefix("meta.") if metadata_field.startswith("meta.") else metadata_field + values = {str(doc.meta[key]) for doc in self.storage.values() if key in doc.meta and doc.meta[key] is not None} + if search_term: - docs = [doc for doc in self.storage.values() if doc.content and search_term.lower() in doc.content.lower()] - else: - docs = list(self.storage.values()) - values = sorted({str(doc.meta[key]) for doc in docs if key in doc.meta and doc.meta[key] is not None}, key=str) - return values, len(values) + search_term_lower = search_term.lower() + values = {value for value in values if search_term_lower in value.lower()} + + sorted_values = sorted(values) + return sorted_values[from_ : from_ + size], len(sorted_values) def bm25_retrieval( self, query: str, filters: dict[str, Any] | None = None, top_k: int = 10, scale_score: bool = False @@ -963,19 +967,23 @@ async def get_metadata_field_min_max_async(self, metadata_field: str) -> dict[st ) async def get_metadata_field_unique_values_async( - self, metadata_field: str, search_term: str | None = None + self, metadata_field: str, search_term: str | None = None, from_: int = 0, size: int = 10 ) -> tuple[list[str], int]: """ - Returns unique values for a metadata field, optionally filtered by a search term in content. + Returns unique values for a metadata field, optionally filtered by a search term, with pagination. :param metadata_field: The metadata field name. Can include or omit the "meta." prefix. - :param search_term: If set, only documents whose content contains this term (case-insensitive) - are considered. - :returns: A tuple of (list of unique values, total count of unique values). + :param search_term: Optional search term to filter values, matched as a case-insensitive substring + against the metadata field's value. + :param from_: The offset to start returning values from (for pagination). + :param size: The maximum number of unique values to return. + :returns: A tuple of (paginated list of unique values, total count of unique values). """ return await asyncio.get_running_loop().run_in_executor( self.executor, - lambda: self.get_metadata_field_unique_values(metadata_field=metadata_field, search_term=search_term), + lambda: self.get_metadata_field_unique_values( + metadata_field=metadata_field, search_term=search_term, from_=from_, size=size + ), ) async def delete_all_documents_async(self) -> None: diff --git a/haystack/testing/document_store.py b/haystack/testing/document_store.py index 0139ca0f480..7e1c7aa7c82 100644 --- a/haystack/testing/document_store.py +++ b/haystack/testing/document_store.py @@ -1204,8 +1204,8 @@ class GetMetadataFieldUniqueValuesTest: """ Tests for Document Store get_metadata_field_unique_values(). - Only mix in for stores that implement get_metadata_field_unique_values. - Expects the method to return (values_list, total_count) or (values_list, pagination_key). + Only mix in for stores that implement get_metadata_field_unique_values() with the standardized + signature: get_metadata_field_unique_values(field, search_term=None, from_=0, size=10) -> (values, total_count). """ @staticmethod @@ -1221,27 +1221,87 @@ def test_get_metadata_field_unique_values_basic(document_store: DocumentStore): document_store.write_documents(docs) assert document_store.count_documents() == 5 - sig = inspect.signature(document_store.get_metadata_field_unique_values) # type:ignore[attr-defined] - params: dict = {} - if "search_term" in sig.parameters: - params["search_term"] = None - if "from_" in sig.parameters: - params["from_"] = 0 - elif "offset" in sig.parameters: - params["offset"] = 0 - if "size" in sig.parameters: - params["size"] = 10 - elif "limit" in sig.parameters: - params["limit"] = 10 - - result = document_store.get_metadata_field_unique_values("category", **params) # type:ignore[attr-defined] - - values = result[0] if isinstance(result, tuple) else result + values, total_count = document_store.get_metadata_field_unique_values("category") # type:ignore[attr-defined] + assert isinstance(values, list) assert len(values) == 3 # the returned values must not contain duplicates assert set(values) == {"A", "B", "C"} - if isinstance(result, tuple) and len(result) >= 2 and isinstance(result[1], int): - assert result[1] == 3 + assert total_count == 3 + + @staticmethod + def test_get_metadata_field_unique_values_meta_prefix(document_store: DocumentStore): + """Test get_metadata_field_unique_values() with a field name that includes the 'meta.' prefix.""" + docs = [ + Document(content="Doc 1", meta={"category": "A"}), + Document(content="Doc 2", meta={"category": "B"}), + Document(content="Doc 3", meta={"category": "A"}), + ] + document_store.write_documents(docs) + + values, total_count = document_store.get_metadata_field_unique_values("category") # type:ignore[attr-defined] + prefixed_values, prefixed_total_count = document_store.get_metadata_field_unique_values( # type:ignore[attr-defined] + "meta.category" + ) + + assert set(prefixed_values) == set(values) + assert prefixed_total_count == total_count + + @staticmethod + def test_get_metadata_field_unique_values_search_term(document_store: DocumentStore): + """Test get_metadata_field_unique_values() filters by a case-insensitive substring of the value.""" + docs = [ + Document(content="Doc 1", meta={"category": "All_Beauty"}), + Document(content="Doc 2", meta={"category": "Electronics"}), + # the content mentions "beaut" but the value doesn't - must not match search_term="beaut" + Document(content="something about beauty products", meta={"category": "Home"}), + ] + document_store.write_documents(docs) + + values, total_count = document_store.get_metadata_field_unique_values( # type:ignore[attr-defined] + "category", search_term="beaut" + ) + + assert values == ["All_Beauty"] + assert total_count == 1 + + @staticmethod + def test_get_metadata_field_unique_values_pagination(document_store: DocumentStore): + """Test get_metadata_field_unique_values() paginates a stable ordering via from_/size.""" + docs = [Document(content=f"Doc {i}", meta={"category": f"category_{i}"}) for i in range(5)] + document_store.write_documents(docs) + + all_values = sorted(f"category_{i}" for i in range(5)) + + first_page, total_count = document_store.get_metadata_field_unique_values( # type:ignore[attr-defined] + "category", from_=0, size=2 + ) + assert first_page == all_values[:2] + assert total_count == 5 + + second_page, total_count = document_store.get_metadata_field_unique_values( # type:ignore[attr-defined] + "category", from_=2, size=2 + ) + assert second_page == all_values[2:4] + assert total_count == 5 + + @staticmethod + def test_get_metadata_field_unique_values_empty_store(document_store: DocumentStore): + """Test get_metadata_field_unique_values() on an empty store.""" + assert document_store.count_documents() == 0 + + values, total_count = document_store.get_metadata_field_unique_values("category") # type:ignore[attr-defined] + assert values == [] + assert total_count == 0 + + @staticmethod + def test_get_metadata_field_unique_values_missing_field(document_store: DocumentStore): + """Test get_metadata_field_unique_values() for a field that no document has.""" + docs = [Document(content="Doc 1", meta={"category": "A"})] + document_store.write_documents(docs) + + values, total_count = document_store.get_metadata_field_unique_values("missing_field") # type:ignore[attr-defined] + assert values == [] + assert total_count == 0 class DocumentStoreBaseTests(CountDocumentsTest, DeleteDocumentsTest, FilterDocumentsTest, WriteDocumentsTest): diff --git a/haystack/testing/document_store_async.py b/haystack/testing/document_store_async.py index ebe6f22eaf4..f977b3adcdb 100644 --- a/haystack/testing/document_store_async.py +++ b/haystack/testing/document_store_async.py @@ -709,8 +709,9 @@ class GetMetadataFieldUniqueValuesAsyncTest: """ Tests for Document Store get_metadata_field_unique_values_async(). - Only mix in for stores that implement get_metadata_field_unique_values_async. - Expects the method to return (values_list, total_count) or (values_list, pagination_key). + Only mix in for stores that implement get_metadata_field_unique_values_async() with the standardized + signature: get_metadata_field_unique_values_async(field, search_term=None, from_=0, size=10) + -> (values, total_count). """ @staticmethod @@ -727,27 +728,100 @@ async def test_get_metadata_field_unique_values_basic_async(document_store: Asyn await document_store.write_documents_async(docs) assert await document_store.count_documents_async() == 5 - sig = inspect.signature(document_store.get_metadata_field_unique_values_async) # type:ignore[attr-defined] - params: dict = {} - if "search_term" in sig.parameters: - params["search_term"] = None - if "from_" in sig.parameters: - params["from_"] = 0 - elif "offset" in sig.parameters: - params["offset"] = 0 - if "size" in sig.parameters: - params["size"] = 10 - elif "limit" in sig.parameters: - params["limit"] = 10 - - result = await document_store.get_metadata_field_unique_values_async("category", **params) # type:ignore[attr-defined] - - values = result[0] if isinstance(result, tuple) else result + values, total_count = await document_store.get_metadata_field_unique_values_async( # type:ignore[attr-defined] + "category" + ) + assert isinstance(values, list) assert len(values) == 3 # the returned values must not contain duplicates assert set(values) == {"A", "B", "C"} - if isinstance(result, tuple) and len(result) >= 2 and isinstance(result[1], int): - assert result[1] == 3 + assert total_count == 3 + + @staticmethod + @pytest.mark.asyncio + async def test_get_metadata_field_unique_values_meta_prefix_async(document_store: AsyncDocumentStore): + """Test get_metadata_field_unique_values_async() with a field name that includes the 'meta.' prefix.""" + docs = [ + Document(content="Doc 1", meta={"category": "A"}), + Document(content="Doc 2", meta={"category": "B"}), + Document(content="Doc 3", meta={"category": "A"}), + ] + await document_store.write_documents_async(docs) + + values, total_count = await document_store.get_metadata_field_unique_values_async( # type:ignore[attr-defined] + "category" + ) + prefixed_values, prefixed_total_count = await document_store.get_metadata_field_unique_values_async( # type:ignore[attr-defined] + "meta.category" + ) + + assert set(prefixed_values) == set(values) + assert prefixed_total_count == total_count + + @staticmethod + @pytest.mark.asyncio + async def test_get_metadata_field_unique_values_search_term_async(document_store: AsyncDocumentStore): + """Test get_metadata_field_unique_values_async() filters by a case-insensitive substring of the value.""" + docs = [ + Document(content="Doc 1", meta={"category": "All_Beauty"}), + Document(content="Doc 2", meta={"category": "Electronics"}), + # the content mentions "beaut" but the value doesn't - must not match search_term="beaut" + Document(content="something about beauty products", meta={"category": "Home"}), + ] + await document_store.write_documents_async(docs) + + values, total_count = await document_store.get_metadata_field_unique_values_async( # type:ignore[attr-defined] + "category", search_term="beaut" + ) + + assert values == ["All_Beauty"] + assert total_count == 1 + + @staticmethod + @pytest.mark.asyncio + async def test_get_metadata_field_unique_values_pagination_async(document_store: AsyncDocumentStore): + """Test get_metadata_field_unique_values_async() paginates a stable ordering via from_/size.""" + docs = [Document(content=f"Doc {i}", meta={"category": f"category_{i}"}) for i in range(5)] + await document_store.write_documents_async(docs) + + all_values = sorted(f"category_{i}" for i in range(5)) + + first_page, total_count = await document_store.get_metadata_field_unique_values_async( # type:ignore[attr-defined] + "category", from_=0, size=2 + ) + assert first_page == all_values[:2] + assert total_count == 5 + + second_page, total_count = await document_store.get_metadata_field_unique_values_async( # type:ignore[attr-defined] + "category", from_=2, size=2 + ) + assert second_page == all_values[2:4] + assert total_count == 5 + + @staticmethod + @pytest.mark.asyncio + async def test_get_metadata_field_unique_values_empty_store_async(document_store: AsyncDocumentStore): + """Test get_metadata_field_unique_values_async() on an empty store.""" + assert await document_store.count_documents_async() == 0 + + values, total_count = await document_store.get_metadata_field_unique_values_async( # type:ignore[attr-defined] + "category" + ) + assert values == [] + assert total_count == 0 + + @staticmethod + @pytest.mark.asyncio + async def test_get_metadata_field_unique_values_missing_field_async(document_store: AsyncDocumentStore): + """Test get_metadata_field_unique_values_async() for a field that no document has.""" + docs = [Document(content="Doc 1", meta={"category": "A"})] + await document_store.write_documents_async(docs) + + values, total_count = await document_store.get_metadata_field_unique_values_async( # type:ignore[attr-defined] + "missing_field" + ) + assert values == [] + assert total_count == 0 class FilterDocumentsAsyncTest(AssertDocumentsEqualMixin, FilterableDocsFixtureMixin): diff --git a/releasenotes/notes/inmemory-unique-metadata-values-pagination-a3f6c9d21b7e4081.yaml b/releasenotes/notes/inmemory-unique-metadata-values-pagination-a3f6c9d21b7e4081.yaml new file mode 100644 index 00000000000..eb2c4eef1e4 --- /dev/null +++ b/releasenotes/notes/inmemory-unique-metadata-values-pagination-a3f6c9d21b7e4081.yaml @@ -0,0 +1,7 @@ +--- +enhancements: + - | + ``InMemoryDocumentStore.get_metadata_field_unique_values`` and its async counterpart now support + pagination via ``from_`` and ``size`` parameters, matching the behavior of other Document Stores + (e.g. Chroma). ``search_term`` now matches against the metadata field's own value (case-insensitive + substring) instead of the document's content.