fix: Weaviate match search_term against metadata field value, not content + default size value set to 10 (#3643)

Author: davidsbatista

integrations/weaviate/src/haystack_integrations/document_stores/weaviate/document_store.py

@@ -632,24 +632,53 @@ async def count_unique_metadata_by_filter_async(
 
         return result
 
+    @staticmethod
+    def _compute_field_unique_values(
+        agg_result: Any, search_term: str | None, from_: int, size: int
+    ) -> tuple[list[str], int]:
+        """
+        Computes paginated unique values from a Weaviate group-by aggregation result.
+
+        Weaviate's `like` filter only supports case-sensitive substring matching, so instead of
+        filtering server-side (which would be case-sensitive), all groups are fetched unconditionally
+        and `search_term` is applied here as a case-insensitive substring match in Python.
+
+        :param agg_result: The group-by aggregation result, with one group per unique field value.
+        :param search_term: Optional term to filter the values by, matched as a case-insensitive
+            substring against each value.
+        :param from_: The starting offset for pagination (0-indexed).
+        :param size: The maximum number of unique values to return.
+        :returns: A tuple of (paginated list of unique values, total count of unique values).
+        """
+        all_values = [str(g.grouped_by.value) for g in agg_result.groups] if agg_result.groups else []
+        if search_term:
+            search_term_lower = search_term.lower()
+            all_values = [value for value in all_values if search_term_lower in value.lower()]
+        all_values.sort()  # Sort for consistent pagination
+        total_count = len(all_values)
+
+        paginated_values = all_values[from_ : from_ + size]
+
+        return paginated_values, total_count
+
     def get_metadata_field_unique_values(
         self,
         metadata_field: str,
         search_term: str | None = None,
         from_: int = 0,
-        size: int = 10000,
+        size: int = 10,
     ) -> tuple[list[str], int]:
         """
         Returns unique values for a metadata field with pagination support.
 
         :param metadata_field: The metadata field name to get unique values for.
             Can be prefixed with 'meta.' (e.g., 'meta.category' or 'category').
-        :param search_term: Optional term to filter documents by content before
-            extracting unique values. If provided, only documents whose content
-            contains this term will be considered.
-            Note: Uses substring matching (case-sensitive, no stemming).
+        :param search_term: Optional term to filter the metadata field's values by
+            before returning them. If provided, only values of `metadata_field` that
+            contain this term will be considered.
+            Note: Uses case-insensitive substring matching (no stemming).
         :param from_: The starting offset for pagination (0-indexed). Defaults to 0.
-        :param size: The maximum number of unique values to return. Defaults to 10000.
+        :param size: The maximum number of unique values to return. Defaults to 10.
         :returns: A tuple of (list of unique values, total count of unique values).
         :raises ValueError: If the field is not found in the collection schema.
         """
@@ -661,45 +690,33 @@ def get_metadata_field_unique_values(
             msg = f"Field '{field_name}' not found in collection schema"
             raise ValueError(msg)
 
-        weaviate_filter = None
-        if search_term:
-            weaviate_filter = convert_filters({"field": "content", "operator": "contains", "value": search_term})
-
         # Weaviate's GroupByAggregate has a default limit of 100 groups.
         # We use the document count as the limit to ensure all unique values are retrieved,
         # since the number of unique values cannot exceed the number of documents.
         doc_count = self.collection.aggregate.over_all(total_count=True).total_count or 0
 
-        agg_result = self.collection.aggregate.over_all(
-            filters=weaviate_filter, group_by=GroupByAggregate(prop=field_name, limit=doc_count)
-        )
+        agg_result = self.collection.aggregate.over_all(group_by=GroupByAggregate(prop=field_name, limit=doc_count))
 
-        all_values = [str(g.grouped_by.value) for g in agg_result.groups] if agg_result.groups else []
-        all_values.sort()  # Sort for consistent pagination
-        total_count = len(all_values)
-
-        paginated_values = all_values[from_ : from_ + size]
-
-        return paginated_values, total_count
+        return self._compute_field_unique_values(agg_result, search_term, from_, size)
 
     async def get_metadata_field_unique_values_async(
         self,
         metadata_field: str,
         search_term: str | None = None,
         from_: int = 0,
-        size: int = 10000,
+        size: int = 10,
     ) -> tuple[list[str], int]:
         """
         Asynchronously returns unique values for a metadata field with pagination support.
 
         :param metadata_field: The metadata field name to get unique values for.
             Can be prefixed with 'meta.' (e.g., 'meta.category' or 'category').
-        :param search_term: Optional term to filter documents by content before
-            extracting unique values. If provided, only documents whose content
-            contains this term will be considered.
-            Note: Uses substring matching (case-sensitive, no stemming).
+        :param search_term: Optional term to filter the metadata field's values by
+            before returning them. If provided, only values of `metadata_field` that
+            contain this term will be considered.
+            Note: Uses case-insensitive substring matching (no stemming).
         :param from_: The starting offset for pagination (0-indexed). Defaults to 0.
-        :param size: The maximum number of unique values to return. Defaults to 10000.
+        :param size: The maximum number of unique values to return. Defaults to 10.
         :returns: A tuple of (list of unique values, total count of unique values).
         :raises ValueError: If the field is not found in the collection schema.
         """
@@ -712,27 +729,15 @@ async def get_metadata_field_unique_values_async(
             msg = f"Field '{field_name}' not found in collection schema"
             raise ValueError(msg)
 
-        weaviate_filter = None
-        if search_term:
-            weaviate_filter = convert_filters({"field": "content", "operator": "contains", "value": search_term})
-
         # Weaviate's GroupByAggregate has a default limit of 100 groups.
         # We use the document count as the limit to ensure all unique values are retrieved,
         # since the number of unique values cannot exceed the number of documents.
         doc_count_result = await collection.aggregate.over_all(total_count=True)
         doc_count = doc_count_result.total_count or 0
 
-        agg_result = await collection.aggregate.over_all(
-            filters=weaviate_filter, group_by=GroupByAggregate(prop=field_name, limit=doc_count)
-        )
+        agg_result = await collection.aggregate.over_all(group_by=GroupByAggregate(prop=field_name, limit=doc_count))
 
-        all_values = [str(g.grouped_by.value) for g in agg_result.groups] if agg_result.groups else []
-        all_values.sort()  # Sort for consistent pagination
-        total_count = len(all_values)
-
-        paginated_values = all_values[from_ : from_ + size]
-
-        return paginated_values, total_count
+        return self._compute_field_unique_values(agg_result, search_term, from_, size)
 
     @staticmethod
     def _to_data_object(document: Document) -> dict[str, Any]:

integrations/weaviate/tests/test_document_store.py

@@ -1156,17 +1156,54 @@ def test_get_metadata_field_unique_values_with_meta_prefix(self, document_store)
         assert set(values) == {"TypeA", "TypeB"}
 
     def test_get_metadata_field_unique_values_with_search_term(self, document_store):
+        # search_term must match against the VALUE of the target metadata field,
+        # not the document content.
         docs = [
-            Document(content="Python programming language", meta={"category": "TypeA"}),
-            Document(content="Java programming language", meta={"category": "TypeB"}),
-            Document(content="Python is great", meta={"category": "TypeC"}),
-            Document(content="JavaScript tutorial", meta={"category": "TypeD"}),
+            Document(content="Some article", meta={"category": "Python Programming"}),
+            Document(content="Some article", meta={"category": "Java Programming"}),
+            Document(content="Some article", meta={"category": "Python Basics"}),
+            Document(content="Some article", meta={"category": "JavaScript Tutorial"}),
         ]
         document_store.write_documents(docs)
 
         values, total_count = document_store.get_metadata_field_unique_values("category", search_term="Python")
         assert total_count == 2
-        assert set(values) == {"TypeA", "TypeC"}
+        assert set(values) == {"Python Programming", "Python Basics"}
+
+    def test_get_metadata_field_unique_values_search_term_excludes_content_only_match(self, document_store):
+        # A document whose content contains the search term but whose target metadata
+        # field value does NOT must be excluded from the results.
+        docs = [
+            Document(content="Python programming language", meta={"category": "TypeA"}),
+        ]
+        document_store.write_documents(docs)
+
+        values, total_count = document_store.get_metadata_field_unique_values("category", search_term="Python")
+        assert total_count == 0
+        assert values == []
+
+    def test_get_metadata_field_unique_values_search_term_matches_metadata_value_only(self, document_store):
+        # A document whose metadata field value contains the search term but whose
+        # content does NOT must be included in the results.
+        docs = [
+            Document(content="Unrelated text about cooking", meta={"category": "Python Basics"}),
+        ]
+        document_store.write_documents(docs)
+
+        values, total_count = document_store.get_metadata_field_unique_values("category", search_term="Python")
+        assert total_count == 1
+        assert values == ["Python Basics"]
+
+    def test_get_metadata_field_unique_values_search_term_case_insensitive(self, document_store):
+        docs = [
+            Document(content="n/a", meta={"category": "Python Basics"}),
+            Document(content="n/a", meta={"category": "Java Basics"}),
+        ]
+        document_store.write_documents(docs)
+
+        values, total_count = document_store.get_metadata_field_unique_values("category", search_term="PYTHON")
+        assert total_count == 1
+        assert values == ["Python Basics"]
 
     def test_get_metadata_field_unique_values_with_pagination(self, document_store):
         docs = [

integrations/weaviate/tests/test_document_store_async.py

@@ -454,19 +454,51 @@ async def test_get_metadata_field_unique_values_async_with_meta_prefix(self, doc
 
     @pytest.mark.asyncio
     async def test_get_metadata_field_unique_values_async_with_search_term(self, document_store):
+        # search_term must match against the VALUE of the target metadata field,
+        # not the document content.
         docs = [
-            Document(content="Python programming language", meta={"category": "TypeA"}),
-            Document(content="Java programming language", meta={"category": "TypeB"}),
-            Document(content="Python is great", meta={"category": "TypeC"}),
-            Document(content="JavaScript tutorial", meta={"category": "TypeD"}),
+            Document(content="Some article", meta={"category": "Python Programming"}),
+            Document(content="Some article", meta={"category": "Java Programming"}),
+            Document(content="Some article", meta={"category": "Python Basics"}),
+            Document(content="Some article", meta={"category": "JavaScript Tutorial"}),
         ]
         await document_store.write_documents_async(docs)
 
         values, total_count = await document_store.get_metadata_field_unique_values_async(
             "category", search_term="Python"
         )
         assert total_count == 2
-        assert set(values) == {"TypeA", "TypeC"}
+        assert set(values) == {"Python Programming", "Python Basics"}
+
+    @pytest.mark.asyncio
+    async def test_get_metadata_field_unique_values_async_search_term_excludes_content_only_match(self, document_store):
+        # A document whose content contains the search term but whose target metadata
+        # field value does NOT must be excluded from the results.
+        docs = [
+            Document(content="Python programming language", meta={"category": "TypeA"}),
+        ]
+        await document_store.write_documents_async(docs)
+
+        values, total_count = await document_store.get_metadata_field_unique_values_async(
+            "category", search_term="Python"
+        )
+        assert total_count == 0
+        assert values == []
+
+    @pytest.mark.asyncio
+    async def test_get_metadata_field_unique_values_async_search_term_matches_metadata_value_only(self, document_store):
+        # A document whose metadata field value contains the search term but whose
+        # content does NOT must be included in the results.
+        docs = [
+            Document(content="Unrelated text about cooking", meta={"category": "Python Basics"}),
+        ]
+        await document_store.write_documents_async(docs)
+
+        values, total_count = await document_store.get_metadata_field_unique_values_async(
+            "category", search_term="Python"
+        )
+        assert total_count == 1
+        assert values == ["Python Basics"]
 
     @pytest.mark.asyncio
     async def test_get_metadata_field_unique_values_async_with_pagination(self, document_store):