count number of docs between deleted operation and return the difference after delete operation (#2807)

Author: davidsbatista

integrations/qdrant/src/haystack_integrations/document_stores/qdrant/document_store.py

@@ -517,7 +517,7 @@ async def delete_documents_async(self, document_ids: list[str]) -> None:
                 "Called QdrantDocumentStore.delete_documents_async() on a non-existing ID",
             )
 
-    def delete_by_filter(self, filters: dict[str, Any]) -> None:
+    def delete_by_filter(self, filters: dict[str, Any]) -> int:
         """
         Deletes all documents that match the provided filters.
 
@@ -533,20 +533,26 @@ def delete_by_filter(self, filters: dict[str, Any]) -> None:
         try:
             qdrant_filter = convert_filters_to_qdrant(filters)
             if qdrant_filter is None:
-                return
+                return 0
+
+            count_response = self._client.count(
+                collection_name=self.index,
+                count_filter=qdrant_filter,
+            )
+            deleted_count = count_response.count
 
-            # perform deletion using FilterSelector
             self._client.delete(
                 collection_name=self.index,
                 points_selector=rest.FilterSelector(filter=qdrant_filter),
                 wait=self.wait_result_from_api,
             )
+            return deleted_count
 
         except Exception as e:
             msg = f"Failed to delete documents by filter from Qdrant: {e!s}"
             raise QdrantStoreError(msg) from e
 
-    async def delete_by_filter_async(self, filters: dict[str, Any]) -> None:
+    async def delete_by_filter_async(self, filters: dict[str, Any]) -> int:
         """
         Asynchronously deletes all documents that match the provided filters.
 
@@ -562,14 +568,20 @@ async def delete_by_filter_async(self, filters: dict[str, Any]) -> None:
         try:
             qdrant_filter = convert_filters_to_qdrant(filters)
             if qdrant_filter is None:
-                return
+                return 0
+
+            count_response = await self._async_client.count(
+                collection_name=self.index,
+                count_filter=qdrant_filter,
+            )
+            deleted_count = count_response.count
 
-            # perform deletion using FilterSelector
             await self._async_client.delete(
                 collection_name=self.index,
                 points_selector=rest.FilterSelector(filter=qdrant_filter),
                 wait=self.wait_result_from_api,
             )
+            return deleted_count
 
         except Exception as e:
             msg = f"Failed to delete documents by filter from Qdrant: {e!s}"

integrations/qdrant/tests/test_document_store.py

@@ -348,15 +348,20 @@ def test_delete_by_filter(self, document_store: QdrantDocumentStore):
         ]
         document_store.write_documents(docs)
         assert document_store.count_documents() == 3
-        document_store.delete_by_filter(filters={"field": "meta.category", "operator": "==", "value": "A"})
+
+        deleted_count = document_store.delete_by_filter(
+            filters={"field": "meta.category", "operator": "==", "value": "A"}
+        )
+        assert deleted_count == 2
 
         # Verify only category B remains
         remaining_docs = document_store.filter_documents()
         assert len(remaining_docs) == 1
         assert remaining_docs[0].meta["category"] == "B"
 
         # Delete remaining document by year
-        document_store.delete_by_filter(filters={"field": "meta.year", "operator": "==", "value": 2023})
+        deleted_count = document_store.delete_by_filter(filters={"field": "meta.year", "operator": "==", "value": 2023})
+        assert deleted_count == 1
         assert document_store.count_documents() == 0
 
     def test_delete_by_filter_no_matches(self, document_store: QdrantDocumentStore):
@@ -368,7 +373,10 @@ def test_delete_by_filter_no_matches(self, document_store: QdrantDocumentStore):
         assert document_store.count_documents() == 2
 
         # try to delete documents with category="C" (no matches)
-        document_store.delete_by_filter(filters={"field": "meta.category", "operator": "==", "value": "C"})
+        deleted_count = document_store.delete_by_filter(
+            filters={"field": "meta.category", "operator": "==", "value": "C"}
+        )
+        assert deleted_count == 0
         assert document_store.count_documents() == 2
 
     def test_delete_by_filter_advanced_filters(self, document_store: QdrantDocumentStore):
@@ -380,8 +388,8 @@ def test_delete_by_filter_advanced_filters(self, document_store: QdrantDocumentS
         document_store.write_documents(docs)
         assert document_store.count_documents() == 3
 
-        # AND condition
-        document_store.delete_by_filter(
+        # AND condition (matches only Doc 1)
+        deleted_count = document_store.delete_by_filter(
             filters={
                 "operator": "AND",
                 "conditions": [
@@ -390,10 +398,11 @@ def test_delete_by_filter_advanced_filters(self, document_store: QdrantDocumentS
                 ],
             }
         )
+        assert deleted_count == 1
         assert document_store.count_documents() == 2
 
-        # OR condition
-        document_store.delete_by_filter(
+        # OR condition (matches Doc 2 and Doc 3)
+        deleted_count = document_store.delete_by_filter(
             filters={
                 "operator": "OR",
                 "conditions": [
@@ -402,6 +411,7 @@ def test_delete_by_filter_advanced_filters(self, document_store: QdrantDocumentS
                 ],
             }
         )
+        assert deleted_count == 2
         assert document_store.count_documents() == 0
 
     def test_update_by_filter(self, document_store: QdrantDocumentStore):

integrations/qdrant/tests/test_document_store_async.py

@@ -275,7 +275,10 @@ async def test_delete_by_filter_async(self, document_store: QdrantDocumentStore)
         assert await document_store.count_documents_async() == 3
 
         # Delete documents with category="A"
-        await document_store.delete_by_filter_async(filters={"field": "meta.category", "operator": "==", "value": "A"})
+        deleted_count = await document_store.delete_by_filter_async(
+            filters={"field": "meta.category", "operator": "==", "value": "A"}
+        )
+        assert deleted_count == 2
         assert await document_store.count_documents_async() == 1
 
         # Verify only category B remains
@@ -286,7 +289,10 @@ async def test_delete_by_filter_async(self, document_store: QdrantDocumentStore)
         assert remaining_docs[0].meta["category"] == "B"
 
         # Delete remaining document by year
-        await document_store.delete_by_filter_async(filters={"field": "meta.year", "operator": "==", "value": 2023})
+        deleted_count = await document_store.delete_by_filter_async(
+            filters={"field": "meta.year", "operator": "==", "value": 2023}
+        )
+        assert deleted_count == 1
         assert await document_store.count_documents_async() == 0
 
     @pytest.mark.asyncio
@@ -299,7 +305,10 @@ async def test_delete_by_filter_async_no_matches(self, document_store: QdrantDoc
         assert await document_store.count_documents_async() == 2
 
         # Try to delete documents with category="C" (no matches)
-        await document_store.delete_by_filter_async(filters={"field": "meta.category", "operator": "==", "value": "C"})
+        deleted_count = await document_store.delete_by_filter_async(
+            filters={"field": "meta.category", "operator": "==", "value": "C"}
+        )
+        assert deleted_count == 0
         assert await document_store.count_documents_async() == 2
 
     @pytest.mark.asyncio
@@ -312,8 +321,8 @@ async def test_delete_by_filter_async_advanced_filters(self, document_store: Qdr
         await document_store.write_documents_async(docs)
         assert await document_store.count_documents_async() == 3
 
-        # AND condition
-        await document_store.delete_by_filter_async(
+        # AND condition (matches only Doc 1)
+        deleted_count = await document_store.delete_by_filter_async(
             filters={
                 "operator": "AND",
                 "conditions": [
@@ -322,10 +331,11 @@ async def test_delete_by_filter_async_advanced_filters(self, document_store: Qdr
                 ],
             }
         )
+        assert deleted_count == 1
         assert await document_store.count_documents_async() == 2
 
-        # OR condition
-        await document_store.delete_by_filter_async(
+        # OR condition (matches Doc 2 and Doc 3)
+        deleted_count = await document_store.delete_by_filter_async(
             filters={
                 "operator": "OR",
                 "conditions": [
@@ -334,6 +344,7 @@ async def test_delete_by_filter_async_advanced_filters(self, document_store: Qdr
                 ],
             }
         )
+        assert deleted_count == 2
         assert await document_store.count_documents_async() == 0
 
     @pytest.mark.asyncio