chore: deprecate SerperDevWebSearch + update docs (#11577)

Co-authored-by: Claude Fable 5 <noreply@anthropic.com>

Author: julian-risch

docs-website/docs/pipeline-components/websearch/serperdevwebsearch.mdx

@@ -17,9 +17,9 @@ Search engine using SerperDev API.
 | **Mandatory init variables** | `api_key`: The SearchAPI API key. Can be set with `SERPERDEV_API_KEY` env var. |
 | **Mandatory run variables** | `query`: A string with your query |
 | **Output variables** | `documents`: A list of documents  <br /> <br />`links`: A list of strings of resulting links |
-| **API reference** | [Websearch](/reference/websearch-api) |
-| **GitHub link** | https://github.com/deepset-ai/haystack/blob/main/haystack/components/websearch/serper_dev.py |
-| **Package name** | `haystack-ai` |
+| **API reference** | [SerperDev](/reference/integrations-serperdev) |
+| **GitHub link** | https://github.com/deepset-ai/haystack-core-integrations/tree/main/integrations/serperdev |
+| **Package name** | `serperdev-haystack` |
 
 </div>
 
@@ -38,12 +38,18 @@ To use [Search API](https://www.searchapi.io/) as an alternative, see its respec
 
 ## Usage
 
+Install the `serperdev-haystack` package to use the `SerperDevWebSearch` component:
+
+```shell
+pip install serperdev-haystack
+```
+
 ### On its own
 
 This is an example of how `SerperDevWebSearch` looks up answers to our query on the web and converts the results into a list of documents with content snippets of the results, as well as URLs as strings.
 
 ```python
-from haystack.components.websearch import SerperDevWebSearch
+from haystack_integrations.components.websearch.serperdev import SerperDevWebSearch
 from haystack.utils import Secret
 
 web_search = SerperDevWebSearch(api_key=Secret.from_token("<your-api-key>"))
@@ -63,7 +69,7 @@ from haystack.components.builders.chat_prompt_builder import ChatPromptBuilder
 from haystack.components.fetchers import LinkContentFetcher
 from haystack.components.converters import HTMLToDocument
 from haystack.components.generators.chat import OpenAIChatGenerator
-from haystack.components.websearch import SerperDevWebSearch
+from haystack_integrations.components.websearch.serperdev import SerperDevWebSearch
 from haystack.dataclasses import ChatMessage
 from haystack.utils import Secret
 
@@ -181,7 +187,7 @@ components:
       exclude_subdomains: false
       search_params: {}
       top_k: 2
-    type: haystack.components.websearch.serper_dev.SerperDevWebSearch
+    type: haystack_integrations.components.websearch.serperdev.websearch.SerperDevWebSearch
 connection_type_validation: true
 connections:
 - receiver: fetcher.urls

docs-website/versioned_docs/version-2.30/pipeline-components/websearch/serperdevwebsearch.mdx

@@ -17,9 +17,9 @@ Search engine using SerperDev API.
 | **Mandatory init variables** | `api_key`: The SearchAPI API key. Can be set with `SERPERDEV_API_KEY` env var. |
 | **Mandatory run variables** | `query`: A string with your query |
 | **Output variables** | `documents`: A list of documents  <br /> <br />`links`: A list of strings of resulting links |
-| **API reference** | [Websearch](/reference/websearch-api) |
-| **GitHub link** | https://github.com/deepset-ai/haystack/blob/main/haystack/components/websearch/serper_dev.py |
-| **Package name** | `haystack-ai` |
+| **API reference** | [SerperDev](/reference/integrations-serperdev) |
+| **GitHub link** | https://github.com/deepset-ai/haystack-core-integrations/tree/main/integrations/serperdev |
+| **Package name** | `serperdev-haystack` |
 
 </div>
 
@@ -38,12 +38,18 @@ To use [Search API](https://www.searchapi.io/) as an alternative, see its respec
 
 ## Usage
 
+Install the `serperdev-haystack` package to use the `SerperDevWebSearch` component:
+
+```shell
+pip install serperdev-haystack
+```
+
 ### On its own
 
 This is an example of how `SerperDevWebSearch` looks up answers to our query on the web and converts the results into a list of documents with content snippets of the results, as well as URLs as strings.
 
 ```python
-from haystack.components.websearch import SerperDevWebSearch
+from haystack_integrations.components.websearch.serperdev import SerperDevWebSearch
 from haystack.utils import Secret
 
 web_search = SerperDevWebSearch(api_key=Secret.from_token("<your-api-key>"))
@@ -63,7 +69,7 @@ from haystack.components.builders.chat_prompt_builder import ChatPromptBuilder
 from haystack.components.fetchers import LinkContentFetcher
 from haystack.components.converters import HTMLToDocument
 from haystack.components.generators.chat import OpenAIChatGenerator
-from haystack.components.websearch import SerperDevWebSearch
+from haystack_integrations.components.websearch.serperdev import SerperDevWebSearch
 from haystack.dataclasses import ChatMessage
 from haystack.utils import Secret
 
@@ -181,7 +187,7 @@ components:
       exclude_subdomains: false
       search_params: {}
       top_k: 2
-    type: haystack.components.websearch.serper_dev.SerperDevWebSearch
+    type: haystack_integrations.components.websearch.serperdev.websearch.SerperDevWebSearch
 connection_type_validation: true
 connections:
 - receiver: fetcher.urls

haystack/components/websearch/serper_dev.py

@@ -2,6 +2,7 @@
 #
 # SPDX-License-Identifier: Apache-2.0
 
+import warnings
 from typing import Any
 from urllib.parse import urlparse
 
@@ -73,6 +74,15 @@ def __init__(
             For example, you can set 'num' to 20 to increase the number of search results.
             See the [Serper website](https://serper.dev/) for more details.
         """
+        warnings.warn(
+            "`SerperDevWebSearch` will be removed from Haystack in version 3.0, as it is moving to "
+            "the `serperdev-haystack` package. To continue using it, install that package with "
+            "`pip install serperdev-haystack` and update your import to "
+            "`from haystack_integrations.components.websearch.serperdev import SerperDevWebSearch`.",
+            FutureWarning,
+            stacklevel=2,
+        )
+
         self.api_key = api_key
         self.top_k = top_k
         self.allowed_domains = allowed_domains

releasenotes/notes/deprecate-serperdev-websearch-9de15a703cba06cc.yaml

@@ -0,0 +1,10 @@
+---
+deprecations:
+  - |
+    ``SerperDevWebSearch`` is deprecated and will be removed from Haystack in version 3.0. It is moving to
+    the ``serperdev-haystack`` package. To continue using it, install the package with
+    ``pip install serperdev-haystack`` and update your import as follows:
+
+    .. code-block:: python
+
+        from haystack_integrations.components.websearch.serperdev import SerperDevWebSearch

test/components/websearch/test_serperdev.py

@@ -282,7 +282,10 @@ def test_web_search(self) -> None:
         results = ws.run(query="Who is the boyfriend of Olivia Wilde?")
         documents = results["documents"]
         links = results["links"]
-        assert len(documents) == len(links) == 10
+        # documents can also include answer box and "people also ask" results, so only links
+        # (which come from organic results) are guaranteed to be at most top_k
+        assert 0 < len(documents) <= 10
+        assert 0 < len(links) <= 10
         assert all(isinstance(doc, Document) for doc in documents)
         assert all(isinstance(link, str) for link in links)
         assert all(link.startswith("http") for link in links)
@@ -298,7 +301,10 @@ async def test_web_search_async(self) -> None:
         results = await ws.run_async(query="Who is the boyfriend of Olivia Wilde?")
         documents = results["documents"]
         links = results["links"]
-        assert len(documents) == len(links) == 10
+        # documents can also include answer box and "people also ask" results, so only links
+        # (which come from organic results) are guaranteed to be at most top_k
+        assert 0 < len(documents) <= 10
+        assert 0 < len(links) <= 10
         assert all(isinstance(doc, Document) for doc in documents)
         assert all(isinstance(link, str) for link in links)
         assert all(link.startswith("http") for link in links)