diff --git a/docs-website/docs/concepts/agents.mdx b/docs-website/docs/concepts/agents.mdx index 05adf2955da..3dff04932b4 100644 --- a/docs-website/docs/concepts/agents.mdx +++ b/docs-website/docs/concepts/agents.mdx @@ -76,11 +76,17 @@ export OPENAI_API_KEY= export SERPERDEV_API_KEY= ``` +The examples on this page use SerperDev web search component that have moved to the `serperdev-haystack` package. Install it to run the examples: + +```shell +pip install serperdev-haystack +``` + ```python from haystack.components.agents import Agent from haystack.components.generators.chat import OpenAIChatGenerator from haystack.components.generators.utils import print_streaming_chunk -from haystack.components.websearch import SerperDevWebSearch +from haystack_integrations.components.websearch.serperdev import SerperDevWebSearch from haystack.dataclasses import ChatMessage from haystack.tools import ComponentTool diff --git a/docs-website/docs/concepts/agents/multi-agent-systems.mdx b/docs-website/docs/concepts/agents/multi-agent-systems.mdx index 0f9aaba0834..bcfbccdee85 100644 --- a/docs-website/docs/concepts/agents/multi-agent-systems.mdx +++ b/docs-website/docs/concepts/agents/multi-agent-systems.mdx @@ -31,6 +31,12 @@ Wrapping an agent inside a `@tool` function gives you full control over what the This approach works better with smaller LLMs because the tool has a clean, minimal signature. The coordinator only needs to provide a query string - all the `ChatMessage` construction and result unpacking is hidden inside the function. +The examples on this page use SerperDev web search component that have moved to the `serperdev-haystack` package. Install it to run the examples: + +```shell +pip install serperdev-haystack +``` + ```python from typing import Annotated from haystack.components.agents import Agent @@ -38,7 +44,7 @@ from haystack.components.generators.chat import OpenAIChatGenerator from haystack.components.generators.utils import print_streaming_chunk from haystack.dataclasses import ChatMessage from haystack.tools import ComponentTool, tool -from haystack.components.websearch import SerperDevWebSearch +from haystack_integrations.components.websearch.serperdev import SerperDevWebSearch from haystack.utils import Secret @@ -198,7 +204,7 @@ components: exclude_subdomains: false search_params: {} top_k: 3 - type: haystack.components.websearch.serper_dev.SerperDevWebSearch + type: haystack_integrations.components.websearch.serperdev.websearch.SerperDevWebSearch description: Search the web for current information on any topic inputs_from_state: null name: web_search @@ -254,7 +260,7 @@ from haystack.components.converters import HTMLToDocument from haystack.components.fetchers.link_content import LinkContentFetcher from haystack.components.generators.chat import OpenAIChatGenerator from haystack.components.generators.utils import print_streaming_chunk -from haystack.components.websearch import SerperDevWebSearch +from haystack_integrations.components.websearch.serperdev import SerperDevWebSearch from haystack.dataclasses import ChatMessage from haystack.tools import ComponentTool, tool from haystack.utils import Secret diff --git a/docs-website/docs/concepts/components.mdx b/docs-website/docs/concepts/components.mdx index d067ad5e03d..07820dda6d7 100644 --- a/docs-website/docs/concepts/components.mdx +++ b/docs-website/docs/concepts/components.mdx @@ -45,9 +45,17 @@ Returns a list of Documents ranked by their similarity to the given query. Components that use heavy resources, like LLMs or embedding models, have a `warm_up()` method that loads the necessary resources (such as models) into memory. This method is automatically called the first time the component runs, so you can use components directly without explicitly calling `warm_up()`: +The examples on this page use Sentence Transformers embedders that have moved to the `sentence-transformers-haystack` package. Install it to run the examples: + +```shell +pip install sentence-transformers-haystack +``` + ```python from haystack import Document -from haystack.components.embedders import SentenceTransformersDocumentEmbedder +from haystack_integrations.components.embedders.sentence_transformers import ( + SentenceTransformersDocumentEmbedder, +) doc = Document(content="I love pizza!") doc_embedder = SentenceTransformersDocumentEmbedder() diff --git a/docs-website/docs/concepts/components/supercomponents.mdx b/docs-website/docs/concepts/components/supercomponents.mdx index 87978b0b40b..0e0898b0d12 100644 --- a/docs-website/docs/concepts/components/supercomponents.mdx +++ b/docs-website/docs/concepts/components/supercomponents.mdx @@ -19,12 +19,20 @@ With this decorator, the `to_dict` and `from_dict` serialization is optional, as The custom HybridRetriever example SuperComponent below turns your query into embeddings, then runs both a BM25 search and an embedding-based search at the same time. It finally merges those two result sets and returns the combined documents. +The examples on this page use Sentence Transformers embedders that have moved to the `sentence-transformers-haystack` package. Install it to run the examples: + +```shell +pip install sentence-transformers-haystack +``` + ```python -# pip install haystack-ai datasets "sentence-transformers>=3.0.0" +# pip install haystack-ai datasets sentence-transformers-haystack from haystack import Document, Pipeline, super_component from haystack.components.joiners import DocumentJoiner -from haystack.components.embedders import SentenceTransformersTextEmbedder +from haystack_integrations.components.embedders.sentence_transformers import ( + SentenceTransformersTextEmbedder, +) from haystack.components.retrievers import ( InMemoryBM25Retriever, InMemoryEmbeddingRetriever, diff --git a/docs-website/docs/concepts/pipelines/asyncpipeline.mdx b/docs-website/docs/concepts/pipelines/asyncpipeline.mdx index b291245ae1c..75d8ad20e57 100644 --- a/docs-website/docs/concepts/pipelines/asyncpipeline.mdx +++ b/docs-website/docs/concepts/pipelines/asyncpipeline.mdx @@ -50,12 +50,18 @@ You can find more details in our [API Reference](/reference/pipeline-api#asyncpi ## Example +The examples on this page use Sentence Transformers embedders that have moved to the `sentence-transformers-haystack` package. Install it to run the examples: + +```shell +pip install sentence-transformers-haystack +``` + ```python import asyncio from haystack import AsyncPipeline, Document from haystack.components.builders import ChatPromptBuilder -from haystack.components.embedders import ( +from haystack_integrations.components.embedders.sentence_transformers import ( SentenceTransformersDocumentEmbedder, SentenceTransformersTextEmbedder, ) @@ -75,7 +81,6 @@ documents = [ ] docs_embedder = SentenceTransformersDocumentEmbedder() -docs_embedder.warm_up() document_store = InMemoryDocumentStore() document_store.write_documents(docs_embedder.run(documents=documents)["documents"]) diff --git a/docs-website/docs/concepts/pipelines/creating-pipelines.mdx b/docs-website/docs/concepts/pipelines/creating-pipelines.mdx index fbdc7ea5942..0b713a9eed2 100644 --- a/docs-website/docs/concepts/pipelines/creating-pipelines.mdx +++ b/docs-website/docs/concepts/pipelines/creating-pipelines.mdx @@ -26,10 +26,18 @@ For each component you want to use in your pipeline, you must know the names of Import all the dependencies, like pipeline, documents, Document Store, and all the components you want to use in your pipeline. For example, to create a semantic document search pipelines, you need the `Document` object, the pipeline, the Document Store, Embedders, and a Retriever: +The examples on this page use Sentence Transformers embedders that have moved to the `sentence-transformers-haystack` package. Install it to run the examples: + +```shell +pip install sentence-transformers-haystack +``` + ```python from haystack import Document, Pipeline from haystack.document_stores.in_memory import InMemoryDocumentStore -from haystack.components.embedders import SentenceTransformersTextEmbedder +from haystack_integrations.components.embedders.sentence_transformers import ( + SentenceTransformersTextEmbedder, +) from haystack.components.retrievers.in_memory import InMemoryEmbeddingRetriever ``` diff --git a/docs-website/docs/document-stores/oracledocumentstore.mdx b/docs-website/docs/document-stores/oracledocumentstore.mdx index f5c07281137..6a8b13d80bc 100644 --- a/docs-website/docs/document-stores/oracledocumentstore.mdx +++ b/docs-website/docs/document-stores/oracledocumentstore.mdx @@ -25,6 +25,12 @@ It stores documents alongside dense vector embeddings in a native `VECTOR` colum pip install oracle-haystack ``` +The examples on this page use Sentence Transformers embedders that have moved to the `sentence-transformers-haystack` package. Install it to run the examples: + +```shell +pip install sentence-transformers-haystack +``` + ## Connection `OracleDocumentStore` connects to Oracle using the `OracleConnectionConfig` dataclass, which supports two connection modes: @@ -108,7 +114,7 @@ document_store = OracleDocumentStore( ```python from haystack import Document, Pipeline from haystack.document_stores.types import DuplicatePolicy -from haystack.components.embedders import ( +from haystack_integrations.components.embedders.sentence_transformers import ( SentenceTransformersDocumentEmbedder, SentenceTransformersTextEmbedder, ) @@ -146,7 +152,6 @@ documents = [ doc_embedder = SentenceTransformersDocumentEmbedder( model="sentence-transformers/all-MiniLM-L6-v2", ) -doc_embedder.warm_up() embedded_docs = doc_embedder.run(documents)["documents"] document_store.write_documents(embedded_docs, policy=DuplicatePolicy.OVERWRITE) diff --git a/docs-website/docs/document-stores/supabasedocumentstore.mdx b/docs-website/docs/document-stores/supabasedocumentstore.mdx index f249ae97ebd..e1179db3ed8 100644 --- a/docs-website/docs/document-stores/supabasedocumentstore.mdx +++ b/docs-website/docs/document-stores/supabasedocumentstore.mdx @@ -27,6 +27,12 @@ description: "Use Supabase as a document store in Haystack, with vector search ( pip install supabase-haystack ``` +The examples on this page use Sentence Transformers embedders that have moved to the `sentence-transformers-haystack` package. Install it to run the examples: + +```shell +pip install sentence-transformers-haystack +``` + ## SupabasePgvectorDocumentStore `SupabasePgvectorDocumentStore` is a thin wrapper around [`PgvectorDocumentStore`](./pgvectordocumentstore.mdx) with Supabase-specific defaults: @@ -70,7 +76,7 @@ To learn more about the initialization parameters, see the [API docs](/reference ```python from haystack import Document, Pipeline from haystack.document_stores.types.policy import DuplicatePolicy -from haystack.components.embedders import ( +from haystack_integrations.components.embedders.sentence_transformers import ( SentenceTransformersTextEmbedder, SentenceTransformersDocumentEmbedder, ) diff --git a/docs-website/docs/document-stores/valkeydocumentstore.mdx b/docs-website/docs/document-stores/valkeydocumentstore.mdx index 0a6db044dd6..03af41a0e64 100644 --- a/docs-website/docs/document-stores/valkeydocumentstore.mdx +++ b/docs-website/docs/document-stores/valkeydocumentstore.mdx @@ -28,6 +28,12 @@ You can install the Valkey Haystack integration with: pip install valkey-haystack ``` +The examples on this page use Sentence Transformers embedders that have moved to the `sentence-transformers-haystack` package. Install it to run the examples: + +```shell +pip install sentence-transformers-haystack +``` + ## Initialization To use Valkey as your data storage for Haystack pipelines, you need a Valkey server with the search module running. Initialize a `ValkeyDocumentStore` like this: @@ -65,7 +71,9 @@ To write documents to your `ValkeyDocumentStore`, create an indexing pipeline or from haystack import Pipeline from haystack.components.converters import MarkdownToDocument from haystack.components.writers import DocumentWriter -from haystack.components.embedders import SentenceTransformersDocumentEmbedder +from haystack_integrations.components.embedders.sentence_transformers import ( + SentenceTransformersDocumentEmbedder, +) from haystack.components.preprocessors import DocumentSplitter from haystack_integrations.document_stores.valkey import ValkeyDocumentStore @@ -99,7 +107,9 @@ Once documents are in your `ValkeyDocumentStore`, you can use [`ValkeyEmbeddingR from haystack import Pipeline from haystack.utils import Secret from haystack.dataclasses import ChatMessage -from haystack.components.embedders import SentenceTransformersTextEmbedder +from haystack_integrations.components.embedders.sentence_transformers import ( + SentenceTransformersTextEmbedder, +) from haystack.components.builders import ChatPromptBuilder from haystack.components.generators.chat import OpenAIChatGenerator from haystack_integrations.document_stores.valkey import ValkeyDocumentStore diff --git a/docs-website/docs/optimization/advanced-rag-techniques/hypothetical-document-embeddings-hyde.mdx b/docs-website/docs/optimization/advanced-rag-techniques/hypothetical-document-embeddings-hyde.mdx index c8b2d8fd667..5c8956337cc 100644 --- a/docs-website/docs/optimization/advanced-rag-techniques/hypothetical-document-embeddings-hyde.mdx +++ b/docs-website/docs/optimization/advanced-rag-techniques/hypothetical-document-embeddings-hyde.mdx @@ -28,6 +28,12 @@ Many embedding retrievers generalize poorly to new, unseen domains. This approac First, prepare all the components that you would need: +The examples on this page use Sentence Transformers embedders that have moved to the `sentence-transformers-haystack` package. Install it to run the examples: + +```shell +pip install sentence-transformers-haystack +``` + ```python import os from numpy import array, mean @@ -37,7 +43,9 @@ from haystack.components.generators.chat import OpenAIChatGenerator from haystack.components.builders import ChatPromptBuilder from haystack import component, Document from haystack.components.converters import OutputAdapter -from haystack.components.embedders import SentenceTransformersDocumentEmbedder +from haystack_integrations.components.embedders.sentence_transformers import ( + SentenceTransformersDocumentEmbedder, +) from haystack.dataclasses import ChatMessage # We need to ensure we have the OpenAI API key in our environment variables @@ -69,7 +77,6 @@ adapter = OutputAdapter( embedder = SentenceTransformersDocumentEmbedder( model="sentence-transformers/all-MiniLM-L6-v2", ) -embedder.warm_up() # Adding one custom component that returns one, "average" embedding from multiple (hypothetical) document embeddings diff --git a/docs-website/docs/overview/get-started.mdx b/docs-website/docs/overview/get-started.mdx index e3bce8b76dd..b2bc68027a7 100644 --- a/docs-website/docs/overview/get-started.mdx +++ b/docs-website/docs/overview/get-started.mdx @@ -88,11 +88,19 @@ print(results["llm"]["replies"]) -[HuggingFaceAPIChatGenerator](../pipeline-components/generators/huggingfaceapichatgenerator.mdx) is included in the `haystack-ai` package. You can get a [free Hugging Face token](https://huggingface.co/settings/tokens) to use the Serverless Inference API. +[HuggingFaceAPIChatGenerator](../pipeline-components/generators/huggingfaceapichatgenerator.mdx) is included in the `huggingface-api-haystack` package. You can get a [free Hugging Face token](https://huggingface.co/settings/tokens) to use the Serverless Inference API. + +The examples on this page use the Hugging Face API components and the SerperDev web search component, which have moved to the `huggingface-api-haystack` and `serperdev-haystack` packages. Install them to run the examples: + +```shell +pip install huggingface-api-haystack serperdev-haystack +``` ```python from haystack import Pipeline, Document -from haystack.components.generators.chat import HuggingFaceAPIChatGenerator +from haystack_integrations.components.generators.huggingface_api import ( + HuggingFaceAPIChatGenerator, +) from haystack.components.retrievers import InMemoryBM25Retriever from haystack.document_stores.in_memory import InMemoryDocumentStore from haystack.components.builders import ChatPromptBuilder @@ -389,7 +397,7 @@ from haystack.components.agents import Agent from haystack.components.generators.chat import OpenAIChatGenerator from haystack.dataclasses import ChatMessage from haystack.tools import ComponentTool -from haystack.components.websearch import SerperDevWebSearch +from haystack_integrations.components.websearch.serperdev import SerperDevWebSearch from haystack.utils import Secret search_tool = ComponentTool(component=SerperDevWebSearch()) @@ -411,14 +419,16 @@ print(result["last_message"].text) -[HuggingFaceAPIChatGenerator](../pipeline-components/generators/huggingfaceapichatgenerator.mdx) is included in the `haystack-ai` package. You can get a [free Hugging Face token](https://huggingface.co/settings/tokens) to use the Serverless Inference API. +[HuggingFaceAPIChatGenerator](../pipeline-components/generators/huggingfaceapichatgenerator.mdx) is included in the `huggingface-api-haystack` package. You can get a [free Hugging Face token](https://huggingface.co/settings/tokens) to use the Serverless Inference API. ```python from haystack.components.agents import Agent -from haystack.components.generators.chat import HuggingFaceAPIChatGenerator +from haystack_integrations.components.generators.huggingface_api import ( + HuggingFaceAPIChatGenerator, +) from haystack.dataclasses import ChatMessage from haystack.tools import ComponentTool -from haystack.components.websearch import SerperDevWebSearch +from haystack_integrations.components.websearch.serperdev import SerperDevWebSearch from haystack.utils import Secret search_tool = ComponentTool(component=SerperDevWebSearch()) @@ -454,7 +464,7 @@ from haystack.components.agents import Agent from haystack_integrations.components.generators.anthropic import AnthropicChatGenerator from haystack.dataclasses import ChatMessage from haystack.tools import ComponentTool -from haystack.components.websearch import SerperDevWebSearch +from haystack_integrations.components.websearch.serperdev import SerperDevWebSearch from haystack.utils import Secret search_tool = ComponentTool(component=SerperDevWebSearch()) @@ -492,7 +502,7 @@ from haystack_integrations.components.generators.amazon_bedrock import ( ) from haystack.dataclasses import ChatMessage from haystack.tools import ComponentTool -from haystack.components.websearch import SerperDevWebSearch +from haystack_integrations.components.websearch.serperdev import SerperDevWebSearch os.environ["AWS_ACCESS_KEY_ID"] = "YOUR_AWS_ACCESS_KEY_ID" os.environ["AWS_SECRET_ACCESS_KEY"] = "YOUR_AWS_SECRET_ACCESS_KEY" @@ -531,7 +541,7 @@ from haystack_integrations.components.generators.google_genai import ( ) from haystack.dataclasses import ChatMessage from haystack.tools import ComponentTool -from haystack.components.websearch import SerperDevWebSearch +from haystack_integrations.components.websearch.serperdev import SerperDevWebSearch from haystack.utils import Secret search_tool = ComponentTool(component=SerperDevWebSearch()) diff --git a/docs-website/docs/overview/migrating-from-langgraphlangchain-to-haystack.mdx b/docs-website/docs/overview/migrating-from-langgraphlangchain-to-haystack.mdx index 5b9839150b6..bfde61efe58 100644 --- a/docs-website/docs/overview/migrating-from-langgraphlangchain-to-haystack.mdx +++ b/docs-website/docs/overview/migrating-from-langgraphlangchain-to-haystack.mdx @@ -459,11 +459,11 @@ Both frameworks offer in-memory stores for prototyping and a wide range of produ
- {`# pip install haystack-ai sentence-transformers + {`# pip install haystack-ai sentence-transformers-haystack from haystack import Document from haystack.document_stores.in_memory import InMemoryDocumentStore -from haystack.components.embedders import SentenceTransformersDocumentEmbedder +from haystack_integrations.components.embedders.sentence_transformers import SentenceTransformersDocumentEmbedder # Embed and write documents to the document store document_store = InMemoryDocumentStore() @@ -505,7 +505,7 @@ vectorstore.add_documents([
{`from haystack import Pipeline -from haystack.components.embedders import SentenceTransformersTextEmbedder +from haystack_integrations.components.embedders.sentence_transformers import SentenceTransformersTextEmbedder from haystack.components.retrievers.in_memory import InMemoryEmbeddingRetriever from haystack.components.builders import ChatPromptBuilder from haystack.dataclasses import ChatMessage diff --git a/docs-website/docs/pipeline-components/agents-1/agent.mdx b/docs-website/docs/pipeline-components/agents-1/agent.mdx index fef0f17e649..8c6e3c1aa61 100644 --- a/docs-website/docs/pipeline-components/agents-1/agent.mdx +++ b/docs-website/docs/pipeline-components/agents-1/agent.mdx @@ -250,7 +250,7 @@ components: exclude_subdomains: false search_params: {} top_k: 3 - type: haystack.components.websearch.serper_dev.SerperDevWebSearch + type: haystack_integrations.components.websearch.serperdev.websearch.SerperDevWebSearch description: Search the web for current information on any topic inputs_from_state: null name: web_search diff --git a/docs-website/docs/pipeline-components/audio/funasrtranscriber.mdx b/docs-website/docs/pipeline-components/audio/funasrtranscriber.mdx index d537719a69a..194bdc184ec 100644 --- a/docs-website/docs/pipeline-components/audio/funasrtranscriber.mdx +++ b/docs-website/docs/pipeline-components/audio/funasrtranscriber.mdx @@ -27,7 +27,7 @@ Transcribe audio files to Haystack Documents using FunASR — a local, open-sour The default model is `iic/SenseVoiceSmall`, a multilingual model supporting 50+ languages that is 5–10x faster than Whisper. Models are downloaded from ModelScope on first use and cached in `~/.cache/modelscope`. -The component accepts audio file paths (`str` or `Path`) as well as `ByteStream` objects. Call `warm_up()` before running in a pipeline to load the model into memory. +The component accepts audio file paths (`str` or `Path`) as well as `ByteStream` objects. The model is loaded into memory automatically the first time the component runs. ## Usage @@ -37,7 +37,6 @@ The component accepts audio file paths (`str` or `Path`) as well as `ByteStream` from haystack_integrations.components.audio.funasr import FunASRTranscriber transcriber = FunASRTranscriber() -transcriber.warm_up() result = transcriber.run(sources=["speech.wav"]) print(result["documents"][0].content) @@ -61,7 +60,7 @@ result = pipe.run( "fetcher": { "urls": ["https://example.com/interview.wav"], }, - } + }, ) print(result["transcriber"]["documents"][0].content) ``` diff --git a/docs-website/docs/pipeline-components/classifiers/documentlanguageclassifier.mdx b/docs-website/docs/pipeline-components/classifiers/documentlanguageclassifier.mdx index 6cfffcdfe7d..86d0415276c 100644 --- a/docs-website/docs/pipeline-components/classifiers/documentlanguageclassifier.mdx +++ b/docs-website/docs/pipeline-components/classifiers/documentlanguageclassifier.mdx @@ -44,6 +44,12 @@ pip install langdetect-haystack Below, we are using the `DocumentLanguageClassifier` to classify English and German documents: +The examples on this page use Sentence Transformers embedders that have moved to the `sentence-transformers-haystack` package. Install it to run the examples: + +```shell +pip install sentence-transformers-haystack +``` + ```python from haystack_integrations.components.classifiers.langdetect import ( DocumentLanguageClassifier, @@ -74,7 +80,9 @@ from haystack.document_stores.in_memory import InMemoryDocumentStore from haystack_integrations.components.classifiers.langdetect import ( DocumentLanguageClassifier, ) -from haystack.components.embedders import SentenceTransformersDocumentEmbedder +from haystack_integrations.components.embedders.sentence_transformers import ( + SentenceTransformersDocumentEmbedder, +) from haystack.components.writers import DocumentWriter from haystack.components.routers import MetadataRouter diff --git a/docs-website/docs/pipeline-components/converters/amazontextractconverter.mdx b/docs-website/docs/pipeline-components/converters/amazontextractconverter.mdx index fe340ecb770..2a593e92403 100644 --- a/docs-website/docs/pipeline-components/converters/amazontextractconverter.mdx +++ b/docs-website/docs/pipeline-components/converters/amazontextractconverter.mdx @@ -72,7 +72,6 @@ from haystack_integrations.components.converters.amazon_textract import ( ) converter = AmazonTextractConverter(feature_types=["TABLES", "FORMS"]) -converter.warm_up() result = converter.run(sources=["invoice.pdf"]) documents = result["documents"] raw_responses = result["raw_textract_response"] diff --git a/docs-website/docs/pipeline-components/converters/imagefiletodocument.mdx b/docs-website/docs/pipeline-components/converters/imagefiletodocument.mdx index 07ce1d9305a..0227f69972a 100644 --- a/docs-website/docs/pipeline-components/converters/imagefiletodocument.mdx +++ b/docs-website/docs/pipeline-components/converters/imagefiletodocument.mdx @@ -64,10 +64,16 @@ print(documents) In the following Pipeline, image documents are created using the `ImageFileToDocument` component, then they are enriched with image embeddings and saved in the Document Store. +The examples on this page use Sentence Transformers embedders that have moved to the `sentence-transformers-haystack` package. Install it to run the examples: + +```shell +pip install sentence-transformers-haystack +``` + ```python from haystack import Pipeline from haystack.components.converters.image import ImageFileToDocument -from haystack.components.embedders.image import ( +from haystack_integrations.components.embedders.sentence_transformers import ( SentenceTransformersDocumentImageEmbedder, ) from haystack.components.writers.document_writer import DocumentWriter diff --git a/docs-website/docs/pipeline-components/generators/llamacppchatgenerator.mdx b/docs-website/docs/pipeline-components/generators/llamacppchatgenerator.mdx index cbd09ab9e76..2b3d2e6cf60 100644 --- a/docs-website/docs/pipeline-components/generators/llamacppchatgenerator.mdx +++ b/docs-website/docs/pipeline-components/generators/llamacppchatgenerator.mdx @@ -82,6 +82,12 @@ CMAKE_ARGS="-DGGML_CUDA=on" pip install llama-cpp-python pip install llama-cpp-haystack ``` +The examples on this page use Sentence Transformers embedders that have moved to the `sentence-transformers-haystack` package. Install it to run the examples: + +```shell +pip install sentence-transformers-haystack +``` + ## Usage 1. Download the GGUF version of the desired LLM. The GGUF versions of popular models can be downloaded from [Hugging Face](https://huggingface.co/models?library=gguf). @@ -97,7 +103,6 @@ generator = LlamaCppChatGenerator( model_kwargs={"n_gpu_layers": -1}, generation_kwargs={"max_tokens": 128, "temperature": 0.1}, ) -generator.warm_up() messages = [ChatMessage.from_user("Who is the best American actor?")] result = generator.run(messages) ``` @@ -209,7 +214,7 @@ from datasets import load_dataset from haystack import Document, Pipeline from haystack.components.builders.answer_builder import AnswerBuilder from haystack.components.builders import ChatPromptBuilder -from haystack.components.embedders import ( +from haystack_integrations.components.embedders.sentence_transformers import ( SentenceTransformersDocumentEmbedder, SentenceTransformersTextEmbedder, ) @@ -240,7 +245,7 @@ Index the documents to the `InMemoryDocumentStore` using the `SentenceTransforme ```python doc_store = InMemoryDocumentStore(embedding_similarity_function="cosine") -# Install sentence transformers using "pip install sentence-transformers" +# Install the Sentence Transformers embedders using "pip install sentence-transformers-haystack" doc_embedder = SentenceTransformersDocumentEmbedder( model="sentence-transformers/all-MiniLM-L6-v2", ) diff --git a/docs-website/docs/pipeline-components/generators/llamacppgenerator.mdx b/docs-website/docs/pipeline-components/generators/llamacppgenerator.mdx index 2cfdf2ceec8..f09113d5f26 100644 --- a/docs-website/docs/pipeline-components/generators/llamacppgenerator.mdx +++ b/docs-website/docs/pipeline-components/generators/llamacppgenerator.mdx @@ -52,6 +52,12 @@ CMAKE_ARGS="-DGGML_CUDA=on" pip install llama-cpp-python pip install llama-cpp-haystack ``` +The examples on this page use Sentence Transformers embedders that have moved to the `sentence-transformers-haystack` package. Install it to run the examples: + +```shell +pip install sentence-transformers-haystack +``` + ## Usage 1. You need to download the GGUF version of the desired LLM. The GGUF versions of popular models can be downloaded from [Hugging Face](https://huggingface.co/models?library=gguf). @@ -67,7 +73,6 @@ generator = LlamaCppGenerator( model_kwargs={"n_gpu_layers": -1}, generation_kwargs={"max_tokens": 128, "temperature": 0.1}, ) -generator.warm_up() prompt = f"Who is the best American actor?" result = generator.run(prompt) ``` @@ -147,7 +152,7 @@ from datasets import load_dataset from haystack import Document, Pipeline from haystack.components.builders.answer_builder import AnswerBuilder from haystack.components.builders.prompt_builder import PromptBuilder -from haystack.components.embedders import ( +from haystack_integrations.components.embedders.sentence_transformers import ( SentenceTransformersDocumentEmbedder, SentenceTransformersTextEmbedder, ) diff --git a/docs-website/docs/pipeline-components/joiners/branchjoiner.mdx b/docs-website/docs/pipeline-components/joiners/branchjoiner.mdx index 0c392752296..4f49313e6d9 100644 --- a/docs-website/docs/pipeline-components/joiners/branchjoiner.mdx +++ b/docs-website/docs/pipeline-components/joiners/branchjoiner.mdx @@ -130,6 +130,12 @@ print(json.loads(result["validator"]["validated"][0].text)) In this example, the `TextLanguageRouter` component directs the query to one of three language-specific Retrievers. The next component would be a `PromptBuilder`, but we cannot connect multiple Retrievers to a single `PromptBuilder` directly. Instead, we connect all the Retrievers to the `BranchJoiner` component. The `BranchJoiner` then takes the output from the Retriever that was actually called and passes it as a single list of documents to the `PromptBuilder`. The `BranchJoiner` ensures that the pipeline can handle multiple languages seamlessly by consolidating different outputs from the Retrievers into a unified connection for further processing. +The examples on this page use language classification components that have moved to the `langdetect-haystack` package. Install it to run the examples: + +```shell +pip install langdetect-haystack +``` + ```python from haystack import Document, Pipeline from haystack.document_stores.in_memory import InMemoryDocumentStore @@ -137,7 +143,7 @@ from haystack.components.retrievers.in_memory import InMemoryBM25Retriever from haystack.components.joiners import BranchJoiner from haystack.components.builders import ChatPromptBuilder from haystack.components.generators.chat import OpenAIChatGenerator -from haystack.components.routers import TextLanguageRouter +from haystack_integrations.components.routers.langdetect import TextLanguageRouter from haystack.dataclasses import ChatMessage prompt_template = [ diff --git a/docs-website/docs/pipeline-components/joiners/documentjoiner.mdx b/docs-website/docs/pipeline-components/joiners/documentjoiner.mdx index cc5998e317d..c2fac5a0121 100644 --- a/docs-website/docs/pipeline-components/joiners/documentjoiner.mdx +++ b/docs-website/docs/pipeline-components/joiners/documentjoiner.mdx @@ -63,6 +63,12 @@ joiner.run(documents=[docs_1, docs_2]) Below is an example of a hybrid retrieval pipeline that retrieves documents from an `InMemoryDocumentStore` based on keyword search (using `InMemoryBM25Retriever`) and embedding search (using `InMemoryEmbeddingRetriever`). It then uses the `DocumentJoiner` with its default join mode to concatenate the retrieved documents into one list. The Document Store must contain documents with embeddings, otherwise the `InMemoryEmbeddingRetriever` will not return any documents. +The examples on this page use Sentence Transformers embedders that have moved to the `sentence-transformers-haystack` package. Install it to run the examples: + +```shell +pip install sentence-transformers-haystack +``` + ```python from haystack.components.joiners.document_joiner import DocumentJoiner from haystack import Pipeline @@ -71,7 +77,9 @@ from haystack.components.retrievers.in_memory import ( InMemoryBM25Retriever, InMemoryEmbeddingRetriever, ) -from haystack.components.embedders import SentenceTransformersTextEmbedder +from haystack_integrations.components.embedders.sentence_transformers import ( + SentenceTransformersTextEmbedder, +) document_store = InMemoryDocumentStore() p = Pipeline() @@ -111,7 +119,9 @@ from haystack.components.converters import ( from haystack.components.preprocessors import DocumentSplitter, DocumentCleaner from haystack.components.routers import FileTypeRouter from haystack.components.joiners import DocumentJoiner -from haystack.components.embedders import SentenceTransformersDocumentEmbedder +from haystack_integrations.components.embedders.sentence_transformers import ( + SentenceTransformersDocumentEmbedder, +) from haystack import Pipeline from haystack.document_stores.in_memory import InMemoryDocumentStore from pathlib import Path diff --git a/docs-website/docs/pipeline-components/preprocessors/chinesedocumentsplitter.mdx b/docs-website/docs/pipeline-components/preprocessors/chinesedocumentsplitter.mdx index ce3bd9dd1c4..c926468ee2a 100644 --- a/docs-website/docs/pipeline-components/preprocessors/chinesedocumentsplitter.mdx +++ b/docs-website/docs/pipeline-components/preprocessors/chinesedocumentsplitter.mdx @@ -144,7 +144,6 @@ def custom_split(text: str) -> list[str]: doc = Document(content="第一段,第二段,第三段,第四段") splitter = ChineseDocumentSplitter(split_by="function", splitting_function=custom_split) -splitter.warm_up() result = splitter.run(documents=[doc]) print(result["documents"]) ``` diff --git a/docs-website/docs/pipeline-components/preprocessors/embeddingbaseddocumentsplitter.mdx b/docs-website/docs/pipeline-components/preprocessors/embeddingbaseddocumentsplitter.mdx index e80ce7c52e5..942436fe3f8 100644 --- a/docs-website/docs/pipeline-components/preprocessors/embeddingbaseddocumentsplitter.mdx +++ b/docs-website/docs/pipeline-components/preprocessors/embeddingbaseddocumentsplitter.mdx @@ -36,10 +36,18 @@ This component is inspired by [5 Levels of Text Splitting](https://github.com/Fu ### On its own +The examples on this page use Sentence Transformers embedders that have moved to the `sentence-transformers-haystack` package. Install it to run the examples: + +```shell +pip install sentence-transformers-haystack +``` + ```python from haystack import Document -from haystack.components.embedders import SentenceTransformersDocumentEmbedder +from haystack_integrations.components.embedders.sentence_transformers import ( + SentenceTransformersDocumentEmbedder, +) from haystack.components.preprocessors import EmbeddingBasedDocumentSplitter # Create a document with content that has a clear topic shift diff --git a/docs-website/docs/pipeline-components/preprocessors/textcleaner.mdx b/docs-website/docs/pipeline-components/preprocessors/textcleaner.mdx index 63b13814428..463085718df 100644 --- a/docs-website/docs/pipeline-components/preprocessors/textcleaner.mdx +++ b/docs-website/docs/pipeline-components/preprocessors/textcleaner.mdx @@ -55,14 +55,22 @@ result = cleaner.run(texts=[text_to_clean]) ### In a pipeline -In this example, we are using `TextCleaner` after an `ExtractiveReader` and an `OutputAdapter` to remove the punctuation in texts. Then, our custom-made `ExactMatchEvaluator` component compares the retrieved answer to the ground truth answer. +In this example, we are using `TextCleaner` after a `TransformersExtractiveReader` and an `OutputAdapter` to remove the punctuation in texts. Then, our custom-made `ExactMatchEvaluator` component compares the retrieved answer to the ground truth answer. + +The examples on this page use Transformers components that have moved to the `transformers-haystack` package. Install it to run the examples: + +```shell +pip install transformers-haystack +``` ```python from typing import List from haystack import component, Document, Pipeline from haystack.components.converters import OutputAdapter from haystack.components.preprocessors import TextCleaner -from haystack.components.readers import ExtractiveReader +from haystack_integrations.components.readers.transformers import ( + TransformersExtractiveReader, +) from haystack.components.retrievers.in_memory import InMemoryBM25Retriever from haystack.document_stores.in_memory import InMemoryDocumentStore @@ -96,7 +104,7 @@ adapter = OutputAdapter( p = Pipeline() p.add_component("retriever", InMemoryBM25Retriever(document_store=document_store)) -p.add_component("reader", ExtractiveReader()) +p.add_component("reader", TransformersExtractiveReader()) p.add_component("adapter", adapter) p.add_component("cleaner", TextCleaner(remove_punctuation=True)) p.add_component("evaluator", ExactMatchEvaluator()) diff --git a/docs-website/docs/pipeline-components/rankers/fastembedlateinteractionranker.mdx b/docs-website/docs/pipeline-components/rankers/fastembedlateinteractionranker.mdx index 8cc8a9000cf..b72aff16dce 100644 --- a/docs-website/docs/pipeline-components/rankers/fastembedlateinteractionranker.mdx +++ b/docs-website/docs/pipeline-components/rankers/fastembedlateinteractionranker.mdx @@ -95,19 +95,27 @@ print(result["documents"][0].content) Below is an example of a full RAG pipeline that retrieves documents using embedding similarity, reranks them with `FastembedLateInteractionRanker`, and generates an answer with an LLM. -This example uses the `HuggingFaceLocalChatGenerator`, which requires additional packages: +This example uses the `TransformersChatGenerator`, which requires additional packages: ```shell pip install "transformers[torch]" ``` +The examples on this page use Transformers components that have moved to the `transformers-haystack` package. Install it to run the examples: + +```shell +pip install transformers-haystack +``` + ```python from haystack import Document, Pipeline from haystack.document_stores.in_memory import InMemoryDocumentStore from haystack.components.retrievers.in_memory import InMemoryEmbeddingRetriever from haystack.components.builders.chat_prompt_builder import ChatPromptBuilder -from haystack.components.generators.chat import HuggingFaceLocalChatGenerator +from haystack_integrations.components.generators.transformers import ( + TransformersChatGenerator, +) from haystack.components.writers import DocumentWriter from haystack.dataclasses import ChatMessage from haystack_integrations.components.rankers.fastembed import ( @@ -162,7 +170,7 @@ rag.add_component( ) rag.add_component( "llm", - HuggingFaceLocalChatGenerator(model="HuggingFaceTB/SmolLM2-360M-Instruct"), + TransformersChatGenerator(model="HuggingFaceTB/SmolLM2-360M-Instruct"), ) rag.connect("text_embedder.embedding", "retriever.query_embedding") diff --git a/docs-website/docs/pipeline-components/rankers/pyversityranker.mdx b/docs-website/docs/pipeline-components/rankers/pyversityranker.mdx index 4d084dc819e..af5c37229d1 100644 --- a/docs-website/docs/pipeline-components/rankers/pyversityranker.mdx +++ b/docs-website/docs/pipeline-components/rankers/pyversityranker.mdx @@ -96,9 +96,15 @@ Below is an example of a pipeline that embeds documents and stores them in an `I Note that the retriever must be configured with `return_embedding=True` so that documents have embeddings available for the ranker. +The examples on this page use Sentence Transformers embedders that have moved to the `sentence-transformers-haystack` package. Install it to run the examples: + +```shell +pip install sentence-transformers-haystack +``` + ```python from haystack import Document, Pipeline -from haystack.components.embedders import ( +from haystack_integrations.components.embedders.sentence_transformers import ( SentenceTransformersDocumentEmbedder, SentenceTransformersTextEmbedder, ) diff --git a/docs-website/docs/pipeline-components/retrievers/alloydbembeddingretriever.mdx b/docs-website/docs/pipeline-components/retrievers/alloydbembeddingretriever.mdx index bf598f75565..bd97f726bab 100644 --- a/docs-website/docs/pipeline-components/retrievers/alloydbembeddingretriever.mdx +++ b/docs-website/docs/pipeline-components/retrievers/alloydbembeddingretriever.mdx @@ -13,7 +13,7 @@ An embedding-based Retriever compatible with the AlloyDB Document Store. | | | | --- | --- | -| **Most common position in a pipeline** | 1. After a Text Embedder and before a [`PromptBuilder`](../builders/promptbuilder.mdx) in a RAG pipeline 2. The last component in the semantic search pipeline 3. After a Text Embedder and before an [`ExtractiveReader`](../readers/extractivereader.mdx) in an extractive QA pipeline | +| **Most common position in a pipeline** | 1. After a Text Embedder and before a [`PromptBuilder`](../builders/promptbuilder.mdx) in a RAG pipeline 2. The last component in the semantic search pipeline 3. After a Text Embedder and before a [`TransformersExtractiveReader`](../readers/transformersextractivereader.mdx) in an extractive QA pipeline | | **Mandatory init variables** | `document_store`: An instance of an [AlloyDBDocumentStore](../../document-stores/alloydbdocumentstore.mdx) | | **Mandatory run variables** | `query_embedding`: A vector representing the query (a list of floats) | | **Output variables** | `documents`: A list of documents | @@ -43,6 +43,12 @@ pip install alloydb-haystack To set up an AlloyDB cluster and instance, follow the [AlloyDB quickstart](https://cloud.google.com/alloydb/docs/quickstart). +The examples on this page use Sentence Transformers embedders that have moved to the `sentence-transformers-haystack` package. Install it to run the examples: + +```shell +pip install sentence-transformers-haystack +``` + ## Usage ### On its own @@ -69,7 +75,7 @@ retriever.run(query_embedding=[0.1] * 768) ```python from haystack import Document, Pipeline from haystack.document_stores.types import DuplicatePolicy -from haystack.components.embedders import ( +from haystack_integrations.components.embedders.sentence_transformers import ( SentenceTransformersTextEmbedder, SentenceTransformersDocumentEmbedder, ) diff --git a/docs-website/docs/pipeline-components/retrievers/arangoembeddingretriever.mdx b/docs-website/docs/pipeline-components/retrievers/arangoembeddingretriever.mdx index 78022c10ec7..ce4b645d600 100644 --- a/docs-website/docs/pipeline-components/retrievers/arangoembeddingretriever.mdx +++ b/docs-website/docs/pipeline-components/retrievers/arangoembeddingretriever.mdx @@ -45,6 +45,12 @@ docker run -d -p 8529:8529 \ arangodb:3.12 arangod --vector-index ``` +The examples on this page use Sentence Transformers embedders that have moved to the `sentence-transformers-haystack` package. Install it to run the examples: + +```shell +pip install sentence-transformers-haystack +``` + ## Usage ### On its own @@ -84,7 +90,7 @@ print(result["documents"][0].content) ```python from haystack import Document, Pipeline from haystack.document_stores.types import DuplicatePolicy -from haystack.components.embedders import ( +from haystack_integrations.components.embedders.sentence_transformers import ( SentenceTransformersDocumentEmbedder, SentenceTransformersTextEmbedder, ) diff --git a/docs-website/docs/pipeline-components/retrievers/arcadedbembeddingretriever.mdx b/docs-website/docs/pipeline-components/retrievers/arcadedbembeddingretriever.mdx index 697c9459329..99fdb0262b3 100644 --- a/docs-website/docs/pipeline-components/retrievers/arcadedbembeddingretriever.mdx +++ b/docs-website/docs/pipeline-components/retrievers/arcadedbembeddingretriever.mdx @@ -35,6 +35,12 @@ pip install arcadedb-haystack Ensure ArcadeDB is running, for example via Docker, and credentials are set (`ARCADEDB_USERNAME`, `ARCADEDB_PASSWORD`). +The examples on this page use Sentence Transformers embedders that have moved to the `sentence-transformers-haystack` package. Install it to run the examples: + +```shell +pip install sentence-transformers-haystack +``` + ## Usage ### On its own @@ -63,7 +69,7 @@ for doc in result["documents"]: ```python from haystack import Document, Pipeline from haystack.document_stores.types import DuplicatePolicy -from haystack.components.embedders import ( +from haystack_integrations.components.embedders.sentence_transformers import ( SentenceTransformersTextEmbedder, SentenceTransformersDocumentEmbedder, ) diff --git a/docs-website/docs/pipeline-components/retrievers/astraretriever.mdx b/docs-website/docs/pipeline-components/retrievers/astraretriever.mdx index 9f3773d899d..c79f9723a5c 100644 --- a/docs-website/docs/pipeline-components/retrievers/astraretriever.mdx +++ b/docs-website/docs/pipeline-components/retrievers/astraretriever.mdx @@ -13,7 +13,7 @@ This is an embedding-based Retriever compatible with the Astra Document Store. | | | | --- | --- | -| **Most common position in a pipeline** | 1. After a Text Embedder and before a [`PromptBuilder`](../builders/promptbuilder.mdx) in a RAG pipeline
2. The last component in the semantic search pipeline
3. After a Text Embedder and before an [`ExtractiveReader`](../readers/extractivereader.mdx) in an extractive QA pipeline | +| **Most common position in a pipeline** | 1. After a Text Embedder and before a [`PromptBuilder`](../builders/promptbuilder.mdx) in a RAG pipeline
2. The last component in the semantic search pipeline
3. After a Text Embedder and before a [`TransformersExtractiveReader`](../readers/transformersextractivereader.mdx) in an extractive QA pipeline | | **Mandatory init variables** | `document_store`: An instance of [AstraDocumentStore](../../document-stores/astradocumentstore.mdx) | | **Mandatory run variables** | `query_embedding`: A list of floats | | **Output variables** | `documents`: A list of documents | @@ -43,10 +43,10 @@ From the configuration in AstraDB’s web UI, you need the database ID and a gen You will additionally need a collection name and a namespace. When you create the collection name, you also need to set the embedding dimensions and the similarity metric. The namespace organizes data in a database and is called a keyspace in Apache Cassandra. -Then, optionally, install sentence-transformers as well to run the example below: +Then, optionally, install the `sentence-transformers-haystack` package as well to run the example below: ```shell -pip install sentence-transformers +pip install sentence-transformers-haystack ``` ## Usage @@ -59,7 +59,7 @@ Use this Retriever in a query pipeline like this: ```python from haystack import Document, Pipeline -from haystack.components.embedders import ( +from haystack_integrations.components.embedders.sentence_transformers import ( SentenceTransformersTextEmbedder, SentenceTransformersDocumentEmbedder, ) diff --git a/docs-website/docs/pipeline-components/retrievers/azureaisearchembeddingretriever.mdx b/docs-website/docs/pipeline-components/retrievers/azureaisearchembeddingretriever.mdx index 7fb26bedbe8..22cb29506cb 100644 --- a/docs-website/docs/pipeline-components/retrievers/azureaisearchembeddingretriever.mdx +++ b/docs-website/docs/pipeline-components/retrievers/azureaisearchembeddingretriever.mdx @@ -15,7 +15,7 @@ This Retriever accepts the embeddings of a single query as input and returns a l | | | | --- | --- | -| **Most common position in a pipeline** | 1. After a Text Embedder and before a [`PromptBuilder`](../builders/promptbuilder.mdx) in a RAG pipeline 2. The last component in the embedding retrieval pipeline 3. After a Text Embedder and before an [`ExtractiveReader`](../readers/extractivereader.mdx) in an extractive QA pipeline | +| **Most common position in a pipeline** | 1. After a Text Embedder and before a [`PromptBuilder`](../builders/promptbuilder.mdx) in a RAG pipeline 2. The last component in the embedding retrieval pipeline 3. After a Text Embedder and before an [`TransformersExtractiveReader`](../readers/transformersextractivereader.mdx) in an extractive QA pipeline | | **Mandatory init variables** | `document_store`: An instance of [`AzureAISearchDocumentStore`](../../document-stores/azureaisearchdocumentstore.mdx) | | **Mandatory run variables** | `query_embedding`: A list of floats | | **Output variables** | `documents`: A list of documents | @@ -80,9 +80,15 @@ In the indexing pipeline, the documents are passed to the Document Embedder and Then, in the querying pipeline, we use a Text Embedder to get the vector representation of the input query that will be then passed to the `AzureAISearchEmbeddingRetriever` to get the results. +The examples on this page use Sentence Transformers embedders that have moved to the `sentence-transformers-haystack` package. Install it to run the examples: + +```shell +pip install sentence-transformers-haystack +``` + ```python from haystack import Document, Pipeline -from haystack.components.embedders import ( +from haystack_integrations.components.embedders.sentence_transformers import ( SentenceTransformersDocumentEmbedder, SentenceTransformersTextEmbedder, ) diff --git a/docs-website/docs/pipeline-components/retrievers/azureaisearchhybridretriever.mdx b/docs-website/docs/pipeline-components/retrievers/azureaisearchhybridretriever.mdx index fe532a10cc3..68bc03fe740 100644 --- a/docs-website/docs/pipeline-components/retrievers/azureaisearchhybridretriever.mdx +++ b/docs-website/docs/pipeline-components/retrievers/azureaisearchhybridretriever.mdx @@ -15,7 +15,7 @@ This Retriever combines embedding-based retrieval and BM25 text search search to | | | | --- | --- | -| **Most common position in a pipeline** | 1. After a TextEmbedder and before a [`PromptBuilder`](../builders/promptbuilder.mdx) in a RAG pipeline 2. The last component in a hybrid search pipeline 3. After a TextEmbedder and before an [`ExtractiveReader`](../readers/extractivereader.mdx) in an extractive QA pipeline | +| **Most common position in a pipeline** | 1. After a TextEmbedder and before a [`PromptBuilder`](../builders/promptbuilder.mdx) in a RAG pipeline 2. The last component in a hybrid search pipeline 3. After a TextEmbedder and before an [`TransformersExtractiveReader`](../readers/transformersextractivereader.mdx) in an extractive QA pipeline | | **Mandatory init variables** | `document_store`: An instance of [`AzureAISearchDocumentStore`](../../document-stores/azureaisearchdocumentstore.mdx) | | **Mandatory run variables** | `query`: A string

`query_embedding`: A list of floats | | **Output variables** | `documents`: A list of documents (matching the query) | @@ -84,9 +84,15 @@ retriever.run( The following example demonstrates using the `AzureAISearchHybridRetriever` in a pipeline. An indexing pipeline is responsible for indexing and storing documents with embeddings in the `AzureAISearchDocumentStore`, while the query pipeline uses hybrid retrieval to fetch relevant documents based on a given query. +The examples on this page use Sentence Transformers embedders that have moved to the `sentence-transformers-haystack` package. Install it to run the examples: + +```shell +pip install sentence-transformers-haystack +``` + ```python from haystack import Document, Pipeline -from haystack.components.embedders import ( +from haystack_integrations.components.embedders.sentence_transformers import ( SentenceTransformersDocumentEmbedder, SentenceTransformersTextEmbedder, ) diff --git a/docs-website/docs/pipeline-components/retrievers/chromaembeddingretriever.mdx b/docs-website/docs/pipeline-components/retrievers/chromaembeddingretriever.mdx index d108a3985bd..01dc63e5fe5 100644 --- a/docs-website/docs/pipeline-components/retrievers/chromaembeddingretriever.mdx +++ b/docs-website/docs/pipeline-components/retrievers/chromaembeddingretriever.mdx @@ -13,7 +13,7 @@ This is an embedding Retriever compatible with the Chroma Document Store. | | | | --- | --- | -| **Most common position in a pipeline** | 1. After a Text Embedder and before a [`PromptBuilder`](../builders/promptbuilder.mdx) in a RAG pipeline 2. The last component in the semantic search pipeline 3. After a Text Embedder and before an [`ExtractiveReader`](../readers/extractivereader.mdx) in an extractive QA pipeline | +| **Most common position in a pipeline** | 1. After a Text Embedder and before a [`PromptBuilder`](../builders/promptbuilder.mdx) in a RAG pipeline 2. The last component in the semantic search pipeline 3. After a Text Embedder and before a [`TransformersExtractiveReader`](../readers/transformersextractivereader.mdx) in an extractive QA pipeline | | **Mandatory init variables** | `document_store`: An instance of a [ChromaDocumentStore](../../document-stores/chromadocumentstore.mdx) | | **Mandatory run variables** | `query_embedding`: A list of floats | | **Output variables** | `documents`: A list of documents | @@ -65,8 +65,8 @@ from haystack import Pipeline from haystack.dataclasses import Document from haystack.components.writers import DocumentWriter -# Note: the following requires a "pip install sentence-transformers" -from haystack.components.embedders import ( +# Note: the following requires a "pip install sentence-transformers-haystack" +from haystack_integrations.components.embedders.sentence_transformers import ( SentenceTransformersDocumentEmbedder, SentenceTransformersTextEmbedder, ) diff --git a/docs-website/docs/pipeline-components/retrievers/elasticsearchembeddingretriever.mdx b/docs-website/docs/pipeline-components/retrievers/elasticsearchembeddingretriever.mdx index b059e951455..307a62ba24e 100644 --- a/docs-website/docs/pipeline-components/retrievers/elasticsearchembeddingretriever.mdx +++ b/docs-website/docs/pipeline-components/retrievers/elasticsearchembeddingretriever.mdx @@ -13,7 +13,7 @@ An embedding-based Retriever compatible with the Elasticsearch Document Store. | | | | --- | --- | -| **Most common position in a pipeline** | 1. After a Text Embedder and before a [`PromptBuilder`](../builders/promptbuilder.mdx) in a RAG pipeline 2. The last component in the semantic search pipeline 3. After a Text Embedder and before an [`ExtractiveReader`](../readers/extractivereader.mdx) in an extractive QA pipeline | +| **Most common position in a pipeline** | 1. After a Text Embedder and before a [`PromptBuilder`](../builders/promptbuilder.mdx) in a RAG pipeline 2. The last component in the semantic search pipeline 3. After a Text Embedder and before a [`TransformersExtractiveReader`](../readers/transformersextractivereader.mdx) in an extractive QA pipeline | | **Mandatory init variables** | `document_store`: An instance of [ElasticsearchDocumentStore](../../document-stores/elasticsearch-document-store.mdx) | | **Mandatory run variables** | `query_embedding`: A list of floats | | **Output variables** | `documents`: A list of documents | @@ -58,6 +58,12 @@ Once you have a running Elasticsearch instance, install the `elasticsearch-hayst pip install elasticsearch-haystack ``` +The examples on this page use Sentence Transformers embedders that have moved to the `sentence-transformers-haystack` package. Install it to run the examples: + +```shell +pip install sentence-transformers-haystack +``` + ## Usage ### In a pipeline @@ -74,7 +80,7 @@ from haystack_integrations.document_stores.elasticsearch import ( from haystack.document_stores.types import DuplicatePolicy from haystack import Document, Pipeline -from haystack.components.embedders import ( +from haystack_integrations.components.embedders.sentence_transformers import ( SentenceTransformersTextEmbedder, SentenceTransformersDocumentEmbedder, ) diff --git a/docs-website/docs/pipeline-components/retrievers/faissembeddingretriever.mdx b/docs-website/docs/pipeline-components/retrievers/faissembeddingretriever.mdx index ddce627ee57..51328cd7cfc 100644 --- a/docs-website/docs/pipeline-components/retrievers/faissembeddingretriever.mdx +++ b/docs-website/docs/pipeline-components/retrievers/faissembeddingretriever.mdx @@ -13,7 +13,7 @@ An embedding-based Retriever compatible with the FAISSDocumentStore. | | | | --- | --- | -| **Most common position in a pipeline** | 1. After a Text Embedder and before a [`PromptBuilder`](../builders/promptbuilder.mdx) in a RAG pipeline 2. The last component in a semantic search pipeline 3. After a Text Embedder and before an [`ExtractiveReader`](../readers/extractivereader.mdx) in an extractive QA pipeline | +| **Most common position in a pipeline** | 1. After a Text Embedder and before a [`PromptBuilder`](../builders/promptbuilder.mdx) in a RAG pipeline 2. The last component in a semantic search pipeline 3. After a Text Embedder and before a [`TransformersExtractiveReader`](../readers/transformersextractivereader.mdx) in an extractive QA pipeline | | **Mandatory init variables** | `document_store`: An instance of a [`FAISSDocumentStore`](../../document-stores/faissdocumentstore.mdx) | | **Mandatory run variables** | `query_embedding`: A vector representing the query (a list of floats) | | **Output variables** | `documents`: A list of documents | @@ -54,9 +54,15 @@ print(result["documents"]) ### In a pipeline +The examples on this page use Sentence Transformers embedders that have moved to the `sentence-transformers-haystack` package. Install it to run the examples: + +```shell +pip install sentence-transformers-haystack +``` + ```python from haystack import Document, Pipeline -from haystack.components.embedders import ( +from haystack_integrations.components.embedders.sentence_transformers import ( SentenceTransformersDocumentEmbedder, SentenceTransformersTextEmbedder, ) diff --git a/docs-website/docs/pipeline-components/retrievers/falkordbcypherretriever.mdx b/docs-website/docs/pipeline-components/retrievers/falkordbcypherretriever.mdx index 336821f0408..fa70c1cca2a 100644 --- a/docs-website/docs/pipeline-components/retrievers/falkordbcypherretriever.mdx +++ b/docs-website/docs/pipeline-components/retrievers/falkordbcypherretriever.mdx @@ -45,6 +45,12 @@ Ensure FalkorDB is running, for example via Docker: docker run -d -p 6379:6379 falkordb/falkordb:latest ``` +The examples on this page use Transformers components that have moved to the `transformers-haystack` package. Install it to run the examples: + +```shell +pip install transformers-haystack +``` + ## Usage ### On its own @@ -85,7 +91,9 @@ print(result["documents"][0].content) ```python from haystack import Document, Pipeline from haystack.components.builders import ChatPromptBuilder -from haystack.components.generators.chat import HuggingFaceLocalChatGenerator +from haystack_integrations.components.generators.transformers import ( + TransformersChatGenerator, +) from haystack.dataclasses import ChatMessage from haystack_integrations.document_stores.falkordb import FalkorDBDocumentStore from haystack_integrations.components.retrievers.falkordb import FalkorDBCypherRetriever @@ -130,7 +138,7 @@ pipeline.add_component( pipeline.add_component("prompt_builder", ChatPromptBuilder(template=prompt_template)) pipeline.add_component( "llm", - HuggingFaceLocalChatGenerator(model="HuggingFaceTB/SmolLM2-135M-Instruct"), + TransformersChatGenerator(model="HuggingFaceTB/SmolLM2-135M-Instruct"), ) pipeline.connect("retriever.documents", "prompt_builder.documents") pipeline.connect("prompt_builder.prompt", "llm.messages") diff --git a/docs-website/docs/pipeline-components/retrievers/falkordbembeddingretriever.mdx b/docs-website/docs/pipeline-components/retrievers/falkordbembeddingretriever.mdx index f44e59092de..90602992ca1 100644 --- a/docs-website/docs/pipeline-components/retrievers/falkordbembeddingretriever.mdx +++ b/docs-website/docs/pipeline-components/retrievers/falkordbembeddingretriever.mdx @@ -43,6 +43,12 @@ Ensure FalkorDB is running, for example via Docker: docker run -d -p 6379:6379 falkordb/falkordb:latest ``` +The examples on this page use Sentence Transformers embedders that have moved to the `sentence-transformers-haystack` package. Install it to run the examples: + +```shell +pip install sentence-transformers-haystack +``` + ## Usage ### On its own @@ -83,7 +89,7 @@ print(result["documents"][0].content) ```python from haystack import Document, Pipeline from haystack.document_stores.types import DuplicatePolicy -from haystack.components.embedders import ( +from haystack_integrations.components.embedders.sentence_transformers import ( SentenceTransformersDocumentEmbedder, SentenceTransformersTextEmbedder, ) @@ -112,7 +118,6 @@ documents = [ document_embedder = SentenceTransformersDocumentEmbedder( model="sentence-transformers/all-MiniLM-L6-v2", ) -document_embedder.warm_up() documents_with_embeddings = document_embedder.run(documents) document_store.write_documents( diff --git a/docs-website/docs/pipeline-components/retrievers/inmemoryembeddingretriever.mdx b/docs-website/docs/pipeline-components/retrievers/inmemoryembeddingretriever.mdx index 81090e876fb..e7ddab2f245 100644 --- a/docs-website/docs/pipeline-components/retrievers/inmemoryembeddingretriever.mdx +++ b/docs-website/docs/pipeline-components/retrievers/inmemoryembeddingretriever.mdx @@ -13,7 +13,7 @@ Use this Retriever with the InMemoryDocumentStore if you're looking for embeddin | | | | --- | --- | -| **Most common position in a pipeline** | In query pipelines:
In a RAG pipeline, before a [`PromptBuilder`](../builders/promptbuilder.mdx)
In a semantic search pipeline, as the last component
In an extractive QA pipeline, after a Tex tEmbedder and before an [`ExtractiveReader`](../readers/extractivereader.mdx) | +| **Most common position in a pipeline** | In query pipelines:
In a RAG pipeline, before a [`PromptBuilder`](../builders/promptbuilder.mdx)
In a semantic search pipeline, as the last component
In an extractive QA pipeline, after a Tex tEmbedder and before a [`TransformersExtractiveReader`](../readers/transformersextractivereader.mdx) | | **Mandatory init variables** | `document_store`: An instance of [InMemoryDocumentStore](../../document-stores/inmemorydocumentstore.mdx) | | **Mandatory run variables** | `query_embedding`: A list of floating point numbers | | **Output variables** | `documents`: A list of documents | @@ -39,10 +39,16 @@ The `embedding_similarity_function` to use for embedding retrieval must be defi Use this Retriever in a query pipeline like this: +The examples on this page use Sentence Transformers embedders that have moved to the `sentence-transformers-haystack` package. Install it to run the examples: + +```shell +pip install sentence-transformers-haystack +``` + ```python from haystack import Document, Pipeline from haystack.document_stores.in_memory import InMemoryDocumentStore -from haystack.components.embedders import ( +from haystack_integrations.components.embedders.sentence_transformers import ( SentenceTransformersTextEmbedder, SentenceTransformersDocumentEmbedder, ) diff --git a/docs-website/docs/pipeline-components/retrievers/mongodbatlasembeddingretriever.mdx b/docs-website/docs/pipeline-components/retrievers/mongodbatlasembeddingretriever.mdx index df1d707d890..df1db8c4b43 100644 --- a/docs-website/docs/pipeline-components/retrievers/mongodbatlasembeddingretriever.mdx +++ b/docs-website/docs/pipeline-components/retrievers/mongodbatlasembeddingretriever.mdx @@ -13,7 +13,7 @@ This is an embedding Retriever compatible with the MongoDB Atlas Document Store. | | | | --- | --- | -| **Most common position in a pipeline** | 1. After a Text Embedder and before a [`PromptBuilder`](../builders/promptbuilder.mdx) in a RAG pipeline 2. The last component in the semantic search pipeline 3. After a Text Embedder and before an [`ExtractiveReader`](../readers/extractivereader.mdx) in an extractive QA pipeline | +| **Most common position in a pipeline** | 1. After a Text Embedder and before a [`PromptBuilder`](../builders/promptbuilder.mdx) in a RAG pipeline 2. The last component in the semantic search pipeline 3. After a Text Embedder and before a [`TransformersExtractiveReader`](../readers/transformersextractivereader.mdx) in an extractive QA pipeline | | **Mandatory init variables** | `document_store`: An instance of a [MongoDBAtlasDocumentStore](../../document-stores/mongodbatlasdocumentstore.mdx) | | **Mandatory run variables** | `query_embedding`: A list of floats | | **Output variables** | `documents`: A list of documents | @@ -63,6 +63,12 @@ retriever.run(query_embedding=[0.1] * 384) ### In a Pipeline +The examples on this page use Sentence Transformers embedders that have moved to the `sentence-transformers-haystack` package. Install it to run the examples: + +```shell +pip install sentence-transformers-haystack +``` + ```python from haystack import Pipeline, Document from haystack.document_stores.types import DuplicatePolicy @@ -70,7 +76,7 @@ from haystack.components.writers import DocumentWriter from haystack.components.generators.chat import OpenAIChatGenerator from haystack.components.builders import ChatPromptBuilder from haystack.dataclasses import ChatMessage -from haystack.components.embedders import ( +from haystack_integrations.components.embedders.sentence_transformers import ( SentenceTransformersDocumentEmbedder, SentenceTransformersTextEmbedder, ) diff --git a/docs-website/docs/pipeline-components/retrievers/mongodbatlasfulltextretriever.mdx b/docs-website/docs/pipeline-components/retrievers/mongodbatlasfulltextretriever.mdx index 8c9f7468d84..9616cf01c1a 100644 --- a/docs-website/docs/pipeline-components/retrievers/mongodbatlasfulltextretriever.mdx +++ b/docs-website/docs/pipeline-components/retrievers/mongodbatlasfulltextretriever.mdx @@ -13,7 +13,7 @@ This is a full-text search Retriever compatible with the MongoDB Atlas Document | | | | --- | --- | -| **Most common position in a pipeline** | 1. Before a [ChatPromptBuilder](../builders/chatpromptbuilder.mdx) in a RAG pipeline 2. The last component in the semantic search pipeline 3. Before an [ExtractiveReader](../readers/extractivereader.mdx) in an extractive QA pipeline | +| **Most common position in a pipeline** | 1. Before a [ChatPromptBuilder](../builders/chatpromptbuilder.mdx) in a RAG pipeline 2. The last component in the semantic search pipeline 3. Before a [TransformersExtractiveReader](../readers/transformersextractivereader.mdx) in an extractive QA pipeline | | **Mandatory init variables** | `document_store`: An instance of a [MongoDBAtlasDocumentStore](../../document-stores/mongodbatlasdocumentstore.mdx) | | **Mandatory run variables** | `query`: A query string to search for. If the query contains multiple terms, Atlas Search evaluates each term separately for matches. | | **Output variables** | `documents`: A list of documents | @@ -69,11 +69,17 @@ print(results["documents"]) Here's a Hybrid Retrieval pipeline example that makes use of both available MongoDB Atlas Retrievers: +The examples on this page use Sentence Transformers embedders that have moved to the `sentence-transformers-haystack` package. Install it to run the examples: + +```shell +pip install sentence-transformers-haystack +``` + ```python from haystack import Pipeline, Document from haystack.document_stores.types import DuplicatePolicy from haystack.components.writers import DocumentWriter -from haystack.components.embedders import ( +from haystack_integrations.components.embedders.sentence_transformers import ( SentenceTransformersDocumentEmbedder, SentenceTransformersTextEmbedder, ) diff --git a/docs-website/docs/pipeline-components/retrievers/multiqueryembeddingretriever.mdx b/docs-website/docs/pipeline-components/retrievers/multiqueryembeddingretriever.mdx index ce3aeb541d1..d52f9ce73b8 100644 --- a/docs-website/docs/pipeline-components/retrievers/multiqueryembeddingretriever.mdx +++ b/docs-website/docs/pipeline-components/retrievers/multiqueryembeddingretriever.mdx @@ -64,10 +64,16 @@ Before running the pipeline, documents must be embedded using a Document Embedde +The examples on this page use Sentence Transformers embedders that have moved to the `sentence-transformers-haystack` package. Install it to run the examples: + +```shell +pip install sentence-transformers-haystack +``` + ```python from haystack import Document, Pipeline from haystack.document_stores.in_memory import InMemoryDocumentStore -from haystack.components.embedders import ( +from haystack_integrations.components.embedders.sentence_transformers import ( SentenceTransformersTextEmbedder, SentenceTransformersDocumentEmbedder, ) @@ -138,7 +144,7 @@ components: init_parameters: {} top_k: 2 query_embedder: - type: haystack.components.embedders.sentence_transformers_text_embedder.SentenceTransformersTextEmbedder + type: haystack_integrations.components.embedders.sentence_transformers.sentence_transformers_text_embedder.SentenceTransformersTextEmbedder init_parameters: model: sentence-transformers/all-MiniLM-L6-v2 diff --git a/docs-website/docs/pipeline-components/retrievers/multiretriever.mdx b/docs-website/docs/pipeline-components/retrievers/multiretriever.mdx index c1244fc0419..d1f9f0f6fd1 100644 --- a/docs-website/docs/pipeline-components/retrievers/multiretriever.mdx +++ b/docs-website/docs/pipeline-components/retrievers/multiretriever.mdx @@ -54,11 +54,17 @@ The `join_mode` parameter controls how results from multiple retrievers are merg This example sets up a `MultiRetriever` combining a BM25 retriever and an embedding-based retriever (wrapped with `TextEmbeddingRetriever`). Both are queried in parallel and the results are merged using reciprocal rank fusion. +The examples on this page use Sentence Transformers embedders that have moved to the `sentence-transformers-haystack` package. Install it to run the examples: + +```shell +pip install sentence-transformers-haystack +``` + ```python from haystack import Document from haystack.document_stores.in_memory import InMemoryDocumentStore from haystack.document_stores.types import DuplicatePolicy -from haystack.components.embedders import ( +from haystack_integrations.components.embedders.sentence_transformers import ( SentenceTransformersDocumentEmbedder, SentenceTransformersTextEmbedder, ) @@ -126,7 +132,7 @@ from haystack import Document, Pipeline from haystack.document_stores.in_memory import InMemoryDocumentStore from haystack.document_stores.types import DuplicatePolicy from haystack.components.builders import ChatPromptBuilder -from haystack.components.embedders import ( +from haystack_integrations.components.embedders.sentence_transformers import ( SentenceTransformersDocumentEmbedder, SentenceTransformersTextEmbedder, ) diff --git a/docs-website/docs/pipeline-components/retrievers/opensearchembeddingretriever.mdx b/docs-website/docs/pipeline-components/retrievers/opensearchembeddingretriever.mdx index 7735f126caf..194287593be 100644 --- a/docs-website/docs/pipeline-components/retrievers/opensearchembeddingretriever.mdx +++ b/docs-website/docs/pipeline-components/retrievers/opensearchembeddingretriever.mdx @@ -13,7 +13,7 @@ An embedding-based Retriever compatible with the OpenSearch Document Store. | | | | --- | --- | -| **Most common position in a pipeline** | 1. After a Text Embedder and before a [`PromptBuilder`](../builders/promptbuilder.mdx) in a RAG pipeline 2. The last component in the semantic search pipeline 3. After a Text Embedder and before an [`ExtractiveReader`](../readers/extractivereader.mdx) in an extractive QA pipeline | +| **Most common position in a pipeline** | 1. After a Text Embedder and before a [`PromptBuilder`](../builders/promptbuilder.mdx) in a RAG pipeline 2. The last component in the semantic search pipeline 3. After a Text Embedder and before a [`TransformersExtractiveReader`](../readers/transformersextractivereader.mdx) in an extractive QA pipeline | | **Mandatory init variables** | `document_store`: An instance of an [OpenSearchDocumentStore](../../document-stores/opensearch-document-store.mdx) | | **Mandatory run variables** | `query_embedding`: A list of floats | | **Output variables** | `documents`: A list of documents | @@ -62,6 +62,12 @@ pip install opensearch-haystack Use this Retriever in a query Pipeline like this: +The examples on this page use Sentence Transformers embedders that have moved to the `sentence-transformers-haystack` package. Install it to run the examples: + +```shell +pip install sentence-transformers-haystack +``` + ```python from haystack_integrations.components.retrievers.opensearch import ( OpenSearchEmbeddingRetriever, @@ -71,7 +77,7 @@ from haystack_integrations.document_stores.opensearch import OpenSearchDocumentS from haystack.document_stores.types import DuplicatePolicy from haystack import Document from haystack import Pipeline -from haystack.components.embedders import ( +from haystack_integrations.components.embedders.sentence_transformers import ( SentenceTransformersTextEmbedder, SentenceTransformersDocumentEmbedder, ) diff --git a/docs-website/docs/pipeline-components/retrievers/opensearchhybridretriever.mdx b/docs-website/docs/pipeline-components/retrievers/opensearchhybridretriever.mdx index b95a4253f3e..ea74b615cbf 100644 --- a/docs-website/docs/pipeline-components/retrievers/opensearchhybridretriever.mdx +++ b/docs-website/docs/pipeline-components/retrievers/opensearchhybridretriever.mdx @@ -15,7 +15,7 @@ A Hybrid Retriever uses both traditional keyword-based search (such as BM25) and | | | | --- | --- | -| Most common position in a pipeline | 1. After a TextEmbedder and before a PromptBuilder in a RAG pipeline 2. The last component in a hybrid search pipeline 3. After a TextEmbedder and before an ExtractiveReader in an extractive QA pipeline | +| Most common position in a pipeline | 1. After a TextEmbedder and before a PromptBuilder in a RAG pipeline 2. The last component in a hybrid search pipeline 3. After a TextEmbedder and before a TransformersExtractiveReader in an extractive QA pipeline | | Mandatory init variables | `document_store`: An instance of `OpenSearchDocumentStore` to use for retrieval

`embedder`: Any [Embedder](../embedders.mdx) implementing the `TextEmbedder` protocol | | Mandatory run variables | `query`: A query string | | Output variables | `documents`: A list of documents matching the query | @@ -83,9 +83,15 @@ docker run -d \\ opensearchproject/opensearch:2.12.0 ``` +The examples on this page use Sentence Transformers embedders that have moved to the `sentence-transformers-haystack` package. Install it to run the examples: + +```shell +pip install sentence-transformers-haystack +``` + ```python from haystack import Document -from haystack.components.embedders import SentenceTransformersTextEmbedder, SentenceTransformersDocumentEmbedder +from haystack_integrations.components.embedders.sentence_transformers import SentenceTransformersTextEmbedder, SentenceTransformersDocumentEmbedder from haystack_integrations.components.retrievers.opensearch import OpenSearchHybridRetriever from haystack_integrations.document_stores.opensearch import OpenSearchDocumentStore diff --git a/docs-website/docs/pipeline-components/retrievers/oracleembeddingretriever.mdx b/docs-website/docs/pipeline-components/retrievers/oracleembeddingretriever.mdx index 6f63f6f2dbf..108c0c3058f 100644 --- a/docs-website/docs/pipeline-components/retrievers/oracleembeddingretriever.mdx +++ b/docs-website/docs/pipeline-components/retrievers/oracleembeddingretriever.mdx @@ -13,7 +13,7 @@ An embedding-based Retriever compatible with the Oracle Document Store. | | | | --- | --- | -| **Most common position in a pipeline** | 1. After a Text Embedder and before a [`PromptBuilder`](../builders/promptbuilder.mdx) in a RAG pipeline 2. The last component in a semantic search pipeline 3. After a Text Embedder and before an [`ExtractiveReader`](../readers/extractivereader.mdx) in an extractive QA pipeline | +| **Most common position in a pipeline** | 1. After a Text Embedder and before a [`PromptBuilder`](../builders/promptbuilder.mdx) in a RAG pipeline 2. The last component in a semantic search pipeline 3. After a Text Embedder and before a [`TransformersExtractiveReader`](../readers/transformersextractivereader.mdx) in an extractive QA pipeline | | **Mandatory init variables** | `document_store`: An instance of an [OracleDocumentStore](../../document-stores/oracledocumentstore.mdx) | | **Mandatory run variables** | `query_embedding`: A vector representing the query (a list of floats) | | **Output variables** | `documents`: A list of documents | @@ -48,6 +48,12 @@ Install the Oracle integration for Haystack: pip install oracle-haystack ``` +The examples on this page use Sentence Transformers embedders that have moved to the `sentence-transformers-haystack` package. Install it to run the examples: + +```shell +pip install sentence-transformers-haystack +``` + ## Usage ### On its own @@ -82,7 +88,7 @@ retriever.run(query_embedding=[0.1] * 768) ```python from haystack import Document, Pipeline from haystack.document_stores.types import DuplicatePolicy -from haystack.components.embedders import ( +from haystack_integrations.components.embedders.sentence_transformers import ( SentenceTransformersDocumentEmbedder, SentenceTransformersTextEmbedder, ) @@ -116,7 +122,6 @@ documents = [ document_embedder = SentenceTransformersDocumentEmbedder( model="sentence-transformers/all-MiniLM-L6-v2", ) -document_embedder.warm_up() documents_with_embeddings = document_embedder.run(documents) document_store.write_documents( diff --git a/docs-website/docs/pipeline-components/retrievers/pgvectorembeddingretriever.mdx b/docs-website/docs/pipeline-components/retrievers/pgvectorembeddingretriever.mdx index 83f1b61a0cb..af6cda6f1ea 100644 --- a/docs-website/docs/pipeline-components/retrievers/pgvectorembeddingretriever.mdx +++ b/docs-website/docs/pipeline-components/retrievers/pgvectorembeddingretriever.mdx @@ -13,7 +13,7 @@ An embedding-based Retriever compatible with the Pgvector Document Store. | | | | --- | --- | -| **Most common position in a pipeline** | 1. After a Text Embedder and before a [`PromptBuilder`](../builders/promptbuilder.mdx) in a RAG pipeline 2. The last component in the semantic search pipeline 3. After a Text Embedder and before an [`ExtractiveReader`](../readers/extractivereader.mdx) in an extractive QA pipeline | +| **Most common position in a pipeline** | 1. After a Text Embedder and before a [`PromptBuilder`](../builders/promptbuilder.mdx) in a RAG pipeline 2. The last component in the semantic search pipeline 3. After a Text Embedder and before a [`TransformersExtractiveReader`](../readers/transformersextractivereader.mdx) in an extractive QA pipeline | | **Mandatory init variables** | `document_store`: An instance of a [PgvectorDocumentStore](../../document-stores/pgvectordocumentstore.mdx) | | **Mandatory run variables** | `query_embedding`: A vector representing the query (a list of floats) | | **Output variables** | `documents`: A list of documents | @@ -49,6 +49,12 @@ To use pgvector with Haystack, install the `pgvector-haystack` integration: pip install pgvector-haystack ``` +The examples on this page use Sentence Transformers embedders that have moved to the `sentence-transformers-haystack` package. Install it to run the examples: + +```shell +pip install sentence-transformers-haystack +``` + ## Usage ### On its own @@ -77,7 +83,7 @@ retriever.run(query_embedding=[0.1] * 768) import os from haystack.document_stores import DuplicatePolicy from haystack import Document, Pipeline -from haystack.components.embedders import ( +from haystack_integrations.components.embedders.sentence_transformers import ( SentenceTransformersTextEmbedder, SentenceTransformersDocumentEmbedder, ) diff --git a/docs-website/docs/pipeline-components/retrievers/pineconedenseretriever.mdx b/docs-website/docs/pipeline-components/retrievers/pineconedenseretriever.mdx index a1778c1e88b..4fa7c824bdd 100644 --- a/docs-website/docs/pipeline-components/retrievers/pineconedenseretriever.mdx +++ b/docs-website/docs/pipeline-components/retrievers/pineconedenseretriever.mdx @@ -13,7 +13,7 @@ An embedding-based Retriever compatible with the Pinecone Document Store. | | | | --- | --- | -| **Most common position in a pipeline** | 1. After a Text Embedder and before a [`PromptBuilder`](../builders/promptbuilder.mdx) in a RAG pipeline 2. The last component in the semantic search pipeline 3. After a Text Embedder and before an [`ExtractiveReader`](../readers/extractivereader.mdx) in an extractive QA pipeline | +| **Most common position in a pipeline** | 1. After a Text Embedder and before a [`PromptBuilder`](../builders/promptbuilder.mdx) in a RAG pipeline 2. The last component in the semantic search pipeline 3. After a Text Embedder and before a [`TransformersExtractiveReader`](../readers/transformersextractivereader.mdx) in an extractive QA pipeline | | **Mandatory init variables** | `document_store`: An instance of a [PineconeDocumentStore](../../document-stores/pinecone-document-store.mdx) | | **Mandatory run variables** | `query_embedding`: A vector representing the query (a list of floats) | | **Output variables** | `documents`: A list of documents | @@ -64,7 +64,7 @@ Install the dependencies you’ll need: ```shell pip install pinecone-haystack -pip install sentence-transformers +pip install sentence-transformers-haystack ``` Use this Retriever in a query Pipeline like this: @@ -73,7 +73,7 @@ Use this Retriever in a query Pipeline like this: from haystack.document_stores.types import DuplicatePolicy from haystack import Document from haystack import Pipeline -from haystack.components.embedders import ( +from haystack_integrations.components.embedders.sentence_transformers import ( SentenceTransformersTextEmbedder, SentenceTransformersDocumentEmbedder, ) diff --git a/docs-website/docs/pipeline-components/retrievers/qdrantembeddingretriever.mdx b/docs-website/docs/pipeline-components/retrievers/qdrantembeddingretriever.mdx index aad3925c433..9007e93ee7a 100644 --- a/docs-website/docs/pipeline-components/retrievers/qdrantembeddingretriever.mdx +++ b/docs-website/docs/pipeline-components/retrievers/qdrantembeddingretriever.mdx @@ -13,7 +13,7 @@ An embedding-based Retriever compatible with the Qdrant Document Store. | | | | --- | --- | -| **Most common position in a pipeline** | 1\. After a Text Embedder and before a [`PromptBuilder`](../builders/promptbuilder.mdx) in a RAG Pipeline

2. The last component in the semantic search pipeline
3. After a Text Embedder and before an [`ExtractiveReader`](../readers/extractivereader.mdx) in an extractive QA pipeline | +| **Most common position in a pipeline** | 1\. After a Text Embedder and before a [`PromptBuilder`](../builders/promptbuilder.mdx) in a RAG Pipeline

2. The last component in the semantic search pipeline
3. After a Text Embedder and before a [`TransformersExtractiveReader`](../readers/transformersextractivereader.mdx) in an extractive QA pipeline | | **Mandatory init variables** | `document_store`: An instance of a [QdrantDocumentStore](../../document-stores/qdrant-document-store.mdx) | | **Mandatory run variables** | `query_embedding`: A vector representing the query (a list of floats) | | **Output variables** | `documents`: A list of documents | @@ -65,11 +65,17 @@ retriever.run(query_embedding=[0.1] * 768) #### In a Pipeline +The examples on this page use Sentence Transformers embedders that have moved to the `sentence-transformers-haystack` package. Install it to run the examples: + +```shell +pip install sentence-transformers-haystack +``` + ```python from haystack.document_stores.types import DuplicatePolicy from haystack import Document from haystack import Pipeline -from haystack.components.embedders import ( +from haystack_integrations.components.embedders.sentence_transformers import ( SentenceTransformersTextEmbedder, SentenceTransformersDocumentEmbedder, ) diff --git a/docs-website/docs/pipeline-components/retrievers/supabasepgvectorembeddingretriever.mdx b/docs-website/docs/pipeline-components/retrievers/supabasepgvectorembeddingretriever.mdx index 80c6cc870b3..97079c517a8 100644 --- a/docs-website/docs/pipeline-components/retrievers/supabasepgvectorembeddingretriever.mdx +++ b/docs-website/docs/pipeline-components/retrievers/supabasepgvectorembeddingretriever.mdx @@ -13,7 +13,7 @@ An embedding-based Retriever compatible with the SupabasePgvectorDocumentStore. | | | | --- | --- | -| **Most common position in a pipeline** | 1. After a Text Embedder and before a [`PromptBuilder`](../builders/promptbuilder.mdx) in a RAG pipeline 2. The last component in the semantic search pipeline 3. After a Text Embedder and before an [`ExtractiveReader`](../readers/extractivereader.mdx) in an extractive QA pipeline | +| **Most common position in a pipeline** | 1. After a Text Embedder and before a [`PromptBuilder`](../builders/promptbuilder.mdx) in a RAG pipeline 2. The last component in the semantic search pipeline 3. After a Text Embedder and before a [`TransformersExtractiveReader`](../readers/transformersextractivereader.mdx) in an extractive QA pipeline | | **Mandatory init variables** | `document_store`: An instance of a [SupabasePgvectorDocumentStore](../../document-stores/supabasedocumentstore.mdx) | | **Mandatory run variables** | `query_embedding`: A vector representing the query (a list of floats) | | **Output variables** | `documents`: A list of documents | @@ -39,6 +39,12 @@ Some relevant parameters that impact embedding retrieval must be defined when th pip install supabase-haystack ``` +The examples on this page use Sentence Transformers embedders that have moved to the `sentence-transformers-haystack` package. Install it to run the examples: + +```shell +pip install sentence-transformers-haystack +``` + ## Usage ### On its own @@ -65,7 +71,7 @@ retriever.run(query_embedding=[0.1] * 768) ```python from haystack import Document, Pipeline from haystack.document_stores.types import DuplicatePolicy -from haystack.components.embedders import ( +from haystack_integrations.components.embedders.sentence_transformers import ( SentenceTransformersTextEmbedder, SentenceTransformersDocumentEmbedder, ) diff --git a/docs-website/docs/pipeline-components/retrievers/textembeddingretriever.mdx b/docs-website/docs/pipeline-components/retrievers/textembeddingretriever.mdx index ac946ec6b73..a006818832b 100644 --- a/docs-website/docs/pipeline-components/retrievers/textembeddingretriever.mdx +++ b/docs-website/docs/pipeline-components/retrievers/textembeddingretriever.mdx @@ -33,11 +33,17 @@ You can use it anywhere an embedding-based retriever fits: in RAG pipelines befo ### On its own +The examples on this page use Sentence Transformers embedders that have moved to the `sentence-transformers-haystack` package. Install it to run the examples: + +```shell +pip install sentence-transformers-haystack +``` + ```python from haystack import Document from haystack.document_stores.in_memory import InMemoryDocumentStore from haystack.document_stores.types import DuplicatePolicy -from haystack.components.embedders import ( +from haystack_integrations.components.embedders.sentence_transformers import ( SentenceTransformersDocumentEmbedder, SentenceTransformersTextEmbedder, ) @@ -86,7 +92,9 @@ for doc in result["documents"]: `TextEmbeddingRetriever` is most commonly used as one of the retrievers inside a [`MultiRetriever`](multiretriever.mdx): ```python -from haystack.components.embedders import SentenceTransformersTextEmbedder +from haystack_integrations.components.embedders.sentence_transformers import ( + SentenceTransformersTextEmbedder, +) from haystack.components.retrievers import ( InMemoryBM25Retriever, InMemoryEmbeddingRetriever, diff --git a/docs-website/docs/pipeline-components/retrievers/valkeyembeddingretriever.mdx b/docs-website/docs/pipeline-components/retrievers/valkeyembeddingretriever.mdx index 1fdbfe2e598..82f2742068a 100644 --- a/docs-website/docs/pipeline-components/retrievers/valkeyembeddingretriever.mdx +++ b/docs-website/docs/pipeline-components/retrievers/valkeyembeddingretriever.mdx @@ -13,7 +13,7 @@ This is an embedding Retriever compatible with the Valkey Document Store. | | | | --- | --- | -| **Most common position in a pipeline** | 1. After a Text Embedder and before a [`ChatPromptBuilder`](../builders/chatpromptbuilder.mdx) or [`PromptBuilder`](../builders/promptbuilder.mdx) in a RAG pipeline 2. The last component in a semantic search pipeline 3. After a Text Embedder and before an [`ExtractiveReader`](../readers/extractivereader.mdx) in an extractive QA pipeline | +| **Most common position in a pipeline** | 1. After a Text Embedder and before a [`ChatPromptBuilder`](../builders/chatpromptbuilder.mdx) or [`PromptBuilder`](../builders/promptbuilder.mdx) in a RAG pipeline 2. The last component in a semantic search pipeline 3. After a Text Embedder and before a [`TransformersExtractiveReader`](../readers/transformersextractivereader.mdx) in an extractive QA pipeline | | **Mandatory init variables** | `document_store`: An instance of a [ValkeyDocumentStore](../../document-stores/valkeydocumentstore.mdx) | | **Mandatory run variables** | `query_embedding`: A list of floats | | **Output variables** | `documents`: A list of documents | @@ -66,9 +66,15 @@ retriever.run(query_embedding=[0.1] * 768) ### In a Pipeline +The examples on this page use Sentence Transformers embedders that have moved to the `sentence-transformers-haystack` package. Install it to run the examples: + +```shell +pip install sentence-transformers-haystack +``` + ```python from haystack import Document, Pipeline -from haystack.components.embedders import ( +from haystack_integrations.components.embedders.sentence_transformers import ( SentenceTransformersDocumentEmbedder, SentenceTransformersTextEmbedder, ) diff --git a/docs-website/docs/pipeline-components/retrievers/vespaembeddingretriever.mdx b/docs-website/docs/pipeline-components/retrievers/vespaembeddingretriever.mdx index ff4ad486d84..5cfbff411bb 100644 --- a/docs-website/docs/pipeline-components/retrievers/vespaembeddingretriever.mdx +++ b/docs-website/docs/pipeline-components/retrievers/vespaembeddingretriever.mdx @@ -13,7 +13,7 @@ An embedding-based Retriever compatible with the Vespa Document Store. | | | | --- | --- | -| **Most common position in a pipeline** | 1. After a Text Embedder and before a [`PromptBuilder`](../builders/promptbuilder.mdx) in a RAG pipeline 2. The last component in the semantic search pipeline 3. After a Text Embedder and before an [`ExtractiveReader`](../readers/extractivereader.mdx) in an extractive QA pipeline | +| **Most common position in a pipeline** | 1. After a Text Embedder and before a [`PromptBuilder`](../builders/promptbuilder.mdx) in a RAG pipeline 2. The last component in the semantic search pipeline 3. After a Text Embedder and before a [`TransformersExtractiveReader`](../readers/transformersextractivereader.mdx) in an extractive QA pipeline | | **Mandatory init variables** | `document_store`: An instance of a [VespaDocumentStore](../../document-stores/vespadocumentstore.mdx) | | **Mandatory run variables** | `query_embedding`: A vector representing the query (a list of floats) | | **Output variables** | `documents`: A list of documents | @@ -48,6 +48,12 @@ pip install vespa-haystack To run Vespa locally, see the [Vespa quick start](https://docs.vespa.ai/en/vespa-quick-start.html). +The examples on this page use Sentence Transformers embedders that have moved to the `sentence-transformers-haystack` package. Install it to run the examples: + +```shell +pip install sentence-transformers-haystack +``` + ## Usage ### On its own @@ -71,7 +77,7 @@ retriever.run(query_embedding=[0.1] * 768) ```python from haystack import Document, Pipeline -from haystack.components.embedders import ( +from haystack_integrations.components.embedders.sentence_transformers import ( SentenceTransformersDocumentEmbedder, SentenceTransformersTextEmbedder, ) diff --git a/docs-website/docs/pipeline-components/retrievers/weaviateembeddingretriever.mdx b/docs-website/docs/pipeline-components/retrievers/weaviateembeddingretriever.mdx index 85842658d22..f7549227090 100644 --- a/docs-website/docs/pipeline-components/retrievers/weaviateembeddingretriever.mdx +++ b/docs-website/docs/pipeline-components/retrievers/weaviateembeddingretriever.mdx @@ -13,7 +13,7 @@ This is an embedding Retriever compatible with the Weaviate Document Store. | | | | --- | --- | -| **Most common position in a pipeline** | 1. After a Text Embedder and before a [`PromptBuilder`](../builders/promptbuilder.mdx) in a RAG pipeline 2. The last component in the semantic search pipeline 3. After a Text Embedder and before an [`ExtractiveReader`](../readers/extractivereader.mdx) in an extractive QA pipeline | +| **Most common position in a pipeline** | 1. After a Text Embedder and before a [`PromptBuilder`](../builders/promptbuilder.mdx) in a RAG pipeline 2. The last component in the semantic search pipeline 3. After a Text Embedder and before a [`TransformersExtractiveReader`](../readers/transformersextractivereader.mdx) in an extractive QA pipeline | | **Mandatory init variables** | `document_store`: An instance of a [WeaviateDocumentStore](../../document-stores/weaviatedocumentstore.mdx) | | **Mandatory run variables** | `query_embedding`: A list of floats | | **Output variables** | `documents`: A list of documents | @@ -69,11 +69,17 @@ retriever.run(query_embedding=[0.1] * 768) ### In a Pipeline +The examples on this page use Sentence Transformers embedders that have moved to the `sentence-transformers-haystack` package. Install it to run the examples: + +```shell +pip install sentence-transformers-haystack +``` + ```python from haystack.document_stores.types import DuplicatePolicy from haystack import Document from haystack import Pipeline -from haystack.components.embedders import ( +from haystack_integrations.components.embedders.sentence_transformers import ( SentenceTransformersTextEmbedder, SentenceTransformersDocumentEmbedder, ) diff --git a/docs-website/docs/pipeline-components/retrievers/weaviatehybridretriever.mdx b/docs-website/docs/pipeline-components/retrievers/weaviatehybridretriever.mdx index 645da4ddba4..094a50c97cc 100644 --- a/docs-website/docs/pipeline-components/retrievers/weaviatehybridretriever.mdx +++ b/docs-website/docs/pipeline-components/retrievers/weaviatehybridretriever.mdx @@ -13,7 +13,7 @@ A Retriever that combines BM25 keyword search and vector similarity to fetch doc | | | | --- | --- | -| **Most common position in a pipeline** | 1. After a Text Embedder and before a [`PromptBuilder`](../builders/promptbuilder.mdx) in a RAG pipeline 2. The last component in a hybrid search pipeline 3. After a Text Embedder and before an [`ExtractiveReader`](../readers/extractivereader.mdx) in an extractive QA pipeline | +| **Most common position in a pipeline** | 1. After a Text Embedder and before a [`PromptBuilder`](../builders/promptbuilder.mdx) in a RAG pipeline 2. The last component in a hybrid search pipeline 3. After a Text Embedder and before a [`TransformersExtractiveReader`](../readers/transformersextractivereader.mdx) in an extractive QA pipeline | | **Mandatory init variables** | `document_store`: An instance of a [WeaviateDocumentStore](../../document-stores/weaviatedocumentstore.mdx) | | **Mandatory run variables** | `query`: A string

`query_embedding`: A list of floats | | **Output variables** | `documents`: A list of documents (matching the query) | @@ -75,11 +75,17 @@ retriever.run(query="How many languages are there?", query_embedding=[0.1] * 768 ### In a pipeline +The examples on this page use Sentence Transformers embedders that have moved to the `sentence-transformers-haystack` package. Install it to run the examples: + +```shell +pip install sentence-transformers-haystack +``` + ```python from haystack.document_stores.types import DuplicatePolicy from haystack import Document from haystack import Pipeline -from haystack.components.embedders import ( +from haystack_integrations.components.embedders.sentence_transformers import ( SentenceTransformersTextEmbedder, SentenceTransformersDocumentEmbedder, ) diff --git a/docs-website/docs/pipeline-components/routers/llmmessagesrouter.mdx b/docs-website/docs/pipeline-components/routers/llmmessagesrouter.mdx index 370f2f2cb0c..c94e0f08ee6 100644 --- a/docs-website/docs/pipeline-components/routers/llmmessagesrouter.mdx +++ b/docs-website/docs/pipeline-components/routers/llmmessagesrouter.mdx @@ -49,8 +49,16 @@ Below is an example of using `LLMMessagesRouter` to route Chat Messages to two We use Llama Guard 4 for content moderation. To use this model with the Hugging Face API, you need to [request access](https://huggingface.co/meta-llama/Llama-Guard-4-12B) and set the `HF_TOKEN` environment variable. +The examples on this page use Hugging Face API components that have moved to the `huggingface-api-haystack` package. Install it to run the examples: + +```shell +pip install huggingface-api-haystack +``` + ```python -from haystack.components.generators.chat import HuggingFaceAPIChatGenerator +from haystack_integrations.components.generators.huggingface_api import ( + HuggingFaceAPIChatGenerator, +) from haystack.components.routers.llm_messages_router import LLMMessagesRouter from haystack.dataclasses import ChatMessage @@ -129,9 +137,9 @@ from haystack import Document, Pipeline from haystack.dataclasses import ChatMessage from haystack.document_stores.in_memory import InMemoryDocumentStore from haystack.components.builders import ChatPromptBuilder -from haystack.components.generators.chat import ( +from haystack.components.generators.chat import OpenAIChatGenerator +from haystack_integrations.components.generators.huggingface_api import ( HuggingFaceAPIChatGenerator, - OpenAIChatGenerator, ) from haystack.components.retrievers.in_memory import InMemoryBM25Retriever from haystack.components.routers import LLMMessagesRouter diff --git a/docs-website/docs/pipeline-components/routers/metadatarouter.mdx b/docs-website/docs/pipeline-components/routers/metadatarouter.mdx index 41a82f64569..fdd8213430d 100644 --- a/docs-website/docs/pipeline-components/routers/metadatarouter.mdx +++ b/docs-website/docs/pipeline-components/routers/metadatarouter.mdx @@ -80,10 +80,18 @@ result = router.run(documents=streams) Below is an example of an indexing pipeline that converts text files to documents and uses the `DocumentLanguageClassifier` to detect the language of the text and add it to the documents' metadata. It then uses the `MetadataRouter` to forward only English language documents to the `DocumentWriter`. Documents of other languages will not be added to the `DocumentStore`. +The examples on this page use language classification components that have moved to the `langdetect-haystack` package. Install it to run the examples: + +```shell +pip install langdetect-haystack +``` + ```python from haystack import Pipeline from haystack.components.file_converters import TextFileToDocument -from haystack.components.classifiers import DocumentLanguageClassifier +from haystack_integrations.components.classifiers.langdetect import ( + DocumentLanguageClassifier, +) from haystack.components.routers import MetadataRouter from haystack.components.writers import DocumentWriter from haystack.document_stores.in_memory import InMemoryDocumentStore diff --git a/docs-website/docs/pipeline-components/routers/transformerszeroshottextrouter.mdx b/docs-website/docs/pipeline-components/routers/transformerszeroshottextrouter.mdx index 2c79d89f0a1..8c3cd3529ea 100644 --- a/docs-website/docs/pipeline-components/routers/transformerszeroshottextrouter.mdx +++ b/docs-website/docs/pipeline-components/routers/transformerszeroshottextrouter.mdx @@ -55,12 +55,18 @@ We then create a retrieving pipeline with the `TransformersZeroShotTextRouter` t Finally, the pipeline is executed with a sample text: "What is the capital of Germany?” which categorizes this input text as “query” and routes it to Query Embedder and subsequently Query Retriever to return the relevant results. +The examples on this page use Sentence Transformers embedders that have moved to the `sentence-transformers-haystack` package. Install it to run the examples: + +```shell +pip install sentence-transformers-haystack +``` + ```python from haystack import Document from haystack.document_stores.in_memory import InMemoryDocumentStore from haystack.core.pipeline import Pipeline from haystack_integrations.components.routers.transformers import TransformersZeroShotTextRouter -from haystack.components.embedders import SentenceTransformersTextEmbedder, SentenceTransformersDocumentEmbedder +from haystack_integrations.components.embedders.sentence_transformers import SentenceTransformersTextEmbedder, SentenceTransformersDocumentEmbedder from haystack.components.retrievers import InMemoryEmbeddingRetriever document_store = InMemoryDocumentStore() diff --git a/docs-website/docs/pipeline-components/samplers/toppsampler.mdx b/docs-website/docs/pipeline-components/samplers/toppsampler.mdx index 3e1ab2e3b23..d982dede8ce 100644 --- a/docs-website/docs/pipeline-components/samplers/toppsampler.mdx +++ b/docs-website/docs/pipeline-components/samplers/toppsampler.mdx @@ -54,6 +54,12 @@ print(docs) To best understand how can you use a `TopPSampler` and which components to pair it with, explore the following example. +The examples on this page use Sentence Transformers rankers and SerperDev web search component that have moved to the `sentence-transformers-haystack` and `serperdev-haystack` packages. Install them to run the examples: + +```shell +pip install sentence-transformers-haystack serperdev-haystack +``` + ```python # import necessary dependencies from haystack import Pipeline @@ -62,10 +68,12 @@ from haystack.components.fetchers import LinkContentFetcher from haystack.components.converters import HTMLToDocument from haystack.components.generators.chat import OpenAIChatGenerator from haystack.components.preprocessors import DocumentSplitter -from haystack.components.rankers import SentenceTransformersSimilarityRanker +from haystack_integrations.components.rankers.sentence_transformers import ( + SentenceTransformersSimilarityRanker, +) from haystack.components.routers.file_type_router import FileTypeRouter from haystack.components.samplers import TopPSampler -from haystack.components.websearch import SerperDevWebSearch +from haystack_integrations.components.websearch.serperdev import SerperDevWebSearch from haystack.utils import Secret from haystack.dataclasses import ChatMessage diff --git a/docs-website/docs/pipeline-components/writers/documentwriter.mdx b/docs-website/docs/pipeline-components/writers/documentwriter.mdx index ec93c17268a..f4782cd5eec 100644 --- a/docs-website/docs/pipeline-components/writers/documentwriter.mdx +++ b/docs-website/docs/pipeline-components/writers/documentwriter.mdx @@ -63,11 +63,19 @@ document_writer.run(documents=documents) Below is an example of an indexing pipeline that first uses the `SentenceTransformersDocumentEmbedder` to create embeddings of documents and then use the `DocumentWriter` to write the documents to an `InMemoryDocumentStore`: +The examples on this page use Sentence Transformers embedders that have moved to the `sentence-transformers-haystack` package. Install it to run the examples: + +```shell +pip install sentence-transformers-haystack +``` + ```python from haystack.pipeline import Pipeline from haystack.document_stores.in_memory import InMemoryDocumentStore from haystack.document_stores.types import DuplicatePolicy -from haystack.components.embedders import SentenceTransformersDocumentEmbedder +from haystack_integrations.components.embedders.sentence_transformers import ( + SentenceTransformersDocumentEmbedder, +) from haystack.components.writers import DocumentWriter documents = [ diff --git a/docs-website/docs/tools/componenttool.mdx b/docs-website/docs/tools/componenttool.mdx index fb72975f921..71601a76cea 100644 --- a/docs-website/docs/tools/componenttool.mdx +++ b/docs-website/docs/tools/componenttool.mdx @@ -50,12 +50,18 @@ The recommended way to use `ComponentTool` in Haystack is with the [`Agent`](../ ### With the Agent Component +The examples on this page use SerperDev web search component that have moved to the `serperdev-haystack` package. Install it to run the examples: + +```shell +pip install serperdev-haystack +``` + ```python from haystack.components.generators.chat import OpenAIChatGenerator from haystack.dataclasses import ChatMessage from haystack.tools import ComponentTool from haystack.components.agents import Agent -from haystack.components.websearch import SerperDevWebSearch +from haystack_integrations.components.websearch.serperdev import SerperDevWebSearch from haystack.utils import Secret # Create a SerperDev search component @@ -88,7 +94,7 @@ You can also wire `ComponentTool` into a pipeline manually with `ChatGenerator` ```python from haystack import Pipeline from haystack.tools import ComponentTool -from haystack.components.websearch import SerperDevWebSearch +from haystack_integrations.components.websearch.serperdev import SerperDevWebSearch from haystack.utils import Secret from haystack.components.tools.tool_invoker import ToolInvoker from haystack.components.generators.chat import OpenAIChatGenerator diff --git a/docs-website/docs/tools/pipelinetool.mdx b/docs-website/docs/tools/pipelinetool.mdx index 28c16d1a279..3d46c64a980 100644 --- a/docs-website/docs/tools/pipelinetool.mdx +++ b/docs-website/docs/tools/pipelinetool.mdx @@ -51,11 +51,17 @@ The recommended way to use `PipelineTool` in Haystack is with the [`Agent`](../p You can create a `PipelineTool` from any existing Haystack pipeline: +The examples on this page use Sentence Transformers embedders that have moved to the `sentence-transformers-haystack` package. Install it to run the examples: + +```shell +pip install sentence-transformers-haystack +``` + ```python from haystack import Document, Pipeline from haystack.tools import PipelineTool from haystack.components.retrievers.in_memory import InMemoryBM25Retriever -from haystack.components.rankers.sentence_transformers_similarity import ( +from haystack_integrations.components.rankers.sentence_transformers import ( SentenceTransformersSimilarityRanker, ) from haystack.document_stores.in_memory import InMemoryDocumentStore @@ -105,10 +111,8 @@ print(retrieval_tool) from haystack import Document, Pipeline from haystack.tools import PipelineTool from haystack.document_stores.in_memory import InMemoryDocumentStore -from haystack.components.embedders.sentence_transformers_text_embedder import ( +from haystack_integrations.components.embedders.sentence_transformers import ( SentenceTransformersTextEmbedder, -) -from haystack.components.embedders.sentence_transformers_document_embedder import ( SentenceTransformersDocumentEmbedder, ) from haystack.components.retrievers import InMemoryEmbeddingRetriever @@ -173,10 +177,8 @@ You can also wire `PipelineTool` into a pipeline manually with `ChatGenerator` a from haystack import Document, Pipeline from haystack.tools import PipelineTool from haystack.document_stores.in_memory import InMemoryDocumentStore -from haystack.components.embedders.sentence_transformers_text_embedder import ( +from haystack_integrations.components.embedders.sentence_transformers import ( SentenceTransformersTextEmbedder, -) -from haystack.components.embedders.sentence_transformers_document_embedder import ( SentenceTransformersDocumentEmbedder, ) from haystack.components.retrievers import InMemoryEmbeddingRetriever @@ -197,7 +199,6 @@ documents = [ content="He is best known for his contributions to the design of the modern alternating current (AC) electricity supply system.", ), ] -document_embedder.warm_up() docs_with_embeddings = document_embedder.run(documents=documents)["documents"] document_store.write_documents(docs_with_embeddings) diff --git a/haystack/components/joiners/document_joiner.py b/haystack/components/joiners/document_joiner.py index 86c582dad09..0a96837474a 100644 --- a/haystack/components/joiners/document_joiner.py +++ b/haystack/components/joiners/document_joiner.py @@ -57,7 +57,9 @@ class DocumentJoiner: ```python from haystack import Pipeline, Document - from haystack.components.embedders import SentenceTransformersTextEmbedder, SentenceTransformersDocumentEmbedder + # Requires: pip install sentence-transformers-haystack + from haystack_integrations.components.embedders.sentence_transformers import SentenceTransformersTextEmbedder + from haystack_integrations.components.embedders.sentence_transformers import SentenceTransformersDocumentEmbedder from haystack.components.joiners import DocumentJoiner from haystack.components.retrievers import InMemoryBM25Retriever from haystack.components.retrievers import InMemoryEmbeddingRetriever diff --git a/haystack/components/preprocessors/embedding_based_document_splitter.py b/haystack/components/preprocessors/embedding_based_document_splitter.py index 939d24d35f8..5cca234dd5d 100644 --- a/haystack/components/preprocessors/embedding_based_document_splitter.py +++ b/haystack/components/preprocessors/embedding_based_document_splitter.py @@ -37,7 +37,8 @@ class EmbeddingBasedDocumentSplitter: ```python from haystack import Document - from haystack.components.embedders import SentenceTransformersDocumentEmbedder + # Requires: pip install sentence-transformers-haystack + from haystack_integrations.components.embedders.sentence_transformers import SentenceTransformersDocumentEmbedder from haystack.components.preprocessors import EmbeddingBasedDocumentSplitter # Create a document with content that has a clear topic shift diff --git a/haystack/components/retrievers/in_memory/embedding_retriever.py b/haystack/components/retrievers/in_memory/embedding_retriever.py index 0f03b262fea..e1bcd09c714 100644 --- a/haystack/components/retrievers/in_memory/embedding_retriever.py +++ b/haystack/components/retrievers/in_memory/embedding_retriever.py @@ -23,7 +23,9 @@ class InMemoryEmbeddingRetriever: ### Usage example ```python from haystack import Document - from haystack.components.embedders import SentenceTransformersDocumentEmbedder, SentenceTransformersTextEmbedder + # Requires: pip install sentence-transformers-haystack + from haystack_integrations.components.embedders.sentence_transformers import SentenceTransformersDocumentEmbedder + from haystack_integrations.components.embedders.sentence_transformers import SentenceTransformersTextEmbedder from haystack.components.retrievers.in_memory import InMemoryEmbeddingRetriever from haystack.document_stores.in_memory import InMemoryDocumentStore diff --git a/haystack/components/retrievers/multi_query_embedding_retriever.py b/haystack/components/retrievers/multi_query_embedding_retriever.py index a066ae7319f..61869139c1b 100644 --- a/haystack/components/retrievers/multi_query_embedding_retriever.py +++ b/haystack/components/retrievers/multi_query_embedding_retriever.py @@ -28,8 +28,9 @@ class MultiQueryEmbeddingRetriever: from haystack import Document from haystack.document_stores.in_memory import InMemoryDocumentStore from haystack.document_stores.types import DuplicatePolicy - from haystack.components.embedders import SentenceTransformersTextEmbedder - from haystack.components.embedders import SentenceTransformersDocumentEmbedder + # Requires: pip install sentence-transformers-haystack + from haystack_integrations.components.embedders.sentence_transformers import SentenceTransformersTextEmbedder + from haystack_integrations.components.embedders.sentence_transformers import SentenceTransformersDocumentEmbedder from haystack.components.retrievers import InMemoryEmbeddingRetriever from haystack.components.writers import DocumentWriter from haystack.components.retrievers import MultiQueryEmbeddingRetriever diff --git a/haystack/components/retrievers/multi_retriever.py b/haystack/components/retrievers/multi_retriever.py index 911fa47cb07..7bbf245f1a3 100644 --- a/haystack/components/retrievers/multi_retriever.py +++ b/haystack/components/retrievers/multi_retriever.py @@ -38,7 +38,9 @@ class MultiRetriever: from haystack.document_stores.types import DuplicatePolicy from haystack.components.retrievers import InMemoryBM25Retriever, InMemoryEmbeddingRetriever from haystack.components.retrievers import TextEmbeddingRetriever, MultiRetriever - from haystack.components.embedders import SentenceTransformersTextEmbedder, SentenceTransformersDocumentEmbedder + # Requires: pip install sentence-transformers-haystack + from haystack_integrations.components.embedders.sentence_transformers import SentenceTransformersTextEmbedder + from haystack_integrations.components.embedders.sentence_transformers import SentenceTransformersDocumentEmbedder from haystack.components.writers import DocumentWriter documents = [ diff --git a/haystack/components/retrievers/text_embedding_retriever.py b/haystack/components/retrievers/text_embedding_retriever.py index f8cc2100267..c126e648da4 100644 --- a/haystack/components/retrievers/text_embedding_retriever.py +++ b/haystack/components/retrievers/text_embedding_retriever.py @@ -26,7 +26,9 @@ class TextEmbeddingRetriever: from haystack import Document from haystack.document_stores.in_memory import InMemoryDocumentStore from haystack.document_stores.types import DuplicatePolicy - from haystack.components.embedders import SentenceTransformersTextEmbedder, SentenceTransformersDocumentEmbedder + # Requires: pip install sentence-transformers-haystack + from haystack_integrations.components.embedders.sentence_transformers import SentenceTransformersTextEmbedder + from haystack_integrations.components.embedders.sentence_transformers import SentenceTransformersDocumentEmbedder from haystack.components.retrievers import InMemoryEmbeddingRetriever, TextEmbeddingRetriever from haystack.components.writers import DocumentWriter diff --git a/haystack/tools/pipeline_tool.py b/haystack/tools/pipeline_tool.py index e2368a6e5e8..2caca5d50d8 100644 --- a/haystack/tools/pipeline_tool.py +++ b/haystack/tools/pipeline_tool.py @@ -38,10 +38,9 @@ class PipelineTool(ComponentTool): from haystack import Document, Pipeline from haystack.dataclasses import ChatMessage from haystack.document_stores.in_memory import InMemoryDocumentStore - from haystack.components.embedders.sentence_transformers_text_embedder import SentenceTransformersTextEmbedder - from haystack.components.embedders.sentence_transformers_document_embedder import ( - SentenceTransformersDocumentEmbedder - ) + # Requires: pip install sentence-transformers-haystack + from haystack_integrations.components.embedders.sentence_transformers import SentenceTransformersTextEmbedder + from haystack_integrations.components.embedders.sentence_transformers import SentenceTransformersDocumentEmbedder from haystack.components.generators.chat import OpenAIChatGenerator from haystack.components.retrievers import InMemoryEmbeddingRetriever from haystack.components.agents import Agent