From e618273fe0e9637171cf8d9cb31a8b57781fe4da Mon Sep 17 00:00:00 2001 From: anakin87 Date: Thu, 9 Jul 2026 10:54:31 +0200 Subject: [PATCH 1/4] remove generators mentions --- .../docs/concepts/device-management.mdx | 24 ++- .../development/enabling-gpu-acceleration.mdx | 2 +- .../builders/promptbuilder.mdx | 19 +- .../docs/pipeline-components/generators.mdx | 4 - .../generators/azureopenaigenerator.mdx | 142 ------------- .../huggingfaceapichatgenerator.mdx | 4 - .../generators/huggingfaceapigenerator.mdx | 191 ----------------- .../huggingfacelocalchatgenerator.mdx | 4 - .../generators/huggingfacelocalgenerator.mdx | 124 ----------- .../generators/openaigenerator.mdx | 198 ------------------ .../docs/pipeline-components/rankers.mdx | 1 - .../rankers/choosing-the-right-ranker.mdx | 3 +- .../rankers/transformerssimilarityranker.mdx | 114 ---------- docs-website/sidebars.js | 5 - 14 files changed, 29 insertions(+), 806 deletions(-) delete mode 100644 docs-website/docs/pipeline-components/generators/azureopenaigenerator.mdx delete mode 100644 docs-website/docs/pipeline-components/generators/huggingfaceapigenerator.mdx delete mode 100644 docs-website/docs/pipeline-components/generators/huggingfacelocalgenerator.mdx delete mode 100644 docs-website/docs/pipeline-components/generators/openaigenerator.mdx delete mode 100644 docs-website/docs/pipeline-components/rankers/transformerssimilarityranker.mdx diff --git a/docs-website/docs/concepts/device-management.mdx b/docs-website/docs/concepts/device-management.mdx index b445dbc435f..76adea72265 100644 --- a/docs-website/docs/concepts/device-management.mdx +++ b/docs-website/docs/concepts/device-management.mdx @@ -9,7 +9,7 @@ description: "This page discusses the concept of device management in the contex This page discusses the concept of device management in the context of Haystack. -Many Haystack components, such as `HuggingFaceLocalGenerator` , `AzureOpenAIGenerator`, and others, allow users the ability to pick and choose which language model is to be queried and executed. For components that interface with cloud-based services, the service provider automatically takes care of the details of provisioning the requisite hardware (like GPUs). However, if you wish to use models on your local machine, you’ll need to figure out how to deploy them on your hardware. Further complicating things, different ML libraries have different APIs to launch models on specific devices. +Many Haystack components, such as `TransformersChatGenerator`, `AzureOpenAIChatGenerator`, and others, allow users the ability to pick and choose which language model is to be queried and executed. For components that interface with cloud-based services, the service provider automatically takes care of the details of provisioning the requisite hardware (like GPUs). However, if you wish to use models on your local machine, you’ll need to figure out how to deploy them on your hardware. Further complicating things, different ML libraries have different APIs to launch models on specific devices. To make the process of running inference on local models as straightforward as possible, Haystack uses a framework-agnostic device management implementation. Exposing devices through this interface means you no longer need to worry about library-specific invocations and device representations. @@ -35,17 +35,23 @@ To use a single device for inference, use either the `ComponentDevice.from_singl ```python from haystack.utils import ComponentDevice, Device +from haystack_integrations.components.generators.transformers import ( + TransformersChatGenerator, +) device = ComponentDevice.from_single(Device.gpu(id=1)) # Alternatively, use a PyTorch device string device = ComponentDevice.from_str("cuda:1") -generator = HuggingFaceLocalGenerator(model="llama2", device=device) +generator = TransformersChatGenerator(model="Qwen/Qwen3-0.6B", device=device) ``` To use multiple devices, use the `ComponentDevice.from_multiple` class method: ```python from haystack.utils import ComponentDevice, Device, DeviceMap +from haystack_integrations.components.generators.transformers import ( + TransformersChatGenerator, +) device_map = DeviceMap( { @@ -56,7 +62,7 @@ device_map = DeviceMap( }, ) device = ComponentDevice.from_multiple(device_map) -generator = HuggingFaceLocalGenerator(model="llama2", device=device) +generator = TransformersChatGenerator(model="Qwen/Qwen3-0.6B", device=device) ``` ### Integrating Devices in Custom Components @@ -115,13 +121,17 @@ c = MyComponent(device=ComponentDevice.from_multiple(DeviceMap({ }))) ``` -If the component’s backend provides a more specialized API to manage devices, it could add an additional init parameter that acts as a conduit. For instance, `HuggingFaceLocalGenerator` exposes a `huggingface_pipeline_kwargs` parameter through which Hugging Face-specific `device_map` arguments can be passed: +If the component’s backend provides a more specialized API to manage devices, it could add an additional init parameter that acts as a conduit. For instance, `TransformersChatGenerator` exposes a `huggingface_pipeline_kwargs` parameter through which Hugging Face-specific `device_map` arguments can be passed: ```python -generator = HuggingFaceLocalGenerator( - model="llama2", +from haystack_integrations.components.generators.transformers import ( + TransformersChatGenerator, +) + +generator = TransformersChatGenerator( + model="Qwen/Qwen3-0.6B", huggingface_pipeline_kwargs={"device_map": "balanced"}, ) ``` -In such cases, ensure that the parameter precedence and selection behavior is clearly documented. In the case of `HuggingFaceLocalGenerator`, the device map passed through the `huggingface_pipeline_kwargs` parameter overrides the explicit `device` parameter and is documented as such. +In such cases, ensure that the parameter precedence and selection behavior is clearly documented. In the case of `TransformersChatGenerator`, the device map passed through the `huggingface_pipeline_kwargs` parameter overrides the explicit `device` parameter and is documented as such. diff --git a/docs-website/docs/development/enabling-gpu-acceleration.mdx b/docs-website/docs/development/enabling-gpu-acceleration.mdx index 80e4efea0eb..9dfc2920c0f 100644 --- a/docs-website/docs/development/enabling-gpu-acceleration.mdx +++ b/docs-website/docs/development/enabling-gpu-acceleration.mdx @@ -15,7 +15,7 @@ The Transformer models used in Haystack are designed to be run on GPU-accelerate Once you have GPU enabled on your machine, you can set the `device` on which a given model for a component is loaded. -For example, to load a model for the `HuggingFaceLocalGenerator`, set `device="ComponentDevice.from_single(Device.gpu(id=0))` or `device = ComponentDevice.from_str("cuda:0")` when initializing. +For example, to load a model for the `TransformersChatGenerator`, set `device=ComponentDevice.from_single(Device.gpu(id=0))` or `device = ComponentDevice.from_str("cuda:0")` when initializing. You can find more information on the [Device management](../concepts/device-management.mdx) page. diff --git a/docs-website/docs/pipeline-components/builders/promptbuilder.mdx b/docs-website/docs/pipeline-components/builders/promptbuilder.mdx index e9f64bcccc0..a595b93cf60 100644 --- a/docs-website/docs/pipeline-components/builders/promptbuilder.mdx +++ b/docs-website/docs/pipeline-components/builders/promptbuilder.mdx @@ -159,7 +159,7 @@ Below is an example of a RAG pipeline where we use a `PromptBuilder` to render a ```python from haystack import Pipeline, Document from haystack.utils import Secret -from haystack.components.generators import OpenAIGenerator +from haystack.components.generators.chat import OpenAIChatGenerator from haystack.components.builders.prompt_builder import PromptBuilder # in a real world use case documents could come from a retriever, web, or any other source @@ -179,7 +179,7 @@ prompt_template = """ p = Pipeline() p.add_component(instance=PromptBuilder(template=prompt_template), name="prompt_builder") p.add_component( - instance=OpenAIGenerator(api_key=Secret.from_env_var("OPENAI_API_KEY")), + instance=OpenAIChatGenerator(api_key=Secret.from_env_var("OPENAI_API_KEY")), name="llm", ) p.connect("prompt_builder", "llm") @@ -278,20 +278,21 @@ components: model: gpt-5-mini organization: null streaming_callback: null - system_prompt: null timeout: null - type: haystack.components.generators.openai.OpenAIGenerator + tools: null + tools_strict: false + type: haystack.components.generators.chat.openai.OpenAIChatGenerator prompt_builder: init_parameters: - required_variables: null - template: "\n Given these documents, answer the question.\\nDocuments:\n\ - \ {% for doc in documents %}\n {{ doc.content }}\n {% endfor\ - \ %}\n\n \\nQuestion: {{query}}\n \\nAnswer:\n " + required_variables: '*' + template: "\n Given these documents, answer the question.\nDocuments:\n \ + \ {% for doc in documents %}\n {{ doc.content }}\n {% endfor %}\n\ + \n \nQuestion: {{query}}\n \nAnswer:\n " variables: null type: haystack.components.builders.prompt_builder.PromptBuilder connection_type_validation: true connections: -- receiver: llm.prompt +- receiver: llm.messages sender: prompt_builder.prompt max_runs_per_component: 100 metadata: {} diff --git a/docs-website/docs/pipeline-components/generators.mdx b/docs-website/docs/pipeline-components/generators.mdx index fa7d5edb50c..04270c9654e 100644 --- a/docs-website/docs/pipeline-components/generators.mdx +++ b/docs-website/docs/pipeline-components/generators.mdx @@ -19,7 +19,6 @@ Generators are responsible for generating text after you give them a prompt. The | [AnthropicVertexChatGenerator](generators/anthropicvertexchatgenerator.mdx) | This component enables chat completions using AnthropicVertex API. | ✅ | | [AnthropicGenerator](generators/anthropicgenerator.mdx) | This component enables text completions using Anthropic large language models (LLMs). | ✅ | | [AzureOpenAIChatGenerator](generators/azureopenaichatgenerator.mdx) | Enables chat completion using OpenAI's LLMs through Azure services. | ✅ | -| [AzureOpenAIGenerator](generators/azureopenaigenerator.mdx) | Enables text generation using OpenAI's LLMs through Azure services. | ✅ | | [AzureOpenAIResponsesChatGenerator](generators/azureopenairesponseschatgenerator.mdx) | Enables chat completion using OpenAI's Responses API through Azure services with support for reasoning models. | ✅ | | [CohereChatGenerator](generators/coherechatgenerator.mdx) | Enables chat completion using Cohere's LLMs. | ✅ | | [CohereGenerator](generators/coheregenerator.mdx) | Queries the LLM using Cohere API. | ✅ | @@ -29,9 +28,7 @@ Generators are responsible for generating text after you give them a prompt. The | [GoogleAIGeminiGenerator](generators/googleaigeminigenerator.mdx) | Enables text generation using Google Gemini models. **_This integration will be deprecated soon. We recommend using [GoogleGenAIChatGenerator](generators/googlegenaichatgenerator.mdx) integration instead._** | ✅ | | [GoogleGenAIChatGenerator](generators/googlegenaichatgenerator.mdx) | Enables chat completion using Google Gemini models through Google Gen AI SDK. | ✅ | | [HuggingFaceAPIChatGenerator](generators/huggingfaceapichatgenerator.mdx) | Enables chat completion using various Hugging Face APIs. | ✅ | -| [HuggingFaceAPIGenerator](generators/huggingfaceapigenerator.mdx) | Enables text generation using various Hugging Face APIs. | ✅ | | [HuggingFaceLocalChatGenerator](generators/huggingfacelocalchatgenerator.mdx) | Provides an interface for chat completion using a Hugging Face model that runs locally. | ✅ | -| [HuggingFaceLocalGenerator](generators/huggingfacelocalgenerator.mdx) | Provides an interface to generate text using a Hugging Face model that runs locally. | ✅ | | [LiteLLMChatGenerator](generators/litellmchatgenerator.mdx) | Enables chat completion using various LLM providers through LiteLLM. | ✅ | | [LlamaCppChatGenerator](generators/llamacppchatgenerator.mdx) | Enables chat completion using an LLM running on Llama.cpp. | ❌ | | [LlamaCppGenerator](generators/llamacppgenerator.mdx) | Generate text using an LLM running with Llama.cpp. | ❌ | @@ -43,7 +40,6 @@ Generators are responsible for generating text after you give them a prompt. The | [OllamaChatGenerator](generators/ollamachatgenerator.mdx) | Enables chat completion using an LLM running on Ollama. | ✅ | | [OllamaGenerator](generators/ollamagenerator.mdx) | Provides an interface to generate text using an LLM running on Ollama. | ✅ | | [OpenAIChatGenerator](generators/openaichatgenerator.mdx) | Enables chat completion using OpenAI's large language models (LLMs). | ✅ | -| [OpenAIGenerator](generators/openaigenerator.mdx) | Enables text generation using OpenAI's large language models (LLMs). | ✅ | | [OpenAIImageGenerator](generators/openaiimagegenerator.mdx) | Generate images using OpenAI's image generation models such as `gpt-image-2`. | ❌ | | [OpenAIResponsesChatGenerator](generators/openairesponseschatgenerator.mdx) | Enables chat completion using OpenAI's Responses API with support for reasoning models. | ✅ | | [OpenRouterChatGenerator](generators/openrouterchatgenerator.mdx) | Enables chat completion with any model hosted on OpenRouter. | ✅ | diff --git a/docs-website/docs/pipeline-components/generators/azureopenaigenerator.mdx b/docs-website/docs/pipeline-components/generators/azureopenaigenerator.mdx deleted file mode 100644 index 70609181419..00000000000 --- a/docs-website/docs/pipeline-components/generators/azureopenaigenerator.mdx +++ /dev/null @@ -1,142 +0,0 @@ ---- -title: "AzureOpenAIGenerator" -id: azureopenaigenerator -slug: "/azureopenaigenerator" -description: "This component enables text generation using OpenAI's large language models (LLMs) through Azure services." ---- - -# AzureOpenAIGenerator - -This component enables text generation using OpenAI's large language models (LLMs) through Azure services. - -
- -| | | -| --- | --- | -| **Most common position in a pipeline** | After a [`PromptBuilder`](../builders/promptbuilder.mdx) | -| **Mandatory init variables** | `api_key`: The Azure OpenAI API key. Can be set with `AZURE_OPENAI_API_KEY` env var.

`azure_ad_token`: Microsoft Entra ID token. Can be set with `AZURE_OPENAI_AD_TOKEN` env var. | -| **Mandatory run variables** | `prompt`: A string containing the prompt for the LLM | -| **Output variables** | `replies`: A list of strings with all the replies generated by the LLM

`meta`: A list of dictionaries with the metadata associated with each reply, such as token count, finish reason, and so on | -| **API reference** | [Generators](/reference/generators-api) | -| **GitHub link** | https://github.com/deepset-ai/haystack/blob/main/haystack/components/generators/azure.py | -| **Package name** | `haystack-ai` | - -
- -## Overview - -`AzureOpenAIGenerator` supports OpenAI models deployed through Azure services. To see the list of supported models, head over to Azure [documentation](https://learn.microsoft.com/en-us/azure/ai-services/openai/concepts/models?source=recommendations). The default model used with the component is `gpt-4o-mini`. - -To work with Azure components, you will need an Azure OpenAI API key, as well as an Azure OpenAI Endpoint. You can learn more about them in Azure [documentation](https://learn.microsoft.com/en-us/azure/ai-services/openai/reference). - -The component uses `AZURE_OPENAI_API_KEY` and `AZURE_OPENAI_AD_TOKEN` environment variables by default. Otherwise, you can pass `api_key` and `azure_ad_token` at initialization: - -```python -client = AzureOpenAIGenerator( - azure_endpoint="", - api_key=Secret.from_token(""), - azure_deployment="", -) -``` - -:::info -We recommend using environment variables instead of initialization parameters. -::: - -Then, the component needs a prompt to operate, but you can pass any text generation parameters valid for the `openai.ChatCompletion.create` method directly to this component using the `generation_kwargs` parameter, both at initialization and to `run()` method. For more details on the supported parameters, refer to the [Azure documentation](https://learn.microsoft.com/en-us/azure/ai-services/openai/reference). - -You can also specify a model for this component through the `azure_deployment` init parameter. - -### Streaming - -`AzureOpenAIGenerator` supports streaming the tokens from the LLM directly in output. To do so, pass a function to the `streaming_callback` init parameter. Note that streaming the tokens is only compatible with generating a single response, so `n` must be set to 1 for streaming to work. - -:::info -This component is designed for text generation, not for chat. If you want to use LLMs for chat, use [`AzureOpenAIChatGenerator`](azureopenaichatgenerator.mdx) instead. -::: - -## Usage - -### On its own - -Basic usage: - -```python -from haystack.components.generators import AzureOpenAIGenerator -client = AzureOpenAIGenerator() -response = client.run("What's Natural Language Processing? Be brief.") -print(response) - ->> {'replies': ['Natural Language Processing (NLP) is a branch of artificial intelligence that focuses on ->> the interaction between computers and human language. It involves enabling computers to understand, interpret, ->> and respond to natural human language in a way that is both meaningful and useful.'], 'meta': [{'model': ->> 'gpt-4o-mini', 'index': 0, 'finish_reason': 'stop', 'usage': {'prompt_tokens': 16, ->> 'completion_tokens': 49, 'total_tokens': 65}}]} - -``` - -With streaming: - -```python -from haystack.components.generators import AzureOpenAIGenerator - -client = AzureOpenAIGenerator(streaming_callback=lambda chunk: print(chunk.content, end="", flush=True)) -response = client.run("What's Natural Language Processing? Be brief.") -print(response) - ->>> Natural Language Processing (NLP) is a branch of artificial - intelligence that focuses on the interaction between computers and human - language. It involves enabling computers to understand, interpret,and respond - to natural human language in a way that is both meaningful and useful. ->>> {'replies': ['Natural Language Processing (NLP) is a branch of artificial - intelligence that focuses on the interaction between computers and human - language. It involves enabling computers to understand, interpret,and respond - to natural human language in a way that is both meaningful and useful.'], - 'meta': [{'model': 'gpt-4o-mini', 'index': 0, 'finish_reason': - 'stop', 'usage': {'prompt_tokens': 16, 'completion_tokens': 49, - 'total_tokens': 65}}]} - -``` - -### In a Pipeline - -```python -from haystack import Pipeline -from haystack.components.retrievers.in_memory import InMemoryBM25Retriever -from haystack.components.builders.prompt_builder import PromptBuilder -from haystack.components.generators import AzureOpenAIGenerator -from haystack.document_stores.in_memory import InMemoryDocumentStore -from haystack import Document - -docstore = InMemoryDocumentStore() -docstore.write_documents( - [ - Document(content="Rome is the capital of Italy"), - Document(content="Paris is the capital of France"), - ], -) - -query = "What is the capital of France?" - -template = """ -Given the following information, answer the question. - -Context: -{% for document in documents %} - {{ document.content }} -{% endfor %} - -Question: {{ query }}? -""" -pipe = Pipeline() - -pipe.add_component("retriever", InMemoryBM25Retriever(document_store=docstore)) -pipe.add_component("prompt_builder", PromptBuilder(template=template)) -pipe.add_component("llm", AzureOpenAIGenerator()) -pipe.connect("retriever", "prompt_builder.documents") -pipe.connect("prompt_builder", "llm") - -res = pipe.run({"prompt_builder": {"query": query}, "retriever": {"query": query}}) - -print(res) -``` diff --git a/docs-website/docs/pipeline-components/generators/huggingfaceapichatgenerator.mdx b/docs-website/docs/pipeline-components/generators/huggingfaceapichatgenerator.mdx index 343a7f7bf38..11719f2e291 100644 --- a/docs-website/docs/pipeline-components/generators/huggingfaceapichatgenerator.mdx +++ b/docs-website/docs/pipeline-components/generators/huggingfaceapichatgenerator.mdx @@ -33,10 +33,6 @@ This generator enables chat completion using various Hugging Face APIs. This component's main input is a list of `ChatMessage` objects. `ChatMessage` is a data class that contains a message, a role (who generated the message, such as `user`, `assistant`, `system`, `function`), and optional metadata. If a string is passed, it is converted into a list containing a single `ChatMessage` with the `user` role. For more information, check out our [`ChatMessage` docs](../../concepts/data-classes/chatmessage.mdx). -:::info -This component is designed for chat completion. If you want to use Hugging Face APIs for simple text generation (such as translation or summarization tasks) or don't want to use the `ChatMessage` object, use [`HuggingFaceAPIGenerator`](huggingfaceapigenerator.mdx) instead. -::: - The component uses a `HF_API_TOKEN` environment variable by default. Otherwise, you can pass a Hugging Face API token at initialization with `token` – see code examples below. The token is needed: diff --git a/docs-website/docs/pipeline-components/generators/huggingfaceapigenerator.mdx b/docs-website/docs/pipeline-components/generators/huggingfaceapigenerator.mdx deleted file mode 100644 index 7d51f4477ae..00000000000 --- a/docs-website/docs/pipeline-components/generators/huggingfaceapigenerator.mdx +++ /dev/null @@ -1,191 +0,0 @@ ---- -title: "HuggingFaceAPIGenerator" -id: huggingfaceapigenerator -slug: "/huggingfaceapigenerator" -description: "This generator enables text generation using various Hugging Face APIs." ---- - -# HuggingFaceAPIGenerator - -This generator enables text generation using various Hugging Face APIs. - -
- -| | | -| --- | --- | -| **Most common position in a pipeline** | After a [`PromptBuilder`](../builders/promptbuilder.mdx) | -| **Mandatory init variables** | `api_type`: The type of Hugging Face API to use

`api_params`: A dictionary with one of the following keys:

- `model`: Hugging Face model ID. Required when `api_type` is `SERVERLESS_INFERENCE_API`.**OR** - `url`: URL of the inference endpoint. Required when `api_type` is `INFERENCE_ENDPOINTS` or `TEXT_EMBEDDINGS_INFERENCE`.`token`: The Hugging Face API token. Can be set with `HF_API_TOKEN` or `HF_TOKEN` env var. | -| **Mandatory run variables** | `prompt`: A string containing the prompt for the LLM | -| **Output variables** | `replies`: A list of strings with all the replies generated by the LLM

`meta`: A list of dictionaries with the metadata associated with each reply, such as token count, finish reason, and others | -| **API reference** | [Generators](/reference/generators-api) | -| **GitHub link** | https://github.com/deepset-ai/haystack/blob/main/haystack/components/generators/hugging_face_api.py | -| **Package name** | `haystack-ai` | - -
- -## Overview - -`HuggingFaceAPIGenerator` can be used to generate text using different Hugging Face APIs: - -- [Paid Inference Endpoints](https://huggingface.co/inference-endpoints) -- [Self-hosted Text Generation Inference](https://github.com/huggingface/text-generation-inference) - -:::note[Important Note] - -As of July 2025, the Hugging Face Inference API no longer offers generative models through the `text_generation` endpoint. Generative models are now only available through providers supporting the `chat_completion` endpoint. As a result, this component might no longer work with the Hugging Face Inference API. - -Use the [`HuggingFaceAPIChatGenerator`](huggingfaceapichatgenerator.mdx) component instead, which supports the `chat_completion` endpoint and works with the free Serverless Inference API. -::: - -:::info -This component is designed for text generation, not for chat. If you want to use these LLMs for chat, use [`HuggingFaceAPIChatGenerator`](huggingfaceapichatgenerator.mdx) instead. -::: - -The component uses a `HF_API_TOKEN` environment variable by default. Otherwise, you can pass a Hugging Face API token at initialization with `token` – see code examples below. -The token is needed when you use the Inference Endpoints. - -### Streaming - -This Generator supports [streaming](guides-to-generators/choosing-the-right-generator.mdx#streaming-support) the tokens from the LLM directly in output. To do so, pass a function to the `streaming_callback` init parameter. - -## Usage - -### On its own - -#### Using Paid Inference Endpoints - -In this case, a private instance of the model is deployed by Hugging Face, and you typically pay per hour. - -To understand how to spin up an Inference Endpoint, visit [Hugging Face documentation](https://huggingface.co/inference-endpoints/dedicated). - -Additionally, in this case, you need to provide your Hugging Face token. -The Generator expects the `url` of your endpoint in `api_params`. - -```python -from haystack.components.generators import HuggingFaceAPIGenerator -from haystack.utils import Secret - -generator = HuggingFaceAPIGenerator( - api_type="inference_endpoints", - api_params={"url": ""}, - token=Secret.from_token(""), -) - -result = generator.run(prompt="What's Natural Language Processing?") -print(result) -``` - -#### Using Self-Hosted Text Generation Inference (TGI) - -[Hugging Face Text Generation Inference](https://github.com/huggingface/text-generation-inference) is a toolkit for efficiently deploying and serving LLMs. - -While it powers the most recent versions of Serverless Inference API and Inference Endpoints, it can be used easily on-premise through Docker. - -For example, you can run a TGI container as follows: - -```shell -model=mistralai/Mistral-7B-v0.1 -volume=$PWD/data # share a volume with the Docker container to avoid downloading weights every run - -docker run --gpus all --shm-size 1g -p 8080:80 -v $volume:/data ghcr.io/huggingface/text-generation-inference:1.4 --model-id $model -``` - -For more information, refer to the [official TGI repository](https://github.com/huggingface/text-generation-inference). - -The Generator expects the `url` of your TGI instance in `api_params`. - -```python -from haystack.components.generators import HuggingFaceAPIGenerator - -generator = HuggingFaceAPIGenerator( - api_type="text_generation_inference", - api_params={"url": "http://localhost:8080"}, -) - -result = generator.run(prompt="What's Natural Language Processing?") -print(result) -``` - -#### Using the Free Serverless Inference API (Not Recommended) - -:::warning -This example might not work as the Hugging Face Inference API no longer offers models that support the `text_generation` endpoint. Use the [`HuggingFaceAPIChatGenerator`](huggingfaceapichatgenerator.mdx) for generative models through the `chat_completion` endpoint. - -::: - -Formerly known as (free) Hugging Face Inference API, this API allows you to quickly experiment with many models hosted on the Hugging Face Hub, offloading the inference to Hugging Face servers. It's rate-limited and not meant for production. - -To use this API, you need a [free Hugging Face token](https://huggingface.co/settings/tokens). -The Generator expects the `model` in `api_params`. - -```python -from haystack.components.generators import HuggingFaceAPIGenerator -from haystack.utils import Secret - -generator = HuggingFaceAPIGenerator( - api_type="serverless_inference_api", - api_params={"model": "HuggingFaceH4/zephyr-7b-beta"}, - token=Secret.from_token(""), -) - -result = generator.run(prompt="What's Natural Language Processing?") -print(result) -``` - -### In a pipeline - -```python -from haystack import Pipeline -from haystack.components.retrievers.in_memory import InMemoryBM25Retriever -from haystack.components.builders.prompt_builder import PromptBuilder -from haystack.components.generators import HuggingFaceAPIGenerator -from haystack.document_stores.in_memory import InMemoryDocumentStore -from haystack import Document - -docstore = InMemoryDocumentStore() -docstore.write_documents( - [ - Document(content="Rome is the capital of Italy"), - Document(content="Paris is the capital of France"), - ], -) - -query = "What is the capital of France?" - -template = """ -Given the following information, answer the question. - -Context: -{% for document in documents %} - {{ document.content }} -{% endfor %} - -Question: {{ query }}? -""" - -generator = HuggingFaceAPIGenerator( - api_type="inference_endpoints", - api_params={"url": ""}, - token=Secret.from_token(""), -) - -pipe = Pipeline() - -pipe.add_component("retriever", InMemoryBM25Retriever(document_store=docstore)) -pipe.add_component("prompt_builder", PromptBuilder(template=template)) -pipe.add_component("llm", generator) -pipe.connect("retriever", "prompt_builder.documents") -pipe.connect("prompt_builder", "llm") - -res = pipe.run({"prompt_builder": {"query": query}, "retriever": {"query": query}}) - -print(res) -``` - -## Additional References - -🧑‍🍳 Cookbooks: - -- [Multilingual RAG from a podcast with Whisper, Qdrant and Mistral](https://haystack.deepset.ai/cookbook/multilingual_rag_podcast) -- [Information Extraction with Raven](https://haystack.deepset.ai/cookbook/information_extraction_raven) -- [Web QA with Mixtral-8x7B-Instruct-v0.1](https://haystack.deepset.ai/cookbook/mixtral-8x7b-for-web-qa) diff --git a/docs-website/docs/pipeline-components/generators/huggingfacelocalchatgenerator.mdx b/docs-website/docs/pipeline-components/generators/huggingfacelocalchatgenerator.mdx index 9301ad65b1c..c0e2cdd9144 100644 --- a/docs-website/docs/pipeline-components/generators/huggingfacelocalchatgenerator.mdx +++ b/docs-website/docs/pipeline-components/generators/huggingfacelocalchatgenerator.mdx @@ -33,10 +33,6 @@ Provides an interface for chat completion using a Hugging Face model that runs l Keep in mind that if LLMs run locally, you may need a powerful machine to run them. This depends strongly on the model you select and its parameter count. -:::info -This component is designed for chat completion, not for text generation. If you want to use Hugging Face LLMs for text generation, use [`HuggingFaceLocalGenerator`](huggingfacelocalgenerator.mdx) instead. -::: - If a string is passed to `messages`, it is converted into a list containing a single `ChatMessage` with the `user` role. For remote file authorization, this component uses a `HF_API_TOKEN` environment variable by default. Otherwise, you can pass a Hugging Face API token at initialization with `token`: diff --git a/docs-website/docs/pipeline-components/generators/huggingfacelocalgenerator.mdx b/docs-website/docs/pipeline-components/generators/huggingfacelocalgenerator.mdx deleted file mode 100644 index 264dd06ec92..00000000000 --- a/docs-website/docs/pipeline-components/generators/huggingfacelocalgenerator.mdx +++ /dev/null @@ -1,124 +0,0 @@ ---- -title: "HuggingFaceLocalGenerator" -id: huggingfacelocalgenerator -slug: "/huggingfacelocalgenerator" -description: "`HuggingFaceLocalGenerator` provides an interface to generate text using a Hugging Face model that runs locally." ---- - -# HuggingFaceLocalGenerator - -`HuggingFaceLocalGenerator` provides an interface to generate text using a Hugging Face model that runs locally. - -
- -| | | -| --- | --- | -| **Most common position in a pipeline** | After a [`PromptBuilder`](../builders/promptbuilder.mdx) | -| **Mandatory init variables** | `token`: The Hugging Face API token. Can be set with `HF_API_TOKEN` or `HF_TOKEN` env var. | -| **Mandatory run variables** | `prompt`: A string containing the prompt for the LLM | -| **Output variables** | `replies`: A list of strings with all the replies generated by the LLM | -| **API reference** | [Generators](/reference/generators-api) | -| **GitHub link** | https://github.com/deepset-ai/haystack/blob/main/haystack/components/generators/hugging_face_local.py | -| **Package name** | `haystack-ai` | - -
- -## Overview - -Keep in mind that if LLMs run locally, you may need a powerful machine to run them. This depends strongly on the model you select and its parameter count. - -:::info[Looking for chat completion?] - -This component is designed for text generation, not for chat. If you want to use Hugging Face LLMs for chat, consider using [`HuggingFaceLocalChatGenerator`](huggingfacelocalchatgenerator.mdx) instead. -::: - -For remote files authorization, this component uses a `HF_API_TOKEN` environment variable by default. Otherwise, you can pass a Hugging Face API token at initialization with `token`: - -```python -local_generator = HuggingFaceLocalGenerator(token=Secret.from_token("")) -``` - -### Streaming - -This Generator supports [streaming](guides-to-generators/choosing-the-right-generator.mdx#streaming-support) the tokens from the LLM directly in output. To do so, pass a function to the `streaming_callback` init parameter. - -## Usage - -### On its own - -```python -from haystack.components.generators import HuggingFaceLocalGenerator - -generator = HuggingFaceLocalGenerator( - model="google/flan-t5-large", - task="text2text-generation", - generation_kwargs={ - "max_new_tokens": 100, - "temperature": 0.9, - }, -) - -print(generator.run("Who is the best American actor?")) -# {'replies': ['john wayne']} -``` - -### In a Pipeline - -```python -from haystack import Pipeline -from haystack.components.retrievers.in_memory import InMemoryBM25Retriever -from haystack.components.builders.prompt_builder import PromptBuilder -from haystack.components.generators import HuggingFaceLocalGenerator -from haystack.document_stores.in_memory import InMemoryDocumentStore -from haystack import Document - -docstore = InMemoryDocumentStore() -docstore.write_documents( - [ - Document(content="Rome is the capital of Italy"), - Document(content="Paris is the capital of France"), - ], -) - -generator = HuggingFaceLocalGenerator( - model="google/flan-t5-large", - task="text2text-generation", - generation_kwargs={ - "max_new_tokens": 100, - "temperature": 0.9, - }, -) - -query = "What is the capital of France?" - -template = """ -Given the following information, answer the question. - -Context: -{% for document in documents %} - {{ document.content }} -{% endfor %} - -Question: {{ query }}? -""" -pipe = Pipeline() - -pipe.add_component("retriever", InMemoryBM25Retriever(document_store=docstore)) -pipe.add_component("prompt_builder", PromptBuilder(template=template)) -pipe.add_component("llm", generator) -pipe.connect("retriever", "prompt_builder.documents") -pipe.connect("prompt_builder", "llm") - -res = pipe.run({"prompt_builder": {"query": query}, "retriever": {"query": query}}) - -print(res) -``` - -## Additional References - -🧑‍🍳 Cookbooks: - -- [Use Zephyr 7B Beta with Hugging Face for RAG](https://haystack.deepset.ai/cookbook/zephyr-7b-beta-for-rag) -- [Information Extraction with Gorilla](https://haystack.deepset.ai/cookbook/information-extraction-gorilla) -- [RAG on the Oscars using Llama 3.1 models](https://haystack.deepset.ai/cookbook/llama3_rag) -- [Agentic RAG with Llama 3.2 3B](https://haystack.deepset.ai/cookbook/llama32_agentic_rag) diff --git a/docs-website/docs/pipeline-components/generators/openaigenerator.mdx b/docs-website/docs/pipeline-components/generators/openaigenerator.mdx deleted file mode 100644 index 0c6cf0e7d0d..00000000000 --- a/docs-website/docs/pipeline-components/generators/openaigenerator.mdx +++ /dev/null @@ -1,198 +0,0 @@ ---- -title: "OpenAIGenerator" -id: openaigenerator -slug: "/openaigenerator" -description: "`OpenAIGenerator` enables text generation using OpenAI's large language models (LLMs)." ---- - -# OpenAIGenerator - -`OpenAIGenerator` enables text generation using OpenAI's large language models (LLMs). - -
- -| | | -| --- | --- | -| **Most common position in a pipeline** | After a [`PromptBuilder`](../builders/promptbuilder.mdx) | -| **Mandatory init variables** | `api_key`: An OpenAI API key. Can be set with `OPENAI_API_KEY` env var. | -| **Mandatory run variables** | `prompt`: A string containing the prompt for the LLM | -| **Output variables** | `replies`: A list of strings with all the replies generated by the LLM

`meta`: A list of dictionaries with the metadata associated with each reply, such as token count, finish reason, and so on | -| **API reference** | [Generators](/reference/generators-api) | -| **GitHub link** | https://github.com/deepset-ai/haystack/blob/main/haystack/components/generators/openai.py | -| **Package name** | `haystack-ai` | - -
- -## Overview - -`OpenAIGenerator` supports OpenAI models starting from gpt-3.5-turbo and later (gpt-4, gpt-4-turbo, and so on). - -`OpenAIGenerator` needs an OpenAI key to work. It uses an `OPENAI_API_KEY` environment variable by default. Otherwise, you can pass an API key at initialization with `api_key`: - -``` -generator = OpenAIGenerator(api_key=Secret.from_token(""), model="gpt-4o-mini") -``` - -Then, the component needs a prompt to operate, but you can pass any text generation parameters valid for the `openai.ChatCompletion.create` method directly to this component using the `generation_kwargs` parameter, both at initialization and to `run()` method. For more details on the parameters supported by the OpenAI API, refer to the [OpenAI documentation](https://platform.openai.com/docs/api-reference/chat). - -`OpenAIGenerator` supports custom deployments of your OpenAI models through the `api_base_url` init parameter. - -### Streaming - -`OpenAIGenerator` supports streaming the tokens from the LLM directly in output. To do so, pass a function to the `streaming_callback` init parameter. Note that streaming the tokens is only compatible with generating a single response, so `n` must be set to 1 for streaming to work. - -:::info -This component is designed for text generation, not for chat. If you want to use OpenAI LLMs for chat, use [`OpenAIChatGenerator`](openaichatgenerator.mdx) instead. -::: - -## Usage - -### On its own - -Basic usage: - -```python -from haystack.components.generators import OpenAIGenerator -from haystack.utils import Secret - -client = OpenAIGenerator(model="gpt-4o-mini", api_key=Secret.from_token("")) -response = client.run("What's Natural Language Processing? Be brief.") -print(response) - ->>> {'replies': ['Natural Language Processing, often abbreviated as NLP, is a field - of artificial intelligence that focuses on the interaction between computers - and humans through natural language. The primary aim of NLP is to enable - computers to understand, interpret, and generate human language in a valuable way.'], - 'meta': [{'model': 'gpt-4o-mini', 'index': 0, 'finish_reason': - 'stop', 'usage': {'prompt_tokens': 16, 'completion_tokens': 53, - 'total_tokens': 69}}]} -``` - -With streaming: - -```python -from haystack.components.generators import OpenAIGenerator -from haystack.utils import Secret - -client = OpenAIGenerator(streaming_callback=lambda chunk: print(chunk.content, end="", flush=True)) -response = client.run("What's Natural Language Processing? Be brief.") -print(response) - ->>> Natural Language Processing (NLP) is a branch of artificial - intelligence that focuses on the interaction between computers and human - language. It involves enabling computers to understand, interpret,and respond - to natural human language in a way that is both meaningful and useful. ->>> {'replies': ['Natural Language Processing (NLP) is a branch of artificial - intelligence that focuses on the interaction between computers and human - language. It involves enabling computers to understand, interpret,and respond - to natural human language in a way that is both meaningful and useful.'], - 'meta': [{'model': 'gpt-4o-mini', 'index': 0, 'finish_reason': - 'stop', 'usage': {'prompt_tokens': 16, 'completion_tokens': 49, - 'total_tokens': 65}}]} -``` - -### In a Pipeline - -Here's an example of RAG Pipeline: - -```python -from haystack import Pipeline -from haystack.components.retrievers.in_memory import InMemoryBM25Retriever -from haystack.components.builders.prompt_builder import PromptBuilder -from haystack.components.generators import OpenAIGenerator -from haystack.document_stores.in_memory import InMemoryDocumentStore -from haystack import Document -from haystack.utils import Secret - -docstore = InMemoryDocumentStore() -docstore.write_documents( - [ - Document(content="Rome is the capital of Italy"), - Document(content="Paris is the capital of France"), - ], -) - -query = "What is the capital of France?" - -template = """ -Given the following information, answer the question. - -Context: -{% for document in documents %} - {{ document.content }} -{% endfor %} - -Question: {{ query }}? -""" -pipe = Pipeline() - -pipe.add_component("retriever", InMemoryBM25Retriever(document_store=docstore)) -pipe.add_component("prompt_builder", PromptBuilder(template=template)) -pipe.add_component( - "llm", - OpenAIGenerator(api_key=Secret.from_env_var("OPENAI_API_KEY")), -) -pipe.connect("retriever", "prompt_builder.documents") -pipe.connect("prompt_builder", "llm") - -res = pipe.run({"prompt_builder": {"query": query}, "retriever": {"query": query}}) - -print(res) -``` - -### In YAML - -This is the YAML representation of the RAG pipeline shown above. It retrieves documents based on a query, constructs a prompt using a template, and generates an answer using a chat model. - -```yaml -components: - llm: - init_parameters: - api_base_url: null - api_key: - env_vars: - - OPENAI_API_KEY - strict: true - type: env_var - generation_kwargs: {} - http_client_kwargs: null - max_retries: null - model: gpt-5-mini - organization: null - streaming_callback: null - system_prompt: null - timeout: null - type: haystack.components.generators.openai.OpenAIGenerator - prompt_builder: - init_parameters: - required_variables: null - template: "\nGiven the following information, answer the question.\n\nContext:\n\ - {% for document in documents %}\n {{ document.content }}\n{% endfor %}\n\n\ - Question: {{ query }}?\n" - variables: null - type: haystack.components.builders.prompt_builder.PromptBuilder - retriever: - init_parameters: - document_store: - init_parameters: - bm25_algorithm: BM25L - bm25_parameters: {} - bm25_tokenization_regex: (?u)\b\w+\b - embedding_similarity_function: dot_product - index: 64e4f9ab-87fb-47fd-b390-dabcfda61447 - return_embedding: true - type: haystack.document_stores.in_memory.document_store.InMemoryDocumentStore - filter_policy: replace - filters: null - scale_score: false - top_k: 10 - type: haystack.components.retrievers.in_memory.bm25_retriever.InMemoryBM25Retriever -connection_type_validation: true -connections: -- receiver: prompt_builder.documents - sender: retriever.documents -- receiver: llm.prompt - sender: prompt_builder.prompt -max_runs_per_component: 100 -metadata: {} -``` diff --git a/docs-website/docs/pipeline-components/rankers.mdx b/docs-website/docs/pipeline-components/rankers.mdx index 40d1c621aaa..722cf05c572 100644 --- a/docs-website/docs/pipeline-components/rankers.mdx +++ b/docs-website/docs/pipeline-components/rankers.mdx @@ -23,7 +23,6 @@ Rankers are a group of components that order documents by given criteria. Their | [MetaFieldGroupingRanker](rankers/metafieldgroupingranker.mdx) | Reorders the documents by grouping them based on metadata keys. | | [NvidiaRanker](rankers/nvidiaranker.mdx) | Ranks documents using large-language models from [NVIDIA NIMs](https://ai.nvidia.com) . | | [PyversityRanker](rankers/pyversityranker.mdx) | Reranks documents by balancing relevance and diversity using pyversity's diversification algorithms. | -| [TransformersSimilarityRanker](rankers/transformerssimilarityranker.mdx) | A legacy version of [SentenceTransformersSimilarityRanker](rankers/sentencetransformerssimilarityranker.mdx). | | [SentenceTransformersDiversityRanker](rankers/sentencetransformersdiversityranker.mdx) | A Diversity Ranker based on Sentence Transformers. | | [SentenceTransformersSimilarityRanker](rankers/sentencetransformerssimilarityranker.mdx) | A model-based Ranker that orders documents based on their relevance to the query. It uses a cross-encoder model to produce query and document embeddings. It then compares the similarity of the query embedding to the document embeddings to produce a ranking with the most similar documents appearing first.

It's a powerful Ranker that takes word order and syntax into account. You can use it to improve the initial ranking done by a weaker Retriever, but it's also more expensive computationally than the Rankers that don't use models. | | [VLLMRanker](rankers/vllmranker.mdx) | Ranks documents based on their similarity to the query using reranker models served with vLLM. | diff --git a/docs-website/docs/pipeline-components/rankers/choosing-the-right-ranker.mdx b/docs-website/docs/pipeline-components/rankers/choosing-the-right-ranker.mdx index 1cb27b94926..e9d02dcd25c 100644 --- a/docs-website/docs/pipeline-components/rankers/choosing-the-right-ranker.mdx +++ b/docs-website/docs/pipeline-components/rankers/choosing-the-right-ranker.mdx @@ -38,7 +38,6 @@ These Rankers run entirely on your local infrastructure. They are ideal for team All on-premise Rankers in Haystack use cross-encoder architectures. These models jointly process the query and each document to assess relevance with deep contextual awareness. For example: - [SentenceTransformersSimilarityRanker](sentencetransformerssimilarityranker.mdx) ranks documents based on semantic similarity to the query. In addition to the default PyTorch backend (optimal for GPU), it also offers other memory-efficient options which are suitable for CPU-only cases: ONNX and OpenVINO. -- [TransformersSimilarityRanker](transformerssimilarityranker.mdx) is its legacy predecessor and should generally be avoided in favor of the newer, more flexible SentenceTransformersSimilarityRanker. - [HuggingFaceTEIRanker](huggingfaceteiranker.mdx) is based on the Text Embeddings Inference project: whether you have GPU resources or not, it offers high-performance for serving the models locally. In addition, you can also use this component to perform inference with reranking models hosted on Hugging Face Inference Endpoints. - [FastembedRanker](fastembedranker.mdx) supports a variety of cross-encoder models and is optimal for CPU-only environments. - [SentenceTransformersDiversityRanker](sentencetransformersdiversityranker.mdx) reorders documents to maximize diversity, helping reduce redundancy and cover a broader range of relevant topics. @@ -57,4 +56,4 @@ For example: The **MetaFieldRanker** Ranker is typically used _before_ semantic ranking to filter or restructure documents according to business logic. -In contrast, **LostInTheMiddleRanker and MetaFieldGroupingRanker** are intended for use _after_ ranking, to improve the effectiveness of downstream components like LLMs. These deterministic approaches provide speed, transparency, and fine-grained control, making them well-suited for pipelines requiring explainability or strict operational logic. \ No newline at end of file +In contrast, **LostInTheMiddleRanker and MetaFieldGroupingRanker** are intended for use _after_ ranking, to improve the effectiveness of downstream components like LLMs. These deterministic approaches provide speed, transparency, and fine-grained control, making them well-suited for pipelines requiring explainability or strict operational logic. diff --git a/docs-website/docs/pipeline-components/rankers/transformerssimilarityranker.mdx b/docs-website/docs/pipeline-components/rankers/transformerssimilarityranker.mdx deleted file mode 100644 index 3eb3a95ad10..00000000000 --- a/docs-website/docs/pipeline-components/rankers/transformerssimilarityranker.mdx +++ /dev/null @@ -1,114 +0,0 @@ ---- -title: "TransformersSimilarityRanker" -id: transformerssimilarityranker -slug: "/transformerssimilarityranker" -description: "Use this component to rank documents based on their similarity to the query. The `TransformersSimilarityRanker` is a powerful, model-based Ranker that uses a cross-encoder model to produce document and query embeddings." ---- - -# TransformersSimilarityRanker - -Use this component to rank documents based on their similarity to the query. The `TransformersSimilarityRanker` is a powerful, model-based Ranker that uses a cross-encoder model to produce document and query embeddings. - -:::warning[Legacy Component] - -This component is considered legacy and will no longer receive updates. It may be deprecated in a future release, followed by removal after a deprecation period. -Consider using SentenceTransformersSimilarityRanker instead, as it provides the same functionality and additional features. -::: - -
- -| | | -| --- | --- | -| **Most common position in a pipeline** | In a query pipeline, after a component that returns a list of documents such as a [Retriever](../retrievers.mdx) | -| **Mandatory init variables** | `token` (only for private models): The Hugging Face API token. Can be set with `HF_API_TOKEN` or `HF_TOKEN` env var. | -| **Mandatory run variables** | `documents`: A list of documents

`query`: A query string | -| **Output variables** | `documents`: A list of documents | -| **API reference** | [Rankers](/reference/rankers-api) | -| **GitHub link** | https://github.com/deepset-ai/haystack/blob/main/haystack/components/rankers/transformers_similarity.py | -| **Package name** | `haystack-ai` | - -
- -## Overview - -`TransformersSimilarityRanker` ranks documents based on how similar they are to the query. It uses a pre-trained cross-encoder model from the Hugging Face Hub to embed both the query and the documents. It then compares the embeddings to determine how similar they are. The result is a list of `Document `objects in ranked order, with the Documents most similar to the query appearing first. - -`TransformersSimilarityRanker` is most useful in query pipelines, such as a retrieval-augmented generation (RAG) pipeline or a document search pipeline, to ensure the retrieved documents are ordered by relevance. You can use it after a Retriever (such as the `InMemoryEmbeddingRetriever`) to improve the search results. When using `TransformersSimilarityRanker` with a Retriever, consider setting the Retriever's `top_k` to a small number. This way, the Ranker will have fewer documents to process, which can help make your pipeline faster. - -By default, this component uses the `cross-encoder/ms-marco-MiniLM-L-6-v2` model, but it's flexible. You can switch to a different model by adjusting the `model` parameter when initializing the Ranker. For details on different initialization settings, check out the API reference for this component. - -You can also set the `device` parameter to use HF models on your CPU or GPU. - -### Authorization - -The component uses a `HF_API_TOKEN` environment variable by default. Otherwise, you can pass a Hugging Face API token at initialization with `token` – see code examples below. - -```python -ranker = TransformersSimilarityRanker(token=Secret.from_token("")) -``` - -## Usage - -### On its own - -You can use `TransformersSimilarityRanker` outside of a pipeline to order documents based on your query. - -This example uses the `TransformersSimilarityRanker` to rank two simple documents. To run the Ranker, pass a query, provide the documents, and set the number of documents to return in the `top_k` parameter. - -```python -from haystack import Document -from haystack.components.rankers import TransformersSimilarityRanker - -docs = [Document(content="Paris"), Document(content="Berlin")] - -ranker = TransformersSimilarityRanker() - -ranker.run(query="City in France", documents=docs, top_k=1) -``` - -### In a pipeline - -`TransformersSimilarityRanker` is most efficient in query pipelines when used after a Retriever. - -Below is an example of a pipeline that retrieves documents from an `InMemoryDocumentStore` based on keyword search (using `InMemoryBM25Retriever`). It then uses the `TransformersSimilarityRanker` to rank the retrieved documents according to their similarity to the query. The pipeline uses the default settings of the Ranker. - -```python -from haystack import Document, Pipeline -from haystack.document_stores.in_memory import InMemoryDocumentStore -from haystack.components.retrievers.in_memory import InMemoryBM25Retriever -from haystack.components.rankers import TransformersSimilarityRanker - -docs = [ - Document(content="Paris is in France"), - Document(content="Berlin is in Germany"), - Document(content="Lyon is in France"), -] -document_store = InMemoryDocumentStore() -document_store.write_documents(docs) - -retriever = InMemoryBM25Retriever(document_store=document_store) -ranker = TransformersSimilarityRanker() - -document_ranker_pipeline = Pipeline() -document_ranker_pipeline.add_component(instance=retriever, name="retriever") -document_ranker_pipeline.add_component(instance=ranker, name="ranker") - -document_ranker_pipeline.connect("retriever.documents", "ranker.documents") - -query = "Cities in France" -document_ranker_pipeline.run( - data={ - "retriever": {"query": query, "top_k": 3}, - "ranker": {"query": query, "top_k": 2}, - }, -) -``` - -:::note[Ranker `top_k`] - -In the example above, the `top_k` values for the Retriever and the Ranker are different. The Retriever's `top_k` specifies how many documents it returns. The Ranker then orders these documents. - -You can set the same or a smaller `top_k` value for the Ranker. The Ranker's `top_k` is the number of documents it returns (if it's the last component in the pipeline) or forwards to the next component. In the pipeline example above, the Ranker is the last component, so the output you get when you run the pipeline are the top two documents, as per the Ranker's `top_k`. - -Adjusting the `top_k` values can help you optimize performance. In this case, a smaller `top_k` value of the Retriever means fewer documents to process for the Ranker, which can speed up the pipeline. -::: diff --git a/docs-website/sidebars.js b/docs-website/sidebars.js index bf6cbd3a6d4..61989a5e85e 100644 --- a/docs-website/sidebars.js +++ b/docs-website/sidebars.js @@ -421,7 +421,6 @@ export default { 'pipeline-components/generators/anthropicvertexchatgenerator', 'pipeline-components/generators/azureopenaichatgenerator', 'pipeline-components/generators/azureopenairesponseschatgenerator', - 'pipeline-components/generators/azureopenaigenerator', 'pipeline-components/generators/coherechatgenerator', 'pipeline-components/generators/coheregenerator', 'pipeline-components/generators/cometapichatgenerator', @@ -430,9 +429,7 @@ export default { 'pipeline-components/generators/googleaigeminigenerator', 'pipeline-components/generators/googlegenaichatgenerator', 'pipeline-components/generators/huggingfaceapichatgenerator', - 'pipeline-components/generators/huggingfaceapigenerator', 'pipeline-components/generators/huggingfacelocalchatgenerator', - 'pipeline-components/generators/huggingfacelocalgenerator', 'pipeline-components/generators/litellmchatgenerator', 'pipeline-components/generators/llamacppchatgenerator', 'pipeline-components/generators/llamacppgenerator', @@ -445,7 +442,6 @@ export default { 'pipeline-components/generators/ollamagenerator', 'pipeline-components/generators/openaichatgenerator', 'pipeline-components/generators/openairesponseschatgenerator', - 'pipeline-components/generators/openaigenerator', 'pipeline-components/generators/openaiimagegenerator', 'pipeline-components/generators/openrouterchatgenerator', 'pipeline-components/generators/perplexitychatgenerator', @@ -540,7 +536,6 @@ export default { 'pipeline-components/rankers/pyversityranker', 'pipeline-components/rankers/sentencetransformersdiversityranker', 'pipeline-components/rankers/sentencetransformerssimilarityranker', - 'pipeline-components/rankers/transformerssimilarityranker', 'pipeline-components/rankers/vllmranker', 'pipeline-components/rankers/external-integrations-rankers', ], From 7b36b244a662d4048f9f9f8134f730d8713f107a Mon Sep 17 00:00:00 2001 From: anakin87 Date: Thu, 9 Jul 2026 11:08:47 +0200 Subject: [PATCH 2/4] remove migrated components --- docs-website/docs/overview/migration.mdx | 2 +- .../docs/pipeline-components/extractors.mdx | 1 - .../extractors/namedentityextractor.mdx | 109 ------------------ .../docs/pipeline-components/generators.mdx | 2 +- .../choosing-the-right-generator.mdx | 2 +- .../huggingfacelocalchatgenerator.mdx | 97 ---------------- docs-website/sidebars.js | 2 - 7 files changed, 3 insertions(+), 212 deletions(-) delete mode 100644 docs-website/docs/pipeline-components/extractors/namedentityextractor.mdx delete mode 100644 docs-website/docs/pipeline-components/generators/huggingfacelocalchatgenerator.mdx diff --git a/docs-website/docs/overview/migration.mdx b/docs-website/docs/overview/migration.mdx index 92fdffc6476..4b12c21c358 100644 --- a/docs-website/docs/overview/migration.mdx +++ b/docs-website/docs/overview/migration.mdx @@ -159,7 +159,7 @@ If you need help migrating a 1.x node without a 2.x counterpart, open an [issue] | Crawler | Scrapes text from websites. **Example usage:** To run searches on your website content. | Not Available | | DocumentClassifier | Classifies documents by attaching metadata to them. **Example usage:** Labeling documents by their characteristic (for example, sentiment). | [TransformersZeroShotDocumentClassifier](../pipeline-components/classifiers/transformerszeroshotdocumentclassifier.mdx) | | DocumentLanguageClassifier | Detects the language of the documents you pass to it and adds it to the document metadata. | [DocumentLanguageClassifier](../pipeline-components/classifiers/documentlanguageclassifier.mdx) | -| EntityExtractor | Extracts predefined entities out of a piece of text. **Example usage:** Named entity extraction (NER). | [NamedEntityExtractor](../pipeline-components/extractors/namedentityextractor.mdx) | +| EntityExtractor | Extracts predefined entities out of a piece of text. **Example usage:** Named entity extraction (NER). | [NamedEntityExtractor](../pipeline-components/extractors/transformersnamedentityextractor.mdx) | | FileClassifier | Distinguishes between text, PDF, Markdown, Docx, and HTML files. **Example usage:** Routing files to appropriate converters (for example, it routes PDF files to `PDFToTextConverter`). | [FileTypeRouter](../pipeline-components/routers/filetyperouter.mdx) | | FileConverter | Cleans and splits documents in different formats. **Example usage:** In indexing pipelines, extracting text from a file and casting it into the Document class format. | [Converters](../pipeline-components/converters.mdx) | | PreProcessor | Cleans and splits documents. **Example usage:** Normalizing white spaces, getting rid of headers and footers, splitting documents into smaller ones. | [PreProcessors](../pipeline-components/preprocessors.mdx) | diff --git a/docs-website/docs/pipeline-components/extractors.mdx b/docs-website/docs/pipeline-components/extractors.mdx index 24575836dd8..2490daf34dc 100644 --- a/docs-website/docs/pipeline-components/extractors.mdx +++ b/docs-website/docs/pipeline-components/extractors.mdx @@ -10,7 +10,6 @@ slug: "/extractors" | --- | --- | | [LLMDocumentContentExtractor](extractors/llmdocumentcontentextractor.mdx) | Extracts textual content from image-based documents using a vision-enabled Large Language Model (LLM). | | [LLMMetadataExtractor](extractors/llmmetadataextractor.mdx) | Extracts metadata from documents using a Large Language Model. The metadata is extracted by providing a prompt to a LLM that generates it. | -| [NamedEntityExtractor](extractors/namedentityextractor.mdx) | Extracts predefined entities out of a piece of text and writes them into documents' meta field. | | [PresidioEntityExtractor](extractors/presidioentityextractor.mdx) | Detects PII in Documents and stores entities as structured metadata, without modifying the text. Powered by Microsoft Presidio. | | [RegexTextExtractor](extractors/regextextextractor.mdx) | Extracts text from chat messages or strings using a regular expression pattern. | | [SpacyNamedEntityExtractor](extractors/spacynamedentityextractor.mdx) | Extracts predefined entities out of a piece of text and writes them into documents' meta field. Uses a spaCy model. | diff --git a/docs-website/docs/pipeline-components/extractors/namedentityextractor.mdx b/docs-website/docs/pipeline-components/extractors/namedentityextractor.mdx deleted file mode 100644 index 6516cab7b8e..00000000000 --- a/docs-website/docs/pipeline-components/extractors/namedentityextractor.mdx +++ /dev/null @@ -1,109 +0,0 @@ ---- -title: "NamedEntityExtractor" -id: namedentityextractor -slug: "/namedentityextractor" -description: "This component extracts predefined entities out of a piece of text and writes them into documents’ meta field." ---- - -# NamedEntityExtractor - -This component extracts predefined entities out of a piece of text and writes them into documents’ meta field. - -:::warning[Deprecated] - -`NamedEntityExtractor` is deprecated and will be removed in Haystack 3.0. It has moved to dedicated Core Integrations packages depending on the backend: - -- Hugging Face backend: `transformers-haystack` package, renamed to `TransformersNamedEntityExtractor`. See [TransformersNamedEntityExtractor](transformersnamedentityextractor.mdx) for the updated documentation. -- spaCy backend: `spacy-haystack` package, renamed to `SpacyNamedEntityExtractor`. See [SpacyNamedEntityExtractor](spacynamedentityextractor.mdx) for the updated documentation. - -::: - -
- -| | | -| --- | --- | -| **Most common position in a pipeline** | After the [PreProcessor](../preprocessors.mdx) in an indexing pipeline or after a [Retriever](../retrievers.mdx) in a query pipeline | -| **Mandatory init variables** | `backend`: The backend to use for NER

`model`: Name or path of the model to use | -| **Mandatory run variables** | `documents`: A list of documents | -| **Output variables** | `documents`: A list of documents | -| **API reference** | [Extractors](/reference/extractors-api) | -| **GitHub link** | https://github.com/deepset-ai/haystack/blob/main/haystack/components/extractors/named_entity_extractor.py | -| **Package name** | `haystack-ai` | - -
- -## Overview - -`NamedEntityExtractor` looks for entities, which are spans in the text. The extractor automatically recognizes and groups them depending on their class, such as people's names, organizations, locations, and other types. The exact classes are determined by the model that you initialize the component with. - -`NamedEntityExtractor` takes a list of documents as input and returns a list of the same documents with their `meta` data enriched with `NamedEntityAnnotations`. A `NamedEntityAnnotation` consists of the type of the entity, the start and end of the span, and a score calculated by the model, for example: `NamedEntityAnnotation(entity='PER', start=11, end=16, score=0.9)`. - -When the `NamedEntityExtractor` is initialized, you need to set a `model` and a `backend`. The latter can be either `"hugging_face"` or `"spacy"`. Optionally, you can set `pipeline_kwargs`, which are then passed on to the Hugging Face pipeline or the spaCy pipeline. You can additionally set the `device` that is used to run the component. - -## Usage - -The current implementation supports two NER backends: Hugging Face and spaCy. These two backends work with any HF or spaCy model that supports token classification or NER. - -Here’s an example of how you could initialize different backends: - -```python -# Initialize with HF backend -extractor = NamedEntityExtractor(backend="hugging_face", model="dslim/bert-base-NER") - -# Initialize with spaCy backend -extractor = NamedEntityExtractor(backend="spacy", model="en_core_web_sm") -``` - -`NamedEntityExtractor` accepts a list of `Documents` as its input. The extractor annotates the raw text in the documents and stores the annotations in the document's `meta` dictionary under the `named_entities` key. - -```python -from haystack.dataclasses import Document -from haystack.components.extractors import NamedEntityExtractor - -extractor = NamedEntityExtractor(backend="hugging_face", model="dslim/bert-base-NER") - -documents = [ - Document(content="My name is Clara and I live in Berkeley, California."), - Document(content="I'm Merlin, the happy pig!"), - Document(content="New York State is home to the Empire State Building."), -] - -result = extractor.run(documents) -print(result["documents"]) -``` - -Here is the example result: - -```python -[Document(id=aec840d1b6c85609f4f16c3e222a5a25fd8c4c53bd981a40c1268ab9c72cee10, content: 'My name is Clara and I live in Berkeley, California.', meta: {'named_entities': [NamedEntityAnnotation(entity='PER', start=11, end=16, score=np.float32(0.99641764)), NamedEntityAnnotation(entity='LOC', start=31, end=39, score=np.float32(0.996198)), NamedEntityAnnotation(entity='LOC', start=41, end=51, score=np.float32(0.9990196))]}), -Document(id=98f1dc5d0ccd9d9950cd191d1076db0f7af40c401dd7608f11c90cb3fc38c0c2, content: 'I'm Merlin, the happy pig!', meta: {'named_entities': [NamedEntityAnnotation(entity='PER', start=4, end=10, score=np.float32(0.99054915))]}), -Document(id=44948ea0eec018b33aceaaedde4616eb9e93ce075e0090ec1613fc145f84b4a9, content: 'New York State is home to the Empire State Building.', meta: {'named_entities': [NamedEntityAnnotation(entity='LOC', start=0, end=14, score=np.float32(0.9989541)), NamedEntityAnnotation(entity='LOC', start=30, end=51, score=np.float32(0.9574631))]})] -``` - -### Get stored annotations - -This component includes the `get_stored_annotations` helper class method that allows you to retrieve the annotations stored in a `Document` transparently: - -```python -from haystack.dataclasses import Document -from haystack.components.extractors import NamedEntityExtractor - -extractor = NamedEntityExtractor(backend="hugging_face", model="dslim/bert-base-NER") - -documents = [ - Document(content="My name is Clara and I live in Berkeley, California."), - Document(content="I'm Merlin, the happy pig!"), - Document(content="New York State is home to the Empire State Building."), -] - -result = extractor.run(documents) - -annotations = [ - NamedEntityExtractor.get_stored_annotations(doc) for doc in result["documents"] -] -print(annotations) - -# If a Document doesn't contain any annotations, this returns None. -new_doc = Document(content="In one of many possible worlds...") -assert NamedEntityExtractor.get_stored_annotations(new_doc) is None -``` diff --git a/docs-website/docs/pipeline-components/generators.mdx b/docs-website/docs/pipeline-components/generators.mdx index 04270c9654e..5f73e44151a 100644 --- a/docs-website/docs/pipeline-components/generators.mdx +++ b/docs-website/docs/pipeline-components/generators.mdx @@ -28,7 +28,7 @@ Generators are responsible for generating text after you give them a prompt. The | [GoogleAIGeminiGenerator](generators/googleaigeminigenerator.mdx) | Enables text generation using Google Gemini models. **_This integration will be deprecated soon. We recommend using [GoogleGenAIChatGenerator](generators/googlegenaichatgenerator.mdx) integration instead._** | ✅ | | [GoogleGenAIChatGenerator](generators/googlegenaichatgenerator.mdx) | Enables chat completion using Google Gemini models through Google Gen AI SDK. | ✅ | | [HuggingFaceAPIChatGenerator](generators/huggingfaceapichatgenerator.mdx) | Enables chat completion using various Hugging Face APIs. | ✅ | -| [HuggingFaceLocalChatGenerator](generators/huggingfacelocalchatgenerator.mdx) | Provides an interface for chat completion using a Hugging Face model that runs locally. | ✅ | +| [TransformersChatGenerator](generators/transformerschatgenerator.mdx) | Provides an interface for chat completion using a Hugging Face model that runs locally. | ✅ | | [LiteLLMChatGenerator](generators/litellmchatgenerator.mdx) | Enables chat completion using various LLM providers through LiteLLM. | ✅ | | [LlamaCppChatGenerator](generators/llamacppchatgenerator.mdx) | Enables chat completion using an LLM running on Llama.cpp. | ❌ | | [LlamaCppGenerator](generators/llamacppgenerator.mdx) | Generate text using an LLM running with Llama.cpp. | ❌ | diff --git a/docs-website/docs/pipeline-components/generators/guides-to-generators/choosing-the-right-generator.mdx b/docs-website/docs/pipeline-components/generators/guides-to-generators/choosing-the-right-generator.mdx index 5bfed612450..74d6492a013 100644 --- a/docs-website/docs/pipeline-components/generators/guides-to-generators/choosing-the-right-generator.mdx +++ b/docs-website/docs/pipeline-components/generators/guides-to-generators/choosing-the-right-generator.mdx @@ -193,7 +193,7 @@ On-premise models mean that you host open models on your machine or infrastructu #### Local Experimentation -- GPU: [`HuggingFaceLocalChatGenerator`](../huggingfacelocalchatgenerator.mdx) is based on the Hugging Face Transformers library. This is good for experimentation when you have some GPU resources (for example, in Colab). If GPU resources are limited, alternative quantization options like bitsandbytes, GPTQ, and AWQ are supported. For more performant solutions in production use cases, refer to the options below. +- GPU: [`TransformersChatGenerator`](../transformerschatgenerator.mdx) is based on the Hugging Face Transformers library. This is good for experimentation when you have some GPU resources (for example, in Colab). If GPU resources are limited, alternative quantization options like bitsandbytes, GPTQ, and AWQ are supported. For more performant solutions in production use cases, refer to the options below. - CPU (+ GPU if available): [`LlamaCppChatGenerator`](../llamacppchatgenerator.mdx) uses the Llama.cpp library – a project written in C/C++ for efficient inference of LLMs. In particular, it employs the quantized GGUF format, suitable for running these models on standard machines (even without GPUs). If GPU resources are available, some model layers can be offloaded to GPU for enhanced speed. - CPU (+ GPU if available): [`OllamaChatGenerator`](../ollamachatgenerator.mdx) is based on the Ollama project, acting like Docker for LLMs. It provides a simple way to package and deploy these models. Internally based on the Llama.cpp library, it offers a more streamlined process for running on various platforms. diff --git a/docs-website/docs/pipeline-components/generators/huggingfacelocalchatgenerator.mdx b/docs-website/docs/pipeline-components/generators/huggingfacelocalchatgenerator.mdx deleted file mode 100644 index c0e2cdd9144..00000000000 --- a/docs-website/docs/pipeline-components/generators/huggingfacelocalchatgenerator.mdx +++ /dev/null @@ -1,97 +0,0 @@ ---- -title: "HuggingFaceLocalChatGenerator" -id: huggingfacelocalchatgenerator -slug: "/huggingfacelocalchatgenerator" -description: "Provides an interface for chat completion using a Hugging Face model that runs locally." ---- - -# HuggingFaceLocalChatGenerator - -Provides an interface for chat completion using a Hugging Face model that runs locally. - -:::warning[Deprecated] - -`HuggingFaceLocalChatGenerator` is deprecated and will be removed in Haystack 3.0. It has moved to the `transformers-haystack` package and was renamed to `TransformersChatGenerator`. See [TransformersChatGenerator](transformerschatgenerator.mdx) for the updated documentation. - -::: - -
- -| | | -| --- | --- | -| **Most common position in a pipeline** | After a [ChatPromptBuilder](../builders/chatpromptbuilder.mdx) | -| **Mandatory init variables** | `token`: The Hugging Face API token. Can be set with `HF_API_TOKEN` or `HF_TOKEN` env var. | -| **Mandatory run variables** | `messages`: A list of [`ChatMessage`](../../concepts/data-classes/chatmessage.mdx) objects representing the chat or a plain string | -| **Output variables** | `replies`: A list of strings with all the replies generated by the LLM | -| **API reference** | [Generators](/reference/generators-api) | -| **GitHub link** | https://github.com/deepset-ai/haystack/blob/main/haystack/components/generators/chat/hugging_face_local.py | -| **Package name** | `haystack-ai` | - -
- -## Overview - -Keep in mind that if LLMs run locally, you may need a powerful machine to run them. This depends strongly on the model you select and its parameter count. - -If a string is passed to `messages`, it is converted into a list containing a single `ChatMessage` with the `user` role. - -For remote file authorization, this component uses a `HF_API_TOKEN` environment variable by default. Otherwise, you can pass a Hugging Face API token at initialization with `token`: - -```python -local_generator = HuggingFaceLocalChatGenerator( - token=Secret.from_token(""), -) -``` - -### Streaming - -This Generator supports [streaming](guides-to-generators/choosing-the-right-generator.mdx#streaming-support) the tokens from the LLM directly in output. To do so, pass a function to the `streaming_callback` init parameter. - -## Usage - -### On its own - -```python -from haystack.components.generators.chat import HuggingFaceLocalChatGenerator -from haystack.dataclasses import ChatMessage - -generator = HuggingFaceLocalChatGenerator(model="HuggingFaceH4/zephyr-7b-beta") -messages = [ChatMessage.from_user("What's Natural Language Processing? Be brief.")] -print(generator.run(messages)) -``` - -### In a Pipeline - -```python -from haystack import Pipeline -from haystack.components.builders.prompt_builder import ChatPromptBuilder -from haystack.components.generators.chat import HuggingFaceLocalChatGenerator -from haystack.dataclasses import ChatMessage -from haystack.utils import Secret - -prompt_builder = ChatPromptBuilder() -llm = HuggingFaceLocalChatGenerator( - model="HuggingFaceH4/zephyr-7b-beta", - token=Secret.from_env_var("HF_API_TOKEN"), -) - -pipe = Pipeline() -pipe.add_component("prompt_builder", prompt_builder) -pipe.add_component("llm", llm) -pipe.connect("prompt_builder.prompt", "llm.messages") -location = "Berlin" -messages = [ - ChatMessage.from_system( - "Always respond in German even if some input data is in other languages.", - ), - ChatMessage.from_user("Tell me about {{location}}"), -] -pipe.run( - data={ - "prompt_builder": { - "template_variables": {"location": location}, - "template": messages, - }, - }, -) -``` diff --git a/docs-website/sidebars.js b/docs-website/sidebars.js index 61989a5e85e..cd7d92a2dbb 100644 --- a/docs-website/sidebars.js +++ b/docs-website/sidebars.js @@ -374,7 +374,6 @@ export default { items: [ 'pipeline-components/extractors/llmdocumentcontentextractor', 'pipeline-components/extractors/llmmetadataextractor', - 'pipeline-components/extractors/namedentityextractor', 'pipeline-components/extractors/presidioentityextractor', 'pipeline-components/extractors/regextextextractor', 'pipeline-components/extractors/spacynamedentityextractor', @@ -429,7 +428,6 @@ export default { 'pipeline-components/generators/googleaigeminigenerator', 'pipeline-components/generators/googlegenaichatgenerator', 'pipeline-components/generators/huggingfaceapichatgenerator', - 'pipeline-components/generators/huggingfacelocalchatgenerator', 'pipeline-components/generators/litellmchatgenerator', 'pipeline-components/generators/llamacppchatgenerator', 'pipeline-components/generators/llamacppgenerator', From 278a4f6ca4cd8c4a120319b5519bb6b00259e360 Mon Sep 17 00:00:00 2001 From: anakin87 Date: Thu, 9 Jul 2026 11:15:01 +0200 Subject: [PATCH 3/4] add orcarouterchatgenerator to stable docs --- .../generators/orcarouterchatgenerator.mdx | 167 ++++++++++++++++++ 1 file changed, 167 insertions(+) create mode 100644 docs-website/versioned_docs/version-2.31/pipeline-components/generators/orcarouterchatgenerator.mdx diff --git a/docs-website/versioned_docs/version-2.31/pipeline-components/generators/orcarouterchatgenerator.mdx b/docs-website/versioned_docs/version-2.31/pipeline-components/generators/orcarouterchatgenerator.mdx new file mode 100644 index 00000000000..9494708f391 --- /dev/null +++ b/docs-website/versioned_docs/version-2.31/pipeline-components/generators/orcarouterchatgenerator.mdx @@ -0,0 +1,167 @@ +--- +title: "OrcaRouterChatGenerator" +id: orcarouterchatgenerator +slug: "/orcarouterchatgenerator" +description: "This component enables chat completion through [OrcaRouter](https://www.orcarouter.ai/), an OpenAI-compatible model routing gateway." +--- + +# OrcaRouterChatGenerator + +This component enables chat completion through [OrcaRouter](https://www.orcarouter.ai/), an OpenAI-compatible model routing gateway. + +
+ +| | | +| --- | --- | +| **Most common position in a pipeline** | After a [ChatPromptBuilder](../builders/chatpromptbuilder.mdx) | +| **Mandatory init variables** | `api_key`: An OrcaRouter API key. Can be set with `ORCAROUTER_API_KEY` env variable or passed to `init()` method. | +| **Mandatory run variables** | `messages`: A list of [ChatMessage](../../concepts/data-classes/chatmessage.mdx) objects | +| **Output variables** | `replies`: A list of [ChatMessage](../../concepts/data-classes/chatmessage.mdx) objects | +| **API reference** | [OrcaRouter](/reference/integrations-orcarouter) | +| **GitHub link** | https://github.com/deepset-ai/haystack-core-integrations/tree/main/integrations/orcarouter | +| **Package name** | `orcarouter-haystack` | + +
+ +## Overview + +The `OrcaRouterChatGenerator` enables you to use models from multiple providers (such as `openai/gpt-4o-mini`, `anthropic/claude-opus-4.8`, and `google/gemini-2.5-flash`) by making chat completion calls to the [OrcaRouter API](https://docs.orcarouter.ai). Models are addressed with a `provider/model` namespace, and you can browse the available models in the [OrcaRouter model catalog](https://www.orcarouter.ai/models). + +This generator also supports OrcaRouter-specific features such as: + +- Automatic routing with the `orcarouter/auto` model, which lets OrcaRouter pick a live upstream model per request based on the policy configured in your OrcaRouter console. +- Provider routing and model fallback that are configurable with the `generation_kwargs` parameter during initialization or runtime. OrcaRouter-specific routing options are forwarded to the gateway through `extra_body`. + +This component uses the same `ChatMessage` format as other Haystack Chat Generators for structured input and output. For more information, see the [ChatMessage documentation](../../concepts/data-classes/chatmessage.mdx). + +### Tool Support + +`OrcaRouterChatGenerator` supports function calling through the `tools` parameter, which accepts flexible tool configurations: + +- **A list of Tool objects**: Pass individual tools as a list +- **A single Toolset**: Pass an entire Toolset directly +- **Mixed Tools and Toolsets**: Combine multiple Toolsets with standalone tools in a single list + +This allows you to organize related tools into logical groups while also including standalone tools as needed. + +```python +from haystack.tools import Tool, Toolset +from haystack_integrations.components.generators.orcarouter import OrcaRouterChatGenerator + +# Create individual tools +weather_tool = Tool(name="weather", description="Get weather info", ...) +news_tool = Tool(name="news", description="Get latest news", ...) + +# Group related tools into a toolset +math_toolset = Toolset([add_tool, subtract_tool, multiply_tool]) + +# Pass mixed tools and toolsets to the generator +generator = OrcaRouterChatGenerator( + tools=[math_toolset, weather_tool, news_tool] # Mix of Toolset and Tool objects +) +``` + +For more details on working with tools, see the [Tool](../../tools/tool.mdx) and [Toolset](../../tools/toolset.mdx) documentation. + +### Initialization + +To use this integration, you need an OrcaRouter API key. You can provide it with the `ORCAROUTER_API_KEY` environment variable or by using a [Secret](../../concepts/secret-management.mdx). + +Then, install the `orcarouter-haystack` integration: + +```shell +pip install orcarouter-haystack +``` + +### Streaming + +`OrcaRouterChatGenerator` supports [streaming](guides-to-generators/choosing-the-right-generator.mdx#streaming-support) responses from the LLM, allowing tokens to be emitted as they are generated. To enable streaming, pass a callable to the `streaming_callback` parameter during initialization. + +## Usage + +### On its own + +```python +from haystack.dataclasses import ChatMessage +from haystack_integrations.components.generators.orcarouter import ( + OrcaRouterChatGenerator, +) + +client = OrcaRouterChatGenerator(model="openai/gpt-4o-mini") +response = client.run([ChatMessage.from_user("What are Agentic Pipelines? Be brief.")]) +print(response["replies"][0].text) +``` + +With automatic routing and streaming: + +```python +from haystack.dataclasses import ChatMessage +from haystack_integrations.components.generators.orcarouter import ( + OrcaRouterChatGenerator, +) + +client = OrcaRouterChatGenerator( + model="orcarouter/auto", + streaming_callback=lambda chunk: print(chunk.content, end="", flush=True), +) + +response = client.run([ChatMessage.from_user("What are Agentic Pipelines? Be brief.")]) + +# check the model used for the response +print("\n\n Model used: ", response["replies"][0].meta["model"]) +``` + +With a fallback chain: + +```python +from haystack.dataclasses import ChatMessage +from haystack_integrations.components.generators.orcarouter import ( + OrcaRouterChatGenerator, +) + +client = OrcaRouterChatGenerator( + model="openai/gpt-4o-mini", + generation_kwargs={ + "extra_body": { + "route": "fallback", + "models": [ + "openai/gpt-4o-mini", + "anthropic/claude-haiku-4.5", + "google/gemini-2.5-flash", + ], + }, + }, +) + +response = client.run([ChatMessage.from_user("What is Haystack?")]) +print(response["replies"][0].text) +``` + +### In a pipeline + +```python +from haystack import Pipeline +from haystack.components.builders import ChatPromptBuilder +from haystack.dataclasses import ChatMessage +from haystack_integrations.components.generators.orcarouter import ( + OrcaRouterChatGenerator, +) + +prompt_builder = ChatPromptBuilder() +llm = OrcaRouterChatGenerator(model="openai/gpt-4o-mini") + +pipe = Pipeline() +pipe.add_component("builder", prompt_builder) +pipe.add_component("llm", llm) +pipe.connect("builder.prompt", "llm.messages") + +messages = [ + ChatMessage.from_system("Give brief answers."), + ChatMessage.from_user("Tell me about {{city}}"), +] + +response = pipe.run( + data={"builder": {"template": messages, "template_variables": {"city": "Berlin"}}}, +) +print(response) +``` From 67f6d6deb6e4bac9525d8a2d288cfbc528d4715a Mon Sep 17 00:00:00 2001 From: anakin87 Date: Thu, 9 Jul 2026 11:18:15 +0200 Subject: [PATCH 4/4] how to install transformers integration --- docs-website/docs/concepts/device-management.mdx | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/docs-website/docs/concepts/device-management.mdx b/docs-website/docs/concepts/device-management.mdx index 76adea72265..a5666ad0d89 100644 --- a/docs-website/docs/concepts/device-management.mdx +++ b/docs-website/docs/concepts/device-management.mdx @@ -31,6 +31,14 @@ Find the full code for the abstractions above in the Haystack GitHub [repo](http ## Usage +:::info +The examples below use the [`TransformersChatGenerator`](../pipeline-components/generators/transformerschatgenerator.mdx), which is part of the `transformers-haystack` integration. Install it with: + +```bash +pip install transformers-haystack +``` +::: + To use a single device for inference, use either the `ComponentDevice.from_single` or `ComponentDevice.from_str` class method: ```python