Remove generator, tests, and updates as discussed

Author: Ryzhtus

integrations/dspy/README.md

@@ -0,0 +1,65 @@
+# dspy-haystack
+
+[![PyPI - Version](https://img.shields.io/pypi/v/dspy-haystack.svg)](https://pypi.org/project/dspy-haystack)
+[![PyPI - Python Version](https://img.shields.io/pypi/pyversions/dspy-haystack.svg)](https://pypi.org/project/dspy-haystack)
+
+An integration between [DSPy](https://github.com/stanfordnlp/dspy) and [Haystack](https://haystack.deepset.ai/).
+
+DSPy is a framework for algorithmically optimizing prompts for Language Models by applying classical machine learning concepts (training data, evaluation metrics, optimization).
+
+This integration provides:
+- **DSPyChatGenerator** — a Haystack ChatGenerator component that uses DSPy signatures and modules for structured generation
+
+## Installation
+
+```bash
+pip install dspy-haystack
+```
+
+## Quick Start
+
+### DSPyChatGenerator
+
+A Haystack chat generator that uses DSPy signatures for structured generation with built-in reasoning patterns (Chain-of-Thought, Predict, ReAct).
+
+```python
+from haystack import Pipeline
+from haystack.dataclasses import ChatMessage
+from haystack_integrations.components.generators.dspy import DSPyChatGenerator
+import dspy
+
+# Define a DSPy signature
+class QASignature(dspy.Signature):
+    """Answer questions accurately and concisely."""
+    question = dspy.InputField(desc="The user's question")
+    answer = dspy.OutputField(desc="A clear, concise answer")
+
+# Create the generator
+generator = DSPyChatGenerator(
+    model="openai/gpt-5-mini",
+    signature=QASignature,
+    module_type="ChainOfThought"
+)
+
+# Use in pipeline
+pipeline = Pipeline()
+pipeline.add_component("llm", generator)
+
+messages = [ChatMessage.from_user("What is the capital of France?")]
+result = pipeline.run({"llm": {"messages": messages}})
+print(result["llm"]["replies"][0].text)
+```
+
+You can also use string signatures for quick prototyping:
+
+```python
+generator = DSPyChatGenerator(
+    model="openai/gpt-5-mini",
+    signature="question -> answer",
+    module_type="Predict"
+)
+```
+
+## License
+
+`dspy-haystack` is distributed under the terms of the [Apache-2.0](https://spdx.org/licenses/Apache-2.0.html) license.

integrations/dspy/examples/chat_generator_example.py

@@ -15,7 +15,7 @@ def basic_qa_example():
     """Simple question-answering with Chain-of-Thought reasoning."""
 
     generator = DSPyChatGenerator(
-        model="openai/gpt-4o-mini",
+        model="openai/gpt-5-mini",
         signature=QASignature,
         module_type="ChainOfThought",
         output_field="answer",
@@ -34,7 +34,7 @@ def basic_qa_example():
 def string_signature_example():
     """Using a simple string signature instead of a class."""
     generator = DSPyChatGenerator(
-        model="openai/gpt-4o-mini",
+        model="openai/gpt-5-mini",
         signature="question -> answer",
         module_type="Predict",
         output_field="answer",

integrations/dspy/pyproject.toml

@@ -65,8 +65,7 @@ integration = 'pytest -m "integration" {args:tests}'
 all = 'pytest {args:tests}'
 cov-retry = 'pytest --cov=haystack_integrations --reruns 3 --reruns-delay 30 -x {args:tests}'
 
-types = """mypy -p haystack_integrations.components.generators.dspy \
--p haystack_integrations.utils.dspy {args}"""
+types = "mypy -p haystack_integrations.components.generators.dspy {args}"
 
 [tool.mypy]
 install_types = true

integrations/dspy/src/haystack_integrations/components/generators/dspy/__init__.py

@@ -1,5 +1,3 @@
 from haystack_integrations.components.generators.dspy.chat.chat_generator import DSPyChatGenerator
-from haystack_integrations.components.generators.dspy.generator import DSPyGenerator
-from haystack_integrations.components.generators.dspy.program_runner import DSPyProgramRunner
 
-__all__ = ["DSPyChatGenerator", "DSPyGenerator", "DSPyProgramRunner"]
+__all__ = ["DSPyChatGenerator"]

integrations/dspy/src/haystack_integrations/components/generators/dspy/chat/__init__.py

@@ -0,0 +1,3 @@
+from haystack_integrations.components.generators.dspy.chat.chat_generator import DSPyChatGenerator
+
+__all__ = ["DSPyChatGenerator"]

integrations/dspy/src/haystack_integrations/components/generators/dspy/chat/chat_generator.py

@@ -1,15 +1,48 @@
 from typing import Any, Callable, Dict, List, Optional, Type, Union
 
 import dspy
-from haystack import component
+from haystack import component, default_from_dict, default_to_dict
 from haystack.dataclasses import ChatMessage, ChatRole
-from haystack.utils import Secret
+from haystack.utils import Secret, deserialize_secrets_inplace
 
-from haystack_integrations.components.generators.dspy.generator import DSPyGenerator
+VALID_MODULE_TYPES = {"Predict", "ChainOfThought", "ReAct"}
+
+
+def configure_dspy_lm(model: str, api_key: str, **kwargs: Any) -> dspy.LM:
+    """
+    Create and configure a DSPy language model.
+
+    :param model: Model identifier (e.g. ``"openai/gpt-5-mini"``).
+    :param api_key: Resolved API key string.
+    :param kwargs: Additional keyword arguments passed to ``dspy.LM``.
+    :returns: The configured ``dspy.LM`` instance.
+    """
+    lm = dspy.LM(model=model, api_key=api_key, **kwargs)
+    dspy.configure(lm=lm)
+    return lm
+
+
+def get_dspy_module_class(module_type: str):
+    """
+    Map a module type string to the corresponding DSPy module class.
+
+    :param module_type: One of ``"Predict"``, ``"ChainOfThought"``, or ``"ReAct"``.
+    :returns: The DSPy module class.
+    :raises ValueError: If the module type is not recognized.
+    """
+    mapping = {
+        "Predict": dspy.Predict,
+        "ChainOfThought": dspy.ChainOfThought,
+        "ReAct": dspy.ReAct,
+    }
+    if module_type not in mapping:
+        msg = f"Invalid module_type '{module_type}'. Must be one of {sorted(VALID_MODULE_TYPES)}"
+        raise ValueError(msg)
+    return mapping[module_type]
 
 
 @component
-class DSPyChatGenerator(DSPyGenerator):
+class DSPyChatGenerator:
     """
     A Haystack chat generator component that uses DSPy signatures and modules
     for structured generation.
@@ -64,18 +97,94 @@ def __init__(
         :param input_mapping: Maps DSPy signature input field names to run kwarg names.
         :param streaming_callback: Callback for streaming responses.
         """
-        DSPyGenerator.__init__(
-            self,
-            signature=signature,
-            model=model,
-            api_key=api_key,
-            module_type=module_type,
-            output_field=output_field,
-            generation_kwargs=generation_kwargs,
-            input_mapping=input_mapping,
-            streaming_callback=streaming_callback,
+        if module_type not in VALID_MODULE_TYPES:
+            msg = f"Invalid module_type '{module_type}'. Must be one of {sorted(VALID_MODULE_TYPES)}"
+            raise ValueError(msg)
+
+        self.signature = signature
+        self.model = model
+        self.api_key = api_key
+        self.module_type = module_type
+        self.output_field = output_field
+        self.generation_kwargs = generation_kwargs or {}
+        self.input_mapping = input_mapping
+        self.streaming_callback = streaming_callback
+
+        self._lm = configure_dspy_lm(
+            model=self.model,
+            api_key=self.api_key.resolve_value(),
+            **self.generation_kwargs,
         )
 
+        module_class = get_dspy_module_class(self.module_type)
+        self._module = module_class(self.signature)
+
+    def _build_dspy_inputs(self, prompt: str, **kwargs) -> Dict[str, Any]:
+        """Build the input dict for the DSPy module call."""
+        if self.input_mapping:
+            dspy_inputs = {}
+            for sig_field, source in self.input_mapping.items():
+                if source in kwargs:
+                    dspy_inputs[sig_field] = kwargs[source]
+                else:
+                    dspy_inputs[sig_field] = prompt
+            return dspy_inputs
+
+        input_fields = self._get_input_field_names()
+        dspy_inputs = {input_fields[0]: prompt}
+
+        for field in input_fields[1:]:
+            if field in kwargs:
+                dspy_inputs[field] = kwargs[field]
+
+        return dspy_inputs
+
+    def _get_input_field_names(self) -> List[str]:
+        """Get input field names from the signature."""
+        if isinstance(self.signature, str):
+            input_part = self.signature.split("->")[0].strip()
+            return [f.strip() for f in input_part.split(",")]
+        return list(self.signature.input_fields.keys())
+
+    @staticmethod
+    def _extract_last_user_message(messages: List[ChatMessage]) -> str:
+        """Extract the text of the last user message from a list of chat messages."""
+        for msg in reversed(messages):
+            if msg.role == ChatRole.USER:
+                return msg.text
+        return messages[-1].text
+
+    def _signature_to_string(self) -> str:
+        """Convert the signature to a string representation for serialization."""
+        if isinstance(self.signature, str):
+            return self.signature
+        input_names = list(self.signature.input_fields.keys())
+        output_names = list(self.signature.output_fields.keys())
+        return ", ".join(input_names) + " -> " + ", ".join(output_names)
+
+    def to_dict(self) -> Dict[str, Any]:
+        """Serialize this component to a dictionary."""
+        kwargs: Dict[str, Any] = {
+            "signature": self._signature_to_string(),
+            "model": self.model,
+            "module_type": self.module_type,
+            "output_field": self.output_field,
+            "generation_kwargs": self.generation_kwargs,
+            "input_mapping": self.input_mapping,
+        }
+        try:
+            kwargs["api_key"] = self.api_key.to_dict()
+        except ValueError:
+            pass
+        return default_to_dict(self, **kwargs)
+
+    @classmethod
+    def from_dict(cls, data: Dict[str, Any]) -> "DSPyChatGenerator":
+        """Deserialize a component from a dictionary."""
+        init_params = data.get("init_parameters", {})
+        deserialize_secrets_inplace(init_params, ["api_key"])
+        return default_from_dict(cls, data)
+
     @component.output_types(replies=List[ChatMessage])
     def run(
         self,
@@ -96,11 +205,17 @@ def run(
             raise ValueError(msg)
 
         prompt = self._extract_last_user_message(messages)
-        result = DSPyGenerator.run(self, prompt=prompt, generation_kwargs=generation_kwargs, **kwargs)
+        dspy_inputs = self._build_dspy_inputs(prompt, **kwargs)
 
-        replies = [ChatMessage.from_assistant(text=text) for text in result["replies"]]
+        if generation_kwargs:
+            prediction = self._module(**dspy_inputs, config=generation_kwargs)
+        else:
+            prediction = self._module(**dspy_inputs)
 
-        return {"replies": replies, "meta": result["meta"]}
+        output_text = getattr(prediction, self.output_field, str(prediction))
+
+        replies = [ChatMessage.from_assistant(text=output_text)]
+        return {"replies": replies}
 
     @component.output_types(replies=List[ChatMessage])
     async def run_async(
@@ -124,18 +239,14 @@ async def run_async(
             raise ValueError(msg)
 
         prompt = self._extract_last_user_message(messages)
-        result = await DSPyGenerator.run_async(self, prompt=prompt, generation_kwargs=generation_kwargs, **kwargs)
+        dspy_inputs = self._build_dspy_inputs(prompt, **kwargs)
 
-        replies = [ChatMessage.from_assistant(text=text) for text in result["replies"]]
+        if generation_kwargs:
+            prediction = await self._module.acall(**dspy_inputs, config=generation_kwargs)
+        else:
+            prediction = await self._module.acall(**dspy_inputs)
 
-        return {"replies": replies, "meta": result["meta"]}
+        output_text = getattr(prediction, self.output_field, str(prediction))
 
-    @staticmethod
-    def _extract_last_user_message(messages: List[ChatMessage]) -> str:
-        """Extract the text of the last user message from a list of chat messages."""
-        for msg in reversed(messages):
-            if msg.role == ChatRole.USER:
-                return msg.text
-
-        # Fallback to last message if no user message found
-        return messages[-1].text
+        replies = [ChatMessage.from_assistant(text=output_text)]
+        return {"replies": replies}