docs(optimizers): DSPy prompt optimization — user docs, example notebook, and navigation

lib/crewai/pyproject.toml

@@ -66,6 +66,7 @@ openpyxl = [
     "openpyxl~=3.1.5",
 ]
 mem0 = ["mem0ai>=2.0.0,<3"]
+dspy = ["dspy>=2.5,<4"]
 docling = [
     "docling~=2.84.0",
 ]

lib/crewai/src/crewai/agent/core.py

@@ -1277,6 +1277,32 @@ def last_messages(self) -> list[LLMMessage]:
         """
         return self._last_messages
 
+    def get_effective_system_prompt(self) -> str:
+        """Return the rendered system prompt as it would be sent to the LLM.
+
+        Combines role, goal, and backstory per the active template. This is the
+        canonical read path for optimizer instrumentation and prompt inspection.
+
+        The following fields are stable public API — safe to read and write after
+        construction, and reflected immediately in the next call to this method:
+        ``role``, ``goal``, ``backstory``, ``system_template``, ``prompt_template``.
+
+        Returns:
+            The fully rendered system prompt string. Never empty.
+        """
+        result = Prompts(
+            agent=self,
+            has_tools=False,
+            use_native_tool_calling=False,
+            use_system_prompt=self.use_system_prompt,
+            system_template=self.system_template,
+            prompt_template=self.prompt_template,
+            response_template=self.response_template,
+        ).task_execution()
+        if isinstance(result, SystemPromptResult):
+            return result.system
+        return result.prompt
+
     def _get_knowledge_search_query(self, task_prompt: str, task: Task) -> str | None:
         """Generate a search query for the knowledge base based on the task description."""
         crewai_event_bus.emit(

lib/crewai/src/crewai/agents/agent_builder/base_agent.py

@@ -225,9 +225,30 @@ class BaseAgent(BaseModel, ABC, metaclass=AgentMeta):
     _token_process: TokenProcess = PrivateAttr(default_factory=TokenProcess)
     _kickoff_event_id: str | None = PrivateAttr(default=None)
     id: UUID4 = Field(default_factory=uuid.uuid4, frozen=True)
-    role: str = Field(description="Role of the agent")
-    goal: str = Field(description="Objective of the agent")
-    backstory: str = Field(description="Backstory of the agent")
+    role: str = Field(
+        description=(
+            "Role of the agent. "
+            "Stable public API: safe to read and write after construction. "
+            "DSPy optimizers and other instrumentation may update this field "
+            "to apply optimized instructions."
+        )
+    )
+    goal: str = Field(
+        description=(
+            "Objective of the agent. "
+            "Stable public API: safe to read and write after construction. "
+            "DSPy optimizers and other instrumentation may update this field "
+            "to apply optimized instructions."
+        )
+    )
+    backstory: str = Field(
+        description=(
+            "Backstory of the agent. "
+            "Stable public API: safe to read and write after construction. "
+            "DSPy optimizers and other instrumentation may update this field "
+            "to apply optimized instructions."
+        )
+    )
     config: dict[str, Any] | None = Field(
         description="Configuration for the agent", default=None, exclude=True
     )

lib/crewai/src/crewai/events/types/__init__.py

@@ -3,3 +3,21 @@
 This module contains all event types used throughout the CrewAI system
 for monitoring and extending agent, crew, task, and tool execution.
 """
+
+from crewai.events.types.optimizer_events import (
+    OptimizationCompletedEvent,
+    OptimizationFailedEvent,
+    OptimizationStartedEvent,
+)
+
+
+# OptimizationTrialCompletedEvent is intentionally excluded from __all__:
+# DSPy teleprompters do not expose a per-trial callback, so the event cannot
+# be emitted accurately. The class is defined in optimizer_events.py and will
+# be added here when DSPy adds callback support.
+
+__all__ = [
+    "OptimizationCompletedEvent",
+    "OptimizationFailedEvent",
+    "OptimizationStartedEvent",
+]

lib/crewai/src/crewai/events/types/optimizer_events.py

@@ -0,0 +1,48 @@
+"""Pydantic event types emitted by DSPyOptimizer during the optimization lifecycle."""
+
+from __future__ import annotations
+
+from typing import Literal
+
+from crewai.events.base_events import BaseEvent
+
+
+class OptimizationStartedEvent(BaseEvent):
+    """Emitted when DSPyOptimizer.compile() begins (after baseline measurement)."""
+
+    type: Literal["optimization_started"] = "optimization_started"
+    crew_name: str | None = None
+    algorithm: str
+    num_trials: int
+    trainset_size: int
+
+
+class OptimizationTrialCompletedEvent(BaseEvent):
+    """Emitted after each optimization trial (only when the teleprompter exposes a callback)."""
+
+    type: Literal["optimization_trial_completed"] = "optimization_trial_completed"
+    crew_name: str | None = None
+    algorithm: str
+    trial_number: int
+    trial_score: float
+
+
+class OptimizationCompletedEvent(BaseEvent):
+    """Emitted when DSPyOptimizer.compile() succeeds."""
+
+    type: Literal["optimization_completed"] = "optimization_completed"
+    crew_name: str | None = None
+    algorithm: str
+    baseline_score: float
+    optimized_score: float
+    score_delta: float
+    num_trials: int
+    version_id: str
+
+
+class OptimizationFailedEvent(BaseEvent):
+    """Emitted when DSPyOptimizer.compile() raises an exception."""
+
+    type: Literal["optimization_failed"] = "optimization_failed"
+    crew_name: str | None = None
+    error: str

lib/crewai/src/crewai/optimizers/__init__.py

@@ -0,0 +1,21 @@
+"""crewai.optimizers — algorithmic prompt optimization for CrewAI crews.
+
+Install the optional extra before importing DSPyOptimizer:
+    pip install 'crewai[dspy]'
+"""
+
+from typing import Any
+
+from crewai.optimizers.types import AgentInstructions, OptimizationResult
+
+
+__all__ = ["AgentInstructions", "DSPyOptimizer", "OptimizationResult"]
+
+
+def __getattr__(name: str) -> Any:
+    """Lazily import DSPyOptimizer to avoid loading dspy at package import time."""
+    if name == "DSPyOptimizer":
+        from crewai.optimizers.dspy_optimizer import DSPyOptimizer
+
+        return DSPyOptimizer
+    raise AttributeError(f"module {__name__!r} has no attribute {name!r}")