Skip to content

Commit 84d98ff

Browse files
authored
feat(crewai): add new tracing integration (#341)
Add a CrewAI integration that subscribes to the event bus and maps kickoff, task, agent, LLM, and tool events into Braintrust spans. resolves #335
1 parent 1295d84 commit 84d98ff

15 files changed

Lines changed: 4244 additions & 406 deletions

File tree

py/examples/dspy/example.py

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -26,7 +26,8 @@ def main():
2626
print("🔍 Braintrust logging enabled - view traces at https://braintrust.dev")
2727

2828
# Disable DSPy's disk cache (keep memory cache for performance)
29-
dspy.configure_cache(enable_disk_cache=False, enable_memory_cache=True)
29+
if hasattr(dspy, "configure_cache"):
30+
dspy.configure_cache(enable_disk_cache=False, enable_memory_cache=True) # pylint: disable=no-member
3031

3132
# Configure DSPy with Braintrust callback
3233
lm = dspy.LM("openai/gpt-4o-mini")

py/noxfile.py

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -343,6 +343,22 @@ def test_dspy(session, version):
343343
_run_tests(session, f"{INTEGRATION_DIR}/dspy/test_dspy.py", version=version)
344344

345345

346+
CREWAI_VERSIONS = _get_matrix_versions("crewai")
347+
348+
349+
@nox.session()
350+
@nox.parametrize("version", CREWAI_VERSIONS, ids=CREWAI_VERSIONS)
351+
def test_crewai(session, version):
352+
if sys.version_info >= (3, 14):
353+
session.skip(
354+
"CrewAI currently resolves instructor -> pydantic-core builds that do not ship Python 3.14 wheels"
355+
)
356+
_install_test_deps(session)
357+
_install_group_locked(session, "test-crewai")
358+
_install_matrix_dep(session, "crewai", version)
359+
_run_tests(session, f"{INTEGRATION_DIR}/crewai/test_crewai.py", version=version)
360+
361+
346362
GOOGLE_ADK_VERSIONS = _get_matrix_versions("google-adk")
347363

348364

py/pyproject.toml

Lines changed: 26 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -145,6 +145,13 @@ test-langchain = [
145145
"langgraph==1.1.6",
146146
]
147147

148+
test-crewai = [
149+
{include-group = "test"},
150+
# CrewAI's no-network smoke test forces the LiteLLM fallback path via
151+
# ``is_litellm=True`` + ``mock_response``.
152+
"litellm==1.83.10",
153+
]
154+
148155
test-cli = [
149156
{include-group = "test"},
150157
"httpx==0.28.1",
@@ -172,6 +179,15 @@ lint = [
172179
"cohere",
173180
"autoevals",
174181
"braintrust-core",
182+
# crewai's transitive dep instructor pulls in pydantic-core 2.33.x, which
183+
# does not yet ship wheels for Python 3.14. Exclude 3.14 so pylint can
184+
# resolve the rest of the environment. Keep the package itself unpinned so
185+
# the resolver can still pick a mutually compatible version with the other
186+
# lint deps.
187+
"crewai; python_version<'3.14'",
188+
# onnxruntime 1.24.3 dropped cp310 wheels. Constrain the transitive
189+
# resolution on Python 3.10 so pylint can still build its environment.
190+
"onnxruntime<1.24; python_version<'3.11'",
175191
"dspy",
176192
"google-adk",
177193
"google-genai",
@@ -218,6 +234,7 @@ conflicts = [
218234
[
219235
{group = "test-openai-agents"},
220236
{group = "test-litellm"},
237+
{group = "test-crewai"},
221238
{group = "test-agno"},
222239
{group = "test-agentscope"},
223240
{group = "test-langchain"},
@@ -309,6 +326,13 @@ latest = "google-genai==1.73.1"
309326
latest = "dspy==3.2.0"
310327
"2.6.0" = "dspy==2.6.0"
311328

329+
[tool.braintrust.matrix.crewai]
330+
# 1.13.0 is the first release with the full causal-id surface (event_id /
331+
# parent_event_id / started_event_id on BaseEvent) plus the ``usage`` field on
332+
# LLMCallCompletedEvent that the Braintrust CrewAI integration depends on.
333+
latest = "crewai==1.14.2"
334+
"1.13.0" = "crewai==1.13.0"
335+
312336
[tool.braintrust.matrix.google-adk]
313337
latest = "google-adk==1.31.1"
314338
"1.14.1" = "google-adk==1.14.1"
@@ -362,6 +386,7 @@ autogen = ["autogen-agentchat"]
362386
anthropic = ["anthropic"]
363387
cohere = ["cohere"]
364388
claude_agent_sdk = ["claude-agent-sdk"]
389+
crewai = ["crewai"]
365390
dspy = ["dspy"]
366391
google_genai = ["google-genai"]
367392
langchain = ["langchain-core"]
@@ -381,6 +406,7 @@ anthropic = "anthropic"
381406
cohere = "cohere"
382407
autoevals = "autoevals"
383408
braintrust-core = "braintrust_core"
409+
crewai = "crewai"
384410
dspy = "dspy"
385411
google-adk = "google.adk"
386412
google-genai = "google.genai"

py/src/braintrust/auto.py

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,7 @@
1515
AutoGenIntegration,
1616
ClaudeAgentSDKIntegration,
1717
CohereIntegration,
18+
CrewAIIntegration,
1819
DSPyIntegration,
1920
GoogleGenAIIntegration,
2021
LangChainIntegration,
@@ -62,6 +63,7 @@ def auto_instrument(
6263
openai_agents: bool = True,
6364
cohere: bool = True,
6465
autogen: bool = True,
66+
crewai: bool = True,
6567
) -> dict[str, bool]:
6668
"""
6769
Auto-instrument supported AI/ML libraries for Braintrust tracing.
@@ -89,6 +91,7 @@ def auto_instrument(
8991
openai_agents: Enable OpenAI Agents SDK instrumentation (default: True)
9092
cohere: Enable Cohere instrumentation (default: True)
9193
autogen: Enable AutoGen instrumentation (default: True)
94+
crewai: Enable CrewAI instrumentation (default: True)
9295
9396
Returns:
9497
Dict mapping integration name to whether it was successfully instrumented.
@@ -168,6 +171,8 @@ def auto_instrument(
168171
results["cohere"] = _instrument_integration(CohereIntegration)
169172
if autogen:
170173
results["autogen"] = _instrument_integration(AutoGenIntegration)
174+
if crewai:
175+
results["crewai"] = _instrument_integration(CrewAIIntegration)
171176

172177
return results
173178

py/src/braintrust/integrations/__init__.py

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,7 @@
55
from .autogen import AutoGenIntegration
66
from .claude_agent_sdk import ClaudeAgentSDKIntegration
77
from .cohere import CohereIntegration
8+
from .crewai import CrewAIIntegration
89
from .dspy import DSPyIntegration
910
from .google_genai import GoogleGenAIIntegration
1011
from .langchain import LangChainIntegration
@@ -24,6 +25,7 @@
2425
"AutoGenIntegration",
2526
"ClaudeAgentSDKIntegration",
2627
"CohereIntegration",
28+
"CrewAIIntegration",
2729
"DSPyIntegration",
2830
"GoogleGenAIIntegration",
2931
"LiteLLMIntegration",
Lines changed: 38 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,38 @@
1+
"""Test auto_instrument for CrewAI.
2+
3+
Verifies that ``auto_instrument(crewai=True)`` registers the Braintrust
4+
CrewAI listener on ``crewai_event_bus`` and is idempotent across repeated
5+
calls. Full span-shape coverage lives in ``test_crewai.py``.
6+
"""
7+
8+
# pylint: disable=import-error
9+
10+
from braintrust.auto import auto_instrument
11+
from braintrust.integrations.crewai import BraintrustCrewAIListener
12+
from braintrust.integrations.crewai.patchers import _get_registered_listener
13+
14+
15+
# 1. Not registered initially.
16+
assert _get_registered_listener() is None
17+
18+
# 2. Instrument once.
19+
results = auto_instrument()
20+
assert results.get("crewai") is True
21+
listener1 = _get_registered_listener()
22+
assert listener1 is not None
23+
assert isinstance(listener1, BraintrustCrewAIListener)
24+
25+
# 3. Idempotent — same listener, still reports True.
26+
results2 = auto_instrument()
27+
assert results2.get("crewai") is True
28+
assert _get_registered_listener() is listener1
29+
30+
# 4. Listener is actually subscribed on the CrewAI event bus.
31+
from crewai.events import CrewKickoffStartedEvent
32+
from crewai.events.event_bus import crewai_event_bus
33+
34+
35+
sync_handlers = crewai_event_bus._sync_handlers.get(CrewKickoffStartedEvent, frozenset())
36+
assert sync_handlers, "Expected at least one sync handler registered for CrewKickoffStartedEvent"
37+
38+
print("SUCCESS")
Lines changed: 73 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,73 @@
1+
"""Braintrust integration for CrewAI.
2+
3+
Public entry points:
4+
5+
- :func:`setup_crewai` — mirrors :func:`setup_agno`/``setup_dspy`` ergonomics.
6+
Initializes a Braintrust logger (when one is not already set) and
7+
registers the CrewAI event-bus listener.
8+
- :func:`patch_crewai` — thin setup-only helper (no logger init), matches
9+
the ``patch_*`` naming used elsewhere.
10+
- :class:`CrewAIIntegration` — used by :func:`braintrust.auto_instrument`.
11+
- :class:`BraintrustCrewAIListener` — exposed for advanced users who want
12+
to register the listener manually on their own event-bus instance (e.g.
13+
for tests).
14+
"""
15+
16+
import logging
17+
18+
from braintrust.logger import NOOP_SPAN, current_span, init_logger
19+
20+
from .integration import CrewAIIntegration
21+
from .tracing import BraintrustCrewAIListener
22+
23+
24+
logger = logging.getLogger(__name__)
25+
26+
27+
__all__ = [
28+
"BraintrustCrewAIListener",
29+
"CrewAIIntegration",
30+
"patch_crewai",
31+
"setup_crewai",
32+
]
33+
34+
35+
def setup_crewai(
36+
api_key: str | None = None,
37+
project_id: str | None = None,
38+
project_name: str | None = None,
39+
) -> bool:
40+
"""Set up Braintrust tracing for CrewAI.
41+
42+
Initializes a Braintrust logger (unless one is already active) and
43+
registers :class:`BraintrustCrewAIListener` on the singleton
44+
``crewai_event_bus``. Safe to call multiple times.
45+
46+
Args:
47+
api_key: Braintrust API key (optional, ``BRAINTRUST_API_KEY`` env var works too).
48+
project_id: Braintrust project id (optional).
49+
project_name: Braintrust project name (optional, ``BRAINTRUST_PROJECT`` env var works too).
50+
51+
Returns:
52+
``True`` on successful (or already-registered) setup, ``False`` when
53+
CrewAI is not importable at the required minimum version.
54+
"""
55+
span = current_span()
56+
if span == NOOP_SPAN:
57+
init_logger(project=project_name, api_key=api_key, project_id=project_id)
58+
59+
return CrewAIIntegration.setup()
60+
61+
62+
def patch_crewai() -> bool:
63+
"""Register the Braintrust CrewAI listener without initializing a logger.
64+
65+
Equivalent to ``CrewAIIntegration.setup()``. Use this when the calling
66+
code already sets up Braintrust (e.g. via :func:`braintrust.init_logger`)
67+
and only needs CrewAI tracing wired up.
68+
69+
Returns:
70+
``True`` if CrewAI was patched (or already patched), ``False`` when
71+
CrewAI is not installed at the required minimum version.
72+
"""
73+
return CrewAIIntegration.setup()
Lines changed: 29 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,29 @@
1+
"""CrewAI integration — orchestration class.
2+
3+
Registers a single event-bus listener on :data:`crewai.events.crewai_event_bus`
4+
that maps CrewAI events (crew kickoff, tasks, agent execution, LLM calls,
5+
tool usage) into Braintrust spans.
6+
7+
Requires CrewAI 1.13.0 or newer, which is the first release that exposes
8+
the full causal-id surface (``event_id``, ``parent_event_id``,
9+
``started_event_id``) plus the ``usage`` field on
10+
``LLMCallCompletedEvent`` the integration consumes.
11+
"""
12+
13+
from braintrust.integrations.base import BaseIntegration
14+
15+
from .patchers import EventBusPatcher
16+
17+
18+
class CrewAIIntegration(BaseIntegration):
19+
"""Braintrust instrumentation for CrewAI."""
20+
21+
name = "crewai"
22+
import_names = ("crewai",)
23+
# 1.13.0 is the first release with the event-bus surface this
24+
# integration depends on (``started_event_id`` on BaseEvent + ``usage``
25+
# on ``LLMCallCompletedEvent``). Older 1.x releases ship the bus but
26+
# not these fields, so we gate instead of trying to version-branch the
27+
# listener.
28+
min_version = "1.13.0"
29+
patchers = (EventBusPatcher,)
Lines changed: 100 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,100 @@
1+
"""CrewAI patchers — one ``CallbackPatcher`` that registers the listener."""
2+
3+
from typing import Any, ClassVar
4+
5+
from braintrust.integrations.base import CallbackPatcher
6+
7+
8+
# Module-level cache for the single registered listener instance. Keeping
9+
# it at module scope (rather than on the integration class) means
10+
# ``setup_crewai`` / ``patch_crewai`` / ``auto_instrument`` all see the
11+
# same ``BraintrustCrewAIListener`` regardless of entry point.
12+
_LISTENER: Any | None = None
13+
14+
15+
def _unregister_event_handler(event_bus: Any, event_type: Any, handler: Any) -> None:
16+
"""Best-effort wrapper around ``CrewAIEventsBus.off``.
17+
18+
Pylint cannot currently infer the dynamically-generated event-bus API in
19+
CrewAI, so we use ``getattr`` here instead of calling ``off`` directly.
20+
"""
21+
unregister = getattr(event_bus, "off", None)
22+
if callable(unregister):
23+
unregister(event_type, handler)
24+
25+
26+
def _register_braintrust_listener() -> bool:
27+
"""Idempotently create and register the Braintrust listener.
28+
29+
The listener subclasses :class:`crewai.events.BaseEventListener`, whose
30+
``__init__`` registers handlers on the process-singleton
31+
``crewai_event_bus``. We cache the instance at module scope so repeat
32+
calls (e.g. ``setup_crewai()`` followed by ``auto_instrument()``) do
33+
not register a second listener.
34+
"""
35+
global _LISTENER # noqa: PLW0603
36+
if _LISTENER is not None:
37+
return True
38+
39+
# Lazy import: CrewAI may not be installed in the environment.
40+
from .tracing import BraintrustCrewAIListener
41+
42+
_LISTENER = BraintrustCrewAIListener()
43+
return True
44+
45+
46+
def _listener_registered() -> bool:
47+
"""Return whether the Braintrust listener is currently registered."""
48+
return _LISTENER is not None
49+
50+
51+
def _get_registered_listener() -> Any | None:
52+
"""Return the registered listener, or ``None`` when setup has not run."""
53+
return _LISTENER
54+
55+
56+
def _reset_for_testing() -> None:
57+
"""Unregister the Braintrust listener and forget cached runtime state.
58+
59+
Intended for pytest fixtures that need to restart from a clean slate.
60+
Safe to call when CrewAI is not importable or nothing has been
61+
registered. Not part of the public API.
62+
"""
63+
global _LISTENER # noqa: PLW0603
64+
if _LISTENER is None:
65+
return
66+
67+
try:
68+
from crewai.events.event_bus import crewai_event_bus
69+
except ImportError:
70+
_LISTENER = None
71+
return
72+
73+
for event_type, handlers in list(crewai_event_bus._sync_handlers.items()):
74+
for handler in list(handlers):
75+
handler_mod = getattr(handler, "__module__", "")
76+
if "braintrust" in handler_mod and "crewai" in handler_mod:
77+
_unregister_event_handler(crewai_event_bus, event_type, handler)
78+
79+
_LISTENER = None
80+
81+
# Clear the runtime subclass cache so the next setup rebuilds it; this
82+
# matters for tests that monkey-patch the listener base class.
83+
from .tracing import BraintrustCrewAIListener
84+
85+
BraintrustCrewAIListener._cls = None
86+
87+
88+
class EventBusPatcher(CallbackPatcher):
89+
"""Register :class:`BraintrustCrewAIListener` on ``crewai_event_bus``.
90+
91+
The target module check gates this patcher on the event-bus module being
92+
importable, not the top-level ``crewai`` package. That means users who
93+
install only a CrewAI fork missing the event-bus surface get a clean
94+
skip rather than an import error during setup.
95+
"""
96+
97+
name: ClassVar[str] = "crewai.event_bus"
98+
target_module: ClassVar[str] = "crewai.events.event_bus"
99+
callback: ClassVar[Any] = staticmethod(_register_braintrust_listener)
100+
state_getter: ClassVar[Any] = staticmethod(_listener_registered)

0 commit comments

Comments
 (0)