test(langchain): Add basic agent test with Responses call

Author: alexander-alderman-webb

tests/integrations/langchain/test_langchain.py

@@ -38,7 +38,8 @@
     from langchain.agents import tool, AgentExecutor, create_openai_tools_agent
 
 from langchain_core.prompts import ChatPromptTemplate, MessagesPlaceholder
-
+from langchain.agents import create_agent
+from langchain_core.messages import HumanMessage, SystemMessage
 
 from openai.types.chat.chat_completion_chunk import (
     ChatCompletionChunk,
@@ -81,6 +82,128 @@ def _llm_type(self) -> str:
         return llm_type
 
 
+@pytest.mark.parametrize(
+    "send_default_pii, include_prompts",
+    [
+        (True, True),
+        (True, False),
+        (False, True),
+        (False, False),
+    ],
+)
+@pytest.mark.parametrize(
+    "system_instructions_content",
+    [
+        "You are very powerful assistant, but don't know current events",
+        [
+            {"type": "text", "text": "You are a helpful assistant."},
+            {"type": "text", "text": "Be concise and clear."},
+        ],
+    ],
+    ids=["string", "blocks"],
+)
+def test_langchain_create_agent(
+    sentry_init,
+    capture_events,
+    send_default_pii,
+    include_prompts,
+    system_instructions_content,
+    request,
+    get_model_response,
+    nonstreaming_responses_model_response,
+):
+    sentry_init(
+        integrations=[
+            LangchainIntegration(
+                include_prompts=include_prompts,
+            )
+        ],
+        traces_sample_rate=1.0,
+        send_default_pii=send_default_pii,
+    )
+    events = capture_events()
+
+    model_repsonse = get_model_response(
+        nonstreaming_responses_model_response,
+        serialize_pydantic=True,
+        request_headers={
+            "X-Stainless-Raw-Response": "True",
+        },
+    )
+
+    llm = ChatOpenAI(
+        model_name="gpt-3.5-turbo",
+        temperature=0,
+        openai_api_key="badkey",
+        use_responses_api=True,
+    )
+    agent = create_agent(
+        model=llm,
+        tools=[get_word_length],
+        system_prompt=SystemMessage(content=system_instructions_content),
+        name="word_length_agent",
+    )
+
+    with patch.object(
+        llm.client._client._client,
+        "send",
+        return_value=model_repsonse,
+    ) as _:
+        with start_transaction():
+            agent.invoke(
+                {
+                    "messages": [
+                        HumanMessage(content="How many letters in the word eudca"),
+                    ],
+                },
+            )
+
+    tx = events[0]
+    assert tx["type"] == "transaction"
+    assert tx["contexts"]["trace"]["origin"] == "manual"
+
+    chat_spans = list(x for x in tx["spans"] if x["op"] == "gen_ai.chat")
+    assert len(chat_spans) == 1
+    assert chat_spans[0]["origin"] == "auto.ai.langchain"
+
+    # Token usage is only available in newer versions of langchain (v0.2+)
+    # where usage_metadata is supported on AIMessageChunk
+    if "gen_ai.usage.input_tokens" in chat_spans[0]["data"]:
+        assert chat_spans[0]["data"]["gen_ai.usage.input_tokens"] == 10
+        assert chat_spans[0]["data"]["gen_ai.usage.output_tokens"] == 20
+        assert chat_spans[0]["data"]["gen_ai.usage.total_tokens"] == 30
+
+    if send_default_pii and include_prompts:
+        assert (
+            chat_spans[0]["data"][SPANDATA.GEN_AI_RESPONSE_TEXT]
+            == "Hello, how can I help you?"
+        )
+
+        param_id = request.node.callspec.id
+        if "string" in param_id:
+            assert [
+                {
+                    "type": "text",
+                    "content": "You are very powerful assistant, but don't know current events",
+                }
+            ] == json.loads(chat_spans[0]["data"][SPANDATA.GEN_AI_SYSTEM_INSTRUCTIONS])
+        else:
+            assert [
+                {
+                    "type": "text",
+                    "content": "You are a helpful assistant.",
+                },
+                {
+                    "type": "text",
+                    "content": "Be concise and clear.",
+                },
+            ] == json.loads(chat_spans[0]["data"][SPANDATA.GEN_AI_SYSTEM_INSTRUCTIONS])
+    else:
+        assert SPANDATA.GEN_AI_SYSTEM_INSTRUCTIONS not in chat_spans[0].get("data", {})
+        assert SPANDATA.GEN_AI_REQUEST_MESSAGES not in chat_spans[0].get("data", {})
+        assert SPANDATA.GEN_AI_RESPONSE_TEXT not in chat_spans[0].get("data", {})
+
+
 @pytest.mark.parametrize(
     "send_default_pii, include_prompts, use_unknown_llm_type",
     [
@@ -102,7 +225,7 @@ def _llm_type(self) -> str:
     ],
     ids=["string", "list", "blocks"],
 )
-def test_langchain_agent(
+def test_langchain_openai_tools_agent(
     sentry_init,
     capture_events,
     send_default_pii,