Skip to content

Commit 18a7037

Browse files
areibmanareibman
authored
Examples integration tests (#1135)
* examples with validators + new validationcheck logic * pytest fix * readme * fix langgraph * validator fix * check for span kinds * model update * model update * fix validator * update examples * add secrets * workflow secrets * requirements for each dir * reqs * fix tests * fix tests? * add reqs * shorten test * shorten test * tests * add requirements and fix broken notebooks * add tags * add trace naming * pre-commit checks * add trace naming
1 parent fceaab9 commit 18a7037

64 files changed

Lines changed: 1786 additions & 143 deletions

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

‎.github/workflows/examples-integration-test.yml‎

Lines changed: 192 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,192 @@
1+
name: Examples Integration Test
2+
3+
# This workflow runs all example scripts to ensure they work correctly
4+
# and that LLM spans are properly tracked in AgentOps using the
5+
# integrated validation functionality.
6+
7+
on:
8+
push:
9+
branches: [ main, develop ]
10+
paths:
11+
- 'examples/**/*.py'
12+
- 'agentops/**'
13+
- '.github/workflows/examples-integration-test.yml'
14+
pull_request:
15+
branches: [ main, develop ]
16+
paths:
17+
- 'examples/**/*.py'
18+
- 'agentops/**'
19+
- '.github/workflows/examples-integration-test.yml'
20+
workflow_dispatch:
21+
22+
env:
23+
PYTHON_VERSION: '3.11'
24+
25+
jobs:
26+
test-examples:
27+
runs-on: ubuntu-latest
28+
timeout-minutes: 30
29+
30+
strategy:
31+
fail-fast: false
32+
matrix:
33+
example:
34+
# OpenAI examples
35+
- { path: 'examples/openai/openai_example_sync.py', name: 'OpenAI Sync' }
36+
- { path: 'examples/openai/openai_example_async.py', name: 'OpenAI Async' }
37+
- { path: 'examples/openai/multi_tool_orchestration.py', name: 'OpenAI Multi-Tool' }
38+
- { path: 'examples/openai/web_search.py', name: 'OpenAI Web Search' }
39+
40+
# Anthropic examples
41+
- { path: 'examples/anthropic/anthropic-example-sync.py', name: 'Anthropic Sync' }
42+
- { path: 'examples/anthropic/anthropic-example-async.py', name: 'Anthropic Async' }
43+
- { path: 'examples/anthropic/agentops-anthropic-understanding-tools.py', name: 'Anthropic Tools' }
44+
45+
# LangChain examples
46+
- { path: 'examples/langchain/langchain_examples.py', name: 'LangChain' }
47+
48+
# LiteLLM examples
49+
- { path: 'examples/litellm/litellm_example.py', name: 'LiteLLM' }
50+
51+
# Google Generative AI examples
52+
- { path: 'examples/google_genai/gemini_example.py', name: 'Google Gemini' }
53+
54+
# xAI examples
55+
- { path: 'examples/xai/grok_examples.py', name: 'xAI Grok' }
56+
- { path: 'examples/xai/grok_vision_examples.py', name: 'xAI Grok Vision' }
57+
58+
# CrewAI examples
59+
- { path: 'examples/crewai/job_posting.py', name: 'CrewAI Job Posting' }
60+
- { path: 'examples/crewai/markdown_validator.py', name: 'CrewAI Markdown' }
61+
62+
# AutoGen examples
63+
- { path: 'examples/autogen/AgentChat.py', name: 'AutoGen Agent Chat' }
64+
- { path: 'examples/autogen/MathAgent.py', name: 'AutoGen Math Agent' }
65+
66+
# AG2 examples
67+
- { path: 'examples/ag2/async_human_input.py', name: 'AG2 Async Human Input' }
68+
- { path: 'examples/ag2/tools_wikipedia_search.py', name: 'AG2 Wikipedia Search' }
69+
70+
# Context Manager examples
71+
- { path: 'examples/context_manager/basic_usage.py', name: 'Context Manager Basic' }
72+
- { path: 'examples/context_manager/error_handling.py', name: 'Context Manager Errors' }
73+
- { path: 'examples/context_manager/parallel_traces.py', name: 'Context Manager Parallel' }
74+
- { path: 'examples/context_manager/production_patterns.py', name: 'Context Manager Production' }
75+
76+
# Agno examples
77+
- { path: 'examples/agno/agno_async_operations.py', name: 'Agno Async Operations' }
78+
- { path: 'examples/agno/agno_basic_agents.py', name: 'Agno Basic Agents' }
79+
- { path: 'examples/agno/agno_research_team.py', name: 'Agno Research Team' }
80+
- { path: 'examples/agno/agno_tool_integrations.py', name: 'Agno Tool Integrations' }
81+
- { path: 'examples/agno/agno_workflow_setup.py', name: 'Agno Workflow Setup' }
82+
83+
# Google ADK examples
84+
- { path: 'examples/google_adk/human_approval.py', name: 'Google ADK Human Approval' }
85+
86+
# LlamaIndex examples
87+
- { path: 'examples/llamaindex/llamaindex_example.py', name: 'LlamaIndex' }
88+
89+
# Mem0 examples
90+
- { path: 'examples/mem0/mem0_memoryclient_example.py', name: 'Mem0 Memory Client' }
91+
92+
# Watsonx examples
93+
- { path: 'examples/watsonx/watsonx-streaming.py', name: 'Watsonx Streaming' }
94+
- { path: 'examples/watsonx/watsonx-text-chat.py', name: 'Watsonx Text Chat' }
95+
- { path: 'examples/watsonx/watsonx-tokeniation-model.py', name: 'Watsonx Tokenization' }
96+
97+
# LangGraph examples
98+
- { path: 'examples/langgraph/langgraph_example.py', name: 'LangGraph' }
99+
100+
# Smolagents examples
101+
- { path: 'examples/smolagents/multi_smolagents_system.py', name: 'Smolagents Multi System' }
102+
- { path: 'examples/smolagents/text_to_sql.py', name: 'Smolagents Text to SQL' }
103+
104+
# OpenAI Agents examples
105+
- { path: 'examples/openai_agents/agent_guardrails.py', name: 'OpenAI Agents Guardrails' }
106+
- { path: 'examples/openai_agents/agent_patterns.py', name: 'OpenAI Agents Patterns' }
107+
- { path: 'examples/openai_agents/agents_tools.py', name: 'OpenAI Agents Tools' }
108+
- { path: 'examples/openai_agents/customer_service_agent.py', name: 'OpenAI Agents Customer Service' }
109+
110+
# Add more examples as needed
111+
112+
steps:
113+
- uses: actions/checkout@v4
114+
115+
- name: Set up Python
116+
uses: actions/setup-python@v4
117+
with:
118+
python-version: ${{ env.PYTHON_VERSION }}
119+
120+
- name: Install AgentOps
121+
run: |
122+
pip install -e .
123+
124+
- name: Install example dependencies
125+
run: |
126+
# Install common dependencies
127+
pip install python-dotenv requests
128+
129+
# Install from requirements.txt in the example's directory
130+
example_dir=$(dirname "${{ matrix.example.path }}")
131+
if [ -f "$example_dir/requirements.txt" ]; then
132+
echo "Installing dependencies from $example_dir/requirements.txt"
133+
pip install -r "$example_dir/requirements.txt"
134+
else
135+
echo "No requirements.txt found in $example_dir"
136+
fi
137+
138+
- name: Run example - ${{ matrix.example.name }}
139+
env:
140+
AGENTOPS_API_KEY: ${{ secrets.AGENTOPS_API_KEY }}
141+
OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
142+
ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }}
143+
GOOGLE_API_KEY: ${{ secrets.GOOGLE_API_KEY }}
144+
XAI_API_KEY: ${{ secrets.XAI_API_KEY }}
145+
WATSONX_API_KEY: ${{ secrets.WATSONX_API_KEY }}
146+
WATSONX_PROJECT_ID: ${{ secrets.WATSONX_PROJECT_ID }}
147+
WATSONX_URL: ${{ secrets.WATSONX_URL }}
148+
MEM0_API_KEY: ${{ secrets.MEM0_API_KEY }}
149+
GEMINI_API_KEY: ${{ secrets.GEMINI_API_KEY }}
150+
COHERE_API_KEY: ${{ secrets.COHERE_API_KEY }}
151+
GROQ_API_KEY: ${{ secrets.GROQ_API_KEY }}
152+
FIREWORKS_API_KEY: ${{ secrets.FIREWORKS_API_KEY }}
153+
MISTRAL_API_KEY: ${{ secrets.MISTRAL_API_KEY }}
154+
AI21_API_KEY: ${{ secrets.AI21_API_KEY }}
155+
TAVILY_API_KEY: ${{ secrets.TAVILY_API_KEY }}
156+
EXA_API_KEY: ${{ secrets.EXA_API_KEY }}
157+
LLAMA_API_KEY: ${{ secrets.LLAMA_API_KEY }}
158+
PERPLEXITY_API_KEY: ${{ secrets.PERPLEXITY_API_KEY }}
159+
REPLICATE_API_TOKEN: ${{ secrets.REPLICATE_API_TOKEN }}
160+
PYTHONPATH: ${{ github.workspace }}
161+
run: |
162+
echo "Running ${{ matrix.example.name }}..."
163+
python "${{ matrix.example.path }}" || exit 1
164+
165+
- name: Check for errors
166+
if: failure()
167+
run: |
168+
echo "Example ${{ matrix.example.name }} failed!"
169+
echo "Path: ${{ matrix.example.path }}"
170+
171+
# Show last 50 lines of any log files
172+
if [ -f agentops.log ]; then
173+
echo "=== AgentOps Log ==="
174+
tail -n 50 agentops.log
175+
fi
176+
177+
summary:
178+
needs: test-examples
179+
runs-on: ubuntu-latest
180+
if: always()
181+
182+
steps:
183+
- name: Summary
184+
run: |
185+
echo "## Examples Integration Test Summary" >> $GITHUB_STEP_SUMMARY
186+
echo "" >> $GITHUB_STEP_SUMMARY
187+
188+
if [ "${{ needs.test-examples.result }}" == "success" ]; then
189+
echo "✅ All examples passed!" >> $GITHUB_STEP_SUMMARY
190+
else
191+
echo "❌ Some examples failed. Check the logs above." >> $GITHUB_STEP_SUMMARY
192+
fi

‎agentops/__init__.py‎

Lines changed: 19 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -34,6 +34,9 @@
3434
from agentops.helpers.deprecation import deprecated, warn_deprecated_param
3535
import threading
3636

37+
# Import validation functions
38+
from agentops.validation import validate_trace_spans, print_validation_summary, ValidationError
39+
3740
# Thread-safe client management
3841
_client_lock = threading.Lock()
3942
_client = None
@@ -442,37 +445,40 @@ def extract_key_from_attr(attr_value: str) -> str:
442445

443446

444447
__all__ = [
445-
"init",
446-
"configure",
447-
"get_client",
448-
"record",
449-
"start_trace",
450-
"end_trace",
451-
"update_trace_metadata",
448+
# Legacy exports
452449
"start_session",
453450
"end_session",
454451
"track_agent",
455452
"track_tool",
456453
"end_all_sessions",
454+
"Session",
457455
"ToolEvent",
458456
"ErrorEvent",
459457
"ActionEvent",
460458
"LLMEvent",
461-
"Session",
459+
# Modern exports
460+
"init",
461+
"start_trace",
462+
"end_trace",
463+
"update_trace_metadata",
464+
"Client",
465+
"get_client",
466+
# Decorators
462467
"trace",
463468
"session",
464469
"agent",
465470
"task",
466471
"workflow",
467472
"operation",
468-
"guardrail",
469-
"tracer",
470473
"tool",
471-
# Trace state enums
474+
"guardrail",
475+
# Enums
472476
"TraceState",
473477
"SUCCESS",
474478
"ERROR",
475479
"UNSET",
476-
# OpenTelemetry status codes (for advanced users)
477-
"StatusCode",
480+
# Validation
481+
"validate_trace_spans",
482+
"print_validation_summary",
483+
"ValidationError",
478484
]

0 commit comments

Comments
 (0)