Merge branch 'main' into feat/grpc-receiver-support

Author: ajimenez1503

README.md

@@ -26,6 +26,7 @@ agentevals accepts OTLP/HTTP on port 4318 (`http/protobuf` and `http/json`) and
 | Example | Framework | LLM Provider |
 |---------|-----------|-------------|
 | [zero-code-examples/langchain/](./zero-code-examples/langchain/) | LangChain | OpenAI |
+| [zero-code-examples/ollama/](./zero-code-examples/ollama/) | LangChain | Ollama |
 | [zero-code-examples/strands/](./zero-code-examples/strands/) | Strands | OpenAI |
 | [zero-code-examples/adk/](./zero-code-examples/adk/) | Google ADK | Gemini |
 
@@ -99,6 +100,7 @@ Detection checks for `gen_ai.request.model` / `gen_ai.input.messages` (GenAI sem
 | Example | Framework | LLM Provider | Instrumentation | Content Delivery |
 |---------|-----------|-------------|-----------------|-----------------|
 | [zero-code-examples/langchain/](./zero-code-examples/langchain/) | LangChain | OpenAI | GenAI semconv (logs) | Standard OTLP export |
+| [zero-code-examples/ollama/](./zero-code-examples/ollama/) | LangChain | Ollama | GenAI semconv (logs) | Standard OTLP export |
 | [zero-code-examples/strands/](./zero-code-examples/strands/) | Strands | OpenAI | GenAI semconv (events*) | Standard OTLP export |
 | [zero-code-examples/adk/](./zero-code-examples/adk/) | Google ADK | Gemini | ADK built-in | Standard OTLP export |
 | [langchain_agent](./langchain_agent/) | LangChain | OpenAI | GenAI semconv (logs) | SDK WebSocket |
@@ -212,6 +214,7 @@ cd ui && npm run dev
 ```bash
 # Zero-code OTLP (recommended):
 python examples/zero-code-examples/langchain/run.py
+python examples/zero-code-examples/ollama/run.py
 python examples/zero-code-examples/strands/run.py
 python examples/zero-code-examples/adk/run.py
 

examples/README.md

@@ -0,0 +1,7 @@
+langchain>=1.2.0
+langchain-openai>=1.1.10
+
+opentelemetry-sdk>=1.36.0
+opentelemetry-exporter-otlp-proto-http>=1.36.0
+opentelemetry-instrumentation-openai-v2
+python-dotenv>=1.0.0

examples/zero-code-examples/ollama/requirements.txt

@@ -0,0 +1,194 @@
+"""Run the LangChain dice agent against local Ollama via ChatOpenAI with OTLP export.
+
+Demonstrates zero-code integration with a local LLM provider using Ollama's
+OpenAI-compatible endpoint. The agent emits standard OpenTelemetry spans/logs
+and sends them to agentevals via OTLP, without using the agentevals Python SDK
+in agent code.
+
+Prerequisites:
+    1. pip install -r requirements.txt
+    2. agentevals serve --dev
+    3. ollama serve
+    4. ollama pull llama3.2:3b
+
+Usage:
+    python examples/zero-code-examples/ollama/run.py
+
+Optional environment variables:
+    LOCAL_OPENAI_BASE_URL (default: http://localhost:11434/v1)
+    LOCAL_LLM_MODEL (default: llama3.2:3b)
+"""
+
+import json
+import os
+import sys
+
+from dotenv import load_dotenv
+from langchain_core.messages import HumanMessage, ToolMessage
+from langchain_openai import ChatOpenAI
+from opentelemetry import trace
+from opentelemetry._logs import set_logger_provider
+from opentelemetry.exporter.otlp.proto.http._log_exporter import OTLPLogExporter
+from opentelemetry.exporter.otlp.proto.http.trace_exporter import OTLPSpanExporter
+from opentelemetry.instrumentation.openai_v2 import OpenAIInstrumentor
+from opentelemetry.sdk._logs import LoggerProvider
+from opentelemetry.sdk._logs.export import BatchLogRecordProcessor
+from opentelemetry.sdk.resources import Resource
+from opentelemetry.sdk.trace import TracerProvider
+from opentelemetry.sdk.trace.export import BatchSpanProcessor
+
+sys.path.insert(0, os.path.join(os.path.dirname(__file__), "..", "..", "langchain_agent"))
+from agent import check_prime, roll_die
+
+load_dotenv(override=True)
+
+
+def _json_or_value(value):
+    if not isinstance(value, str):
+        return value
+    try:
+        return json.loads(value)
+    except json.JSONDecodeError:
+        return value
+
+
+def _to_int_if_numeric(value):
+    value = _json_or_value(value)
+    if isinstance(value, (int, float)):
+        return int(value)
+    if isinstance(value, str):
+        stripped = value.strip()
+        if stripped.lstrip("-").isdigit():
+            return int(stripped)
+        if "," in stripped:
+            parts = [p.strip() for p in stripped.split(",") if p.strip()]
+            if parts and all(p.lstrip("-").isdigit() for p in parts):
+                return [int(p) for p in parts]
+    return value
+
+
+def _normalize_nums(values):
+    parsed = _json_or_value(values)
+    if not isinstance(parsed, list):
+        parsed = [parsed]
+
+    normalized = []
+    for item in parsed:
+        item = _to_int_if_numeric(item)
+        if isinstance(item, list):
+            normalized.extend(item)
+        else:
+            normalized.append(item)
+    return normalized
+
+
+def _normalize_tool_args(tool_name: str, tool_args):
+    parsed = _json_or_value(tool_args)
+    if tool_name == "roll_die":
+        if isinstance(parsed, dict):
+            if "sides" in parsed:
+                parsed["sides"] = _to_int_if_numeric(parsed["sides"])
+            return parsed
+        return {"sides": _to_int_if_numeric(parsed)}
+
+    if tool_name == "check_prime":
+        if isinstance(parsed, dict):
+            parsed["nums"] = _normalize_nums(parsed.get("nums", []))
+            return parsed
+        if isinstance(parsed, list):
+            return {"nums": _normalize_nums(parsed)}
+        return {"nums": _normalize_nums(parsed)}
+
+    return parsed
+
+
+def main():
+    endpoint = os.environ.get("OTEL_EXPORTER_OTLP_ENDPOINT", "http://localhost:4318")
+    base_url = os.environ.get("LOCAL_OPENAI_BASE_URL", "http://localhost:11434/v1")
+    model = os.environ.get("LOCAL_LLM_MODEL", "llama3.2:3b")
+
+    print(f"OTLP endpoint: {endpoint}")
+    print(f"Local model endpoint: {base_url}")
+    print(f"Local model: {model}")
+
+    os.environ["OTEL_INSTRUMENTATION_GENAI_CAPTURE_MESSAGE_CONTENT"] = "true"
+    os.environ.setdefault(
+        "OTEL_RESOURCE_ATTRIBUTES",
+        "agentevals.eval_set_id=langchain_local_ollama_openai_eval,agentevals.session_name=langchain-ollama-openai-zero-code",
+    )
+
+    resource = Resource.create()
+
+    tracer_provider = TracerProvider(resource=resource)
+    tracer_provider.add_span_processor(BatchSpanProcessor(OTLPSpanExporter(), schedule_delay_millis=1000))
+    trace.set_tracer_provider(tracer_provider)
+
+    logger_provider = LoggerProvider(resource=resource)
+    logger_provider.add_log_record_processor(BatchLogRecordProcessor(OTLPLogExporter(), schedule_delay_millis=1000))
+    set_logger_provider(logger_provider)
+
+    OpenAIInstrumentor().instrument()
+
+    llm = ChatOpenAI(model=model, temperature=0.0, base_url=base_url, api_key="ollama")
+    tools = [roll_die, check_prime]
+    llm_with_tools = llm.bind_tools(tools)
+
+    test_queries = [
+        "Hi! Can you help me?",
+        "Roll a 20-sided die for me",
+        "Is the number you rolled prime?",
+    ]
+
+    messages = []
+
+    for i, query in enumerate(test_queries, 1):
+        print(f"\n[{i}/{len(test_queries)}] User: {query}")
+        messages.append(HumanMessage(content=query))
+
+        max_iterations = 5
+        for _ in range(max_iterations):
+            response = llm_with_tools.invoke(messages)
+            messages.append(response)
+
+            if not response.tool_calls:
+                print(f"     Agent: {response.content}")
+                break
+
+            for tool_call in response.tool_calls:
+                tool_name = tool_call["name"]
+                tool_args = tool_call["args"]
+                tool_call_id = tool_call.get("id", f"tool-call-{tool_name}")
+                normalized_args = _normalize_tool_args(tool_name, tool_args)
+
+                selected_tool = {t.name: t for t in tools}.get(tool_name)
+                if selected_tool:
+                    try:
+                        tool_result = selected_tool.invoke(normalized_args)
+                    except Exception as exc:
+                        tool_result = {
+                            "isError": True,
+                            "error": str(exc),
+                            "tool_name": tool_name,
+                            "args": normalized_args,
+                        }
+
+                    tool_content = json.dumps(tool_result) if isinstance(tool_result, dict) else str(tool_result)
+                    messages.append(ToolMessage(content=tool_content, tool_call_id=tool_call_id))
+                else:
+                    messages.append(
+                        ToolMessage(
+                            content=json.dumps({"isError": True, "error": f"Unknown tool: {tool_name}"}),
+                            tool_call_id=tool_call_id,
+                        )
+                    )
+        else:
+            print("     Agent: [Max iterations reached]")
+
+    print()
+    tracer_provider.force_flush()
+    logger_provider.force_flush()
+    print("All traces and logs flushed to OTLP receiver.")
+
+
+if __name__ == "__main__":
+    main()

examples/zero-code-examples/ollama/run.py

@@ -507,6 +507,7 @@ async def evaluate_traces(
             trace_format=trace_format,
             judge_model=config_dict.get("judgeModel"),
             threshold=threshold,
+            trajectory_match_type=config_dict.get("trajectoryMatchType"),
         )
 
         logger.info(f"Evaluating {len(trace_paths)} trace file(s) with metrics: {metrics}")
@@ -615,6 +616,7 @@ async def event_generator():
                 trace_format=trace_format,
                 judge_model=config_dict.get("judgeModel"),
                 threshold=threshold,
+                trajectory_match_type=config_dict.get("trajectoryMatchType"),
             )
 
             loader = get_loader(eval_config.trace_format)

src/agentevals/api/routes.py

@@ -5,7 +5,7 @@
 import asyncio
 import json
 import logging
-from typing import TYPE_CHECKING
+from typing import TYPE_CHECKING, Literal
 
 from fastapi import APIRouter, Depends, HTTPException
 from fastapi.responses import FileResponse
@@ -46,6 +46,7 @@ class EvaluateSessionsRequest(BaseModel):
     eval_set_id: str
     metrics: list[str] = ["tool_trajectory_avg_score"]
     judge_model: str = "gemini-2.5-flash"
+    trajectory_match_type: Literal["EXACT", "IN_ORDER", "ANY_ORDER"] | None = None
 
 
 class PrepareEvaluationRequest(BaseModel):
@@ -210,6 +211,7 @@ async def eval_one_session(session_id: str, session) -> SessionEvalResult:
                         eval_set_file=eval_set_file.name,
                         metrics=request.metrics,
                         judge_model=request.judge_model,
+                        trajectory_match_type=request.trajectory_match_type,
                     )
 
                     eval_result = await run_evaluation(config)

src/agentevals/api/streaming_routes.py

@@ -60,14 +60,18 @@ def build_eval_metric(
     judge_model: str | None,
     threshold: float | None,
     rubrics: list[str] | None = None,
+    match_type: str | None = None,
 ) -> EvalMetric:
     """Construct an ADK ``EvalMetric`` with the appropriate criterion."""
     effective_threshold = threshold if threshold is not None else 0.5
 
     criterion: BaseCriterion | None = None
 
     if metric_name == "tool_trajectory_avg_score":
-        criterion = ToolTrajectoryCriterion(threshold=effective_threshold)
+        _match = (
+            ToolTrajectoryCriterion.MatchType[match_type] if match_type else ToolTrajectoryCriterion.MatchType.EXACT
+        )
+        criterion = ToolTrajectoryCriterion(threshold=effective_threshold, match_type=_match)
     elif metric_name == "final_response_match_v2":
         judge_opts = JudgeModelOptions()
         if judge_model:
@@ -105,7 +109,11 @@ def build_eval_metric(
             threshold=effective_threshold,
             judge_model_options=judge_opts,
         )
-    elif metric_name in ("response_match_score", "response_evaluation_score", "safety_v1"):
+    elif metric_name in (
+        "response_match_score",
+        "response_evaluation_score",
+        "safety_v1",
+    ):
         criterion = BaseCriterion(threshold=effective_threshold)
 
     return EvalMetric(
@@ -179,6 +187,7 @@ async def evaluate_builtin_metric(
     expected_invocations: list[Invocation] | None,
     judge_model: str | None,
     threshold: float | None,
+    match_type: str | None = None,
 ) -> dict[str, Any]:
     """Evaluate a single built-in ADK metric.
 
@@ -197,7 +206,7 @@ async def evaluate_builtin_metric(
         )
 
     try:
-        eval_metric = build_eval_metric(metric_name, judge_model, threshold)
+        eval_metric = build_eval_metric(metric_name, judge_model, threshold, match_type=match_type)
         evaluator: Evaluator = get_evaluator(eval_metric)
 
         if inspect.iscoroutinefunction(evaluator.evaluate_invocations):

src/agentevals/builtin_metrics.py

@@ -109,6 +109,12 @@ def main(verbose: int) -> None:
     default=None,
     help="Score threshold for pass/fail.",
 )
+@click.option(
+    "--trajectory-match-type",
+    type=click.Choice(["EXACT", "IN_ORDER", "ANY_ORDER"], case_sensitive=False),
+    default=None,
+    help="Match type for tool_trajectory_avg_score: EXACT (default), IN_ORDER, or ANY_ORDER.",
+)
 @click.option(
     "--output",
     "-o",
@@ -131,6 +137,7 @@ def run(
     trace_format: str,
     judge_model: str | None,
     threshold: float | None,
+    trajectory_match_type: str | None,
     output: str,
     config_file: str | None,
 ) -> None:
@@ -153,6 +160,7 @@ def run(
             trace_format=trace_format,
             judge_model=judge_model,
             threshold=threshold,
+            trajectory_match_type=trajectory_match_type,
             output_format=output,
         )
         config = merge_configs(file_config, cli_config)
@@ -165,6 +173,7 @@ def run(
             trace_format=trace_format,
             judge_model=judge_model,
             threshold=threshold,
+            trajectory_match_type=trajectory_match_type,
             output_format=output,
         )
 

src/agentevals/cli.py

@@ -132,6 +132,19 @@ class EvalRunConfig(BaseModel):
         description="Score threshold for pass/fail.",
     )
 
+    trajectory_match_type: str | None = Field(
+        default=None,
+        description="Match type for tool_trajectory_avg_score: 'EXACT', 'IN_ORDER', or 'ANY_ORDER'. Default: EXACT.",
+    )
+
+    @field_validator("trajectory_match_type")
+    @classmethod
+    def _validate_trajectory_match_type(cls, v: str | None) -> str | None:
+        valid = {"EXACT", "IN_ORDER", "ANY_ORDER"}
+        if v is not None and v.upper() not in valid:
+            raise ValueError(f"Invalid trajectory_match_type '{v}'. Valid values: {sorted(valid)}")
+        return v.upper() if v is not None else v
+
     output_format: str = Field(
         default="table",
         description="Output format: 'table', 'json', or 'summary'.",

src/agentevals/config.py

@@ -111,6 +111,8 @@ def load_eval_config(path: str | Path) -> EvalRunConfig:
         config.judge_model = data["judge_model"]
     if "threshold" in data:
         config.threshold = float(data["threshold"])
+    if "trajectory_match_type" in data:
+        config.trajectory_match_type = data["trajectory_match_type"]
     if "trace_format" in data:
         config.trace_format = data["trace_format"]
 
@@ -136,6 +138,8 @@ def merge_configs(file_config: EvalRunConfig, cli_config: EvalRunConfig) -> Eval
         merged.judge_model = cli_config.judge_model
     if cli_config.threshold is not None:
         merged.threshold = cli_config.threshold
+    if cli_config.trajectory_match_type is not None:
+        merged.trajectory_match_type = cli_config.trajectory_match_type
     if cli_config.trace_format != "jaeger-json":
         merged.trace_format = cli_config.trace_format
     if cli_config.output_format != "table":