feat: agent evals — retrieval, scoring, and eval canaries (#91)

Author: kylehounslow

.env

@@ -7,6 +7,7 @@
 # Include additional compose files (comment out to disable)
 INCLUDE_COMPOSE_EXAMPLES=docker-compose.examples.yml
 # INCLUDE_COMPOSE_OTEL_DEMO=docker-compose.otel-demo.yml
+# INCLUDE_COMPOSE_AGENT_EVAL_LLM=docker-compose.agent-eval-llm.yml
 INCLUDE_COMPOSE_LOCAL_OPENSEARCH=docker-compose.local-opensearch.yml
 INCLUDE_COMPOSE_LOCAL_OPENSEARCH_DASHBOARDS=docker-compose.local-opensearch-dashboards.yml
 
@@ -69,6 +70,10 @@ NETWORK_NAME=observability-stack-network
 WEATHER_AGENT_PORT=8000
 CANARY_INTERVAL=120
 
+# Agent Eval Canary Configuration
+EVAL_CANARY_INTERVAL=120
+EVAL_CANARY_LOOKBACK_MINUTES=15
+
 # ******************
 # OpenTelemetry Demo Configuration
 # ******************

docker-compose.agent-eval-llm.yml

@@ -0,0 +1,44 @@
+# LLM Eval Canary
+# Periodically runs LLM-as-judge (Bedrock Claude) on agent traces.
+# Requires AWS credentials via AWS_* env vars.
+#
+# Enable by uncommenting INCLUDE_COMPOSE_AGENT_EVAL_LLM in .env
+
+x-default-logging: &logging
+  driver: "json-file"
+  options:
+    max-size: "5m"
+    max-file: "2"
+    tag: "{{.Name}}"
+
+services:
+  example-agent-eval-canary-llm:
+    build:
+      context: ./docker-compose/agent-eval-canary-llm
+      dockerfile: Dockerfile
+    container_name: agent-eval-canary-llm
+    environment:
+      - OPENSEARCH_HOST=https://${OPENSEARCH_HOST}:${OPENSEARCH_PORT}
+      - OPENSEARCH_USER=${OPENSEARCH_USER}
+      - OPENSEARCH_PASSWORD=${OPENSEARCH_PASSWORD}
+      - OTEL_EXPORTER_OTLP_ENDPOINT=${OTEL_COLLECTOR_HOST}:${OTEL_COLLECTOR_PORT_GRPC}
+      - EVAL_CANARY_LLM_INTERVAL=${EVAL_CANARY_LLM_INTERVAL:-60}
+      - EVAL_CANARY_LLM_LOOKBACK_MINUTES=${EVAL_CANARY_LLM_LOOKBACK_MINUTES:-10}
+      - EVAL_CANARY_LLM_MAX_PER_CYCLE=${EVAL_CANARY_LLM_MAX_PER_CYCLE:-20}
+      - EVAL_CANARY_LLM_CONCURRENCY=${EVAL_CANARY_LLM_CONCURRENCY:-8}
+      - EVAL_JUDGE_MODEL=${EVAL_JUDGE_MODEL:-us.anthropic.claude-sonnet-4-20250514-v1:0}
+      - AWS_ACCESS_KEY_ID=${AWS_ACCESS_KEY_ID:-}
+      - AWS_SECRET_ACCESS_KEY=${AWS_SECRET_ACCESS_KEY:-}
+      - AWS_SESSION_TOKEN=${AWS_SESSION_TOKEN:-}
+      - AWS_REGION=${AWS_REGION:-us-west-2}
+    depends_on:
+      - opensearch
+      - otel-collector
+    networks:
+      - observability-stack-network
+    restart: unless-stopped
+    deploy:
+      resources:
+        limits:
+          memory: 512M
+    logging: *logging

docker-compose.examples.yml

@@ -136,3 +136,28 @@ services:
         limits:
           memory: 128M
     logging: *logging
+
+  # Agent Eval Canary: periodically scores un-evaluated agent traces
+  example-agent-eval-canary:
+    build:
+      context: ./docker-compose/agent-eval-canary
+      dockerfile: Dockerfile
+    container_name: agent-eval-canary
+    environment:
+      - OPENSEARCH_HOST=https://${OPENSEARCH_HOST}:${OPENSEARCH_PORT}
+      - OPENSEARCH_USER=${OPENSEARCH_USER}
+      - OPENSEARCH_PASSWORD=${OPENSEARCH_PASSWORD}
+      - OTEL_EXPORTER_OTLP_ENDPOINT=${OTEL_COLLECTOR_HOST}:${OTEL_COLLECTOR_PORT_GRPC}
+      - EVAL_CANARY_INTERVAL=${EVAL_CANARY_INTERVAL}
+      - EVAL_CANARY_LOOKBACK_MINUTES=${EVAL_CANARY_LOOKBACK_MINUTES}
+    depends_on:
+      - opensearch
+      - otel-collector
+    networks:
+      - observability-stack-network
+    restart: unless-stopped
+    deploy:
+      resources:
+        limits:
+          memory: ${CANARY_MEMORY_LIMIT}
+    logging: *logging

docker-compose.yml

@@ -6,6 +6,7 @@ name: observability-stack
 include:
   - path: ${INCLUDE_COMPOSE_EXAMPLES:-docker-compose/util/docker-compose.empty.yml}
   - path: ${INCLUDE_COMPOSE_OTEL_DEMO:-docker-compose/util/docker-compose.empty.yml}
+  - path: ${INCLUDE_COMPOSE_AGENT_EVAL_LLM:-docker-compose/util/docker-compose.empty.yml}
   - path: ${INCLUDE_COMPOSE_LOCAL_OPENSEARCH:-docker-compose/util/docker-compose.empty.yml}
   - path: ${INCLUDE_COMPOSE_LOCAL_OPENSEARCH_DASHBOARDS:-docker-compose/util/docker-compose.empty.yml}
 

docker-compose/agent-eval-canary-llm/Dockerfile

@@ -0,0 +1,8 @@
+FROM python:3.12-slim
+RUN apt-get update && apt-get install -y --no-install-recommends git && rm -rf /var/lib/apt/lists/*
+COPY --from=ghcr.io/astral-sh/uv:latest /uv /usr/local/bin/uv
+WORKDIR /app
+COPY pyproject.toml ./
+RUN uv sync --no-dev
+COPY eval_canary_llm.py .
+CMD ["uv", "run", "eval_canary_llm.py"]

docker-compose/agent-eval-canary-llm/README.md

@@ -0,0 +1,53 @@
+# LLM Eval Canary (LLM-as-Judge)
+
+Periodically evaluates agent traces using HelpfulnessEvaluator (Bedrock Claude). Opt-in — requires AWS credentials.
+
+Enable by uncommenting `INCLUDE_COMPOSE_AGENT_EVAL_LLM` in `.env`.
+
+## Evaluation Signals
+
+Emits **1 evaluation span** per trace:
+
+| Eval Name | What it measures | Value |
+|---|---|---|
+| `helpfulness` | LLM judges whether the agent's response addressed the user's request | 0.0–1.0 (7-point scale) |
+
+## Configuration
+
+| Env Var | Default | Description |
+|---|---|---|
+| `EVAL_CANARY_LLM_INTERVAL` | `60` | Seconds between poll cycles |
+| `EVAL_CANARY_LLM_LOOKBACK_MINUTES` | `10` | How far back to search for traces |
+| `EVAL_CANARY_LLM_MAX_PER_CYCLE` | `20` | Max traces to evaluate per cycle |
+| `EVAL_CANARY_LLM_CONCURRENCY` | `8` | Parallel Bedrock calls |
+| `EVAL_JUDGE_MODEL` | `us.anthropic.claude-sonnet-4-20250514-v1:0` | Bedrock model ID |
+
+AWS credentials must be available in the environment (`AWS_ACCESS_KEY_ID`, `AWS_SECRET_ACCESS_KEY`, `AWS_SESSION_TOKEN`).
+
+## Running with the Deterministic Canary
+
+Both canaries can run simultaneously without duplicating scores. Each deduplicates by its own evaluation name — this canary checks for existing `helpfulness` scores, the deterministic canary checks for `span_coverage`. Neither blocks the other.
+
+A trace scored by both gets **5 evaluation spans**:
+
+```
+invoke_agent Events Agent
+├── evaluation span_coverage            (deterministic)
+├── evaluation error_free               (deterministic)
+├── evaluation tool_call_success_rate   (deterministic)
+├── evaluation tool_diversity           (deterministic)
+└── evaluation helpfulness              (LLM-as-judge)
+```
+
+## OTel Compliance
+
+All evaluation spans follow [OTel GenAI semantic conventions](https://opentelemetry.io/docs/specs/semconv/gen-ai/gen-ai-events/):
+
+- `gen_ai.operation.name = "evaluation"`
+- `gen_ai.evaluation.name = "helpfulness"`
+- `gen_ai.evaluation.score.value` — numeric score
+- `gen_ai.evaluation.score.label` — human-readable label (e.g. "Very helpful", "Somewhat unhelpful")
+- `gen_ai.evaluation.explanation` — LLM's reasoning
+- `gen_ai.evaluation.result` event emitted per span
+
+Score spans attach as children of the evaluated agent's `invoke_agent` span, so they appear in the same trace waterfall in OpenSearch Dashboards.