chore: use uv for Python dependency management

Signed-off-by: Kyle Hounslow <kylhouns@amazon.com>

Author: kylehounslow

docker-compose/agent-eval-canary/Dockerfile

@@ -1,7 +1,8 @@
 FROM python:3.12-slim
 RUN apt-get update && apt-get install -y --no-install-recommends git && rm -rf /var/lib/apt/lists/*
+COPY --from=ghcr.io/astral-sh/uv:latest /uv /usr/local/bin/uv
 WORKDIR /app
-COPY requirements.txt .
-RUN pip install --no-cache-dir -r requirements.txt
+COPY pyproject.toml uv.lock ./
+RUN uv sync --frozen --no-dev
 COPY eval_canary.py .
-CMD ["python", "-u", "eval_canary.py"]
+CMD ["uv", "run", "eval_canary.py"]

docker-compose/agent-eval-canary/eval_canary.py

@@ -1,3 +1,4 @@
+from datetime import datetime, timedelta, timezone
 """Agent eval canary — periodically scores un-evaluated agent traces.
 
 Polls OpenSearch for recent agent traces, skips any that already have
@@ -14,7 +15,7 @@
 
 from opentelemetry import trace
 from opentelemetry.sdk.trace import TracerProvider
-from opentelemetry.sdk.trace.export import BatchSpanProcessor
+from opentelemetry.sdk.trace.export import SimpleSpanProcessor
 from opentelemetry.exporter.otlp.proto.grpc.trace_exporter import OTLPSpanExporter
 from opentelemetry.sdk.resources import Resource
 
@@ -43,7 +44,7 @@ def setup_otel() -> TracerProvider:
     resource = Resource.create({"service.name": "agent-eval-canary"})
     provider = TracerProvider(resource=resource)
     provider.add_span_processor(
-        BatchSpanProcessor(OTLPSpanExporter(endpoint=OTEL_ENDPOINT, insecure=True))
+        SimpleSpanProcessor(OTLPSpanExporter(endpoint=OTEL_ENDPOINT, insecure=True))
     )
     trace.set_tracer_provider(provider)
     return provider
@@ -100,29 +101,40 @@ def run() -> None:
     # Wait for OpenSearch to be ready
     for attempt in range(30):
         try:
-            retriever.list_traces(since_minutes=1, max_results=1)
+            retriever.list_root_spans(max_results=1)
             log.info("OpenSearch is ready")
             break
         except Exception as e:
             log.info("Waiting for OpenSearch... (%d/30): %s", attempt + 1, e)
             time.sleep(10)
 
+    # Track recently scored traces to avoid duplicates from batch flush delay
+    recently_scored: dict[str, float] = {}  # trace_id -> timestamp
+
     while True:
         try:
-            roots = retriever.list_traces(
+            # Expire entries older than lookback window
+            cutoff = time.time() - (LOOKBACK_MINUTES * 60)
+            recently_scored = {k: v for k, v in recently_scored.items() if v > cutoff}
+
+            roots = retriever.list_root_spans(
                 services=TARGET_SERVICES,
-                since_minutes=LOOKBACK_MINUTES,
+                since=datetime.now(timezone.utc) - timedelta(minutes=LOOKBACK_MINUTES),
             )
             if roots:
                 trace_ids = [r.trace_id for r in roots]
                 evaluated = retriever.find_evaluated_trace_ids(trace_ids)
-                unevaluated = [r for r in roots if r.trace_id not in evaluated]
+                unevaluated = [
+                    r for r in roots
+                    if r.trace_id not in evaluated and r.trace_id not in recently_scored
+                ]
 
                 if unevaluated:
                     log.info("Found %d unevaluated traces", len(unevaluated))
                     for root in unevaluated:
                         try:
                             deterministic_eval(retriever, root.trace_id, root.span_id)
+                            recently_scored[root.trace_id] = time.time()
                         except Exception:
                             log.exception("Failed to eval trace %s", root.trace_id[:12])
                     provider.force_flush()

docker-compose/agent-eval-canary/pyproject.toml

@@ -0,0 +1,11 @@
+[project]
+name = "agent-eval-canary"
+version = "0.1.0"
+description = "Periodically scores un-evaluated agent traces in OpenSearch"
+requires-python = ">=3.10"
+dependencies = [
+    "opensearch-genai-observability-sdk-py[opensearch] @ git+https://github.com/kylehounslow/genai-observability-sdk-py.git@fd9560630ed376c31f1e15d5e60675d9514f02af",
+    "opentelemetry-api",
+    "opentelemetry-sdk",
+    "opentelemetry-exporter-otlp-proto-grpc",
+]

docker-compose/agent-eval-canary/requirements.txt

@@ -11,7 +11,7 @@ End-to-end evaluation loop: retrieve agent traces from OpenSearch, run LLM-as-ju
 ## Setup
 
 ```bash
-pip install -r requirements.txt
+uv sync
 ```
 
 ## Usage

docker-compose/agent-eval-canary/uv.lock

@@ -0,0 +1,11 @@
+[project]
+name = "agent-evals-genai-sdk"
+version = "0.1.0"
+description = "Run evaluations against stored agent traces in OpenSearch"
+readme = "README.md"
+requires-python = ">=3.10"
+dependencies = [
+    "opensearch-genai-observability-sdk-py[opensearch]",
+    "strands-agents",
+    "strands-agents-evals",
+]