fix: align OTEL attributes with gen_ai semconv spec (#3488)

* fix: use --build-local in smoke-otel-tracing to test latest api-proxy code

The workflow was using --skip-pull with pre-built GHCR images (v0.25.29)
which don't include the recently-merged otel.js module. Switch to
--build-local so the api-proxy container is built from source and
includes the OTEL tracing implementation.

Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>

* feat: inject stream_options for token tracking in spans

Streaming OpenAI/Copilot API responses only include usage data in the
final SSE chunk when the client sends stream_options: {include_usage: true}.
Without this, the token tracker never extracts usage and OTEL spans have
no gen_ai.usage.* attributes.

The api-proxy now injects this option automatically for OpenAI-compatible
providers (copilot, openai, opencode) when stream: true is set and
stream_options is not already present.

Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>

* fix: align OTEL attributes with gen_ai semconv spec

Rename attributes to match the OpenTelemetry GenAI semantic conventions:
- awf.provider → gen_ai.provider.name
- gen_ai.system → removed (deprecated)
- awf.cache_read_tokens → gen_ai.usage.cache_read.input_tokens
- awf.cache_write_tokens → gen_ai.usage.cache_creation.input_tokens
- awf.streaming → gen_ai.request.stream

Add new standard attributes:
- gen_ai.operation.name (set to 'chat')
- gen_ai.usage.reasoning.output_tokens

Reference: https://opentelemetry.io/docs/specs/semconv/registry/attributes/gen-ai/

Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>

---------

Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>

Author: lpcox

containers/api-proxy/otel.js

@@ -415,9 +415,10 @@ function startRequestSpan({ provider, method, path, requestId }) {
       attributes: {
         [ATTR_HTTP_REQUEST_METHOD]: method,
         [ATTR_URL_PATH]:            path,
-        'awf.provider':             provider,
+        'gen_ai.provider.name':     provider,
+        'gen_ai.operation.name':    'chat',
+        'gen_ai.request.stream':    true,
         'awf.request_id':           requestId,
-        'gen_ai.system':            provider,
       },
     },
     parentCtx,
@@ -444,17 +445,19 @@ function setTokenAttributes(span, { provider, model, normalizedUsage, streaming
       span.setAttribute('gen_ai.response.model', model);
     }
     span.setAttributes({
-      'gen_ai.usage.input_tokens':  normalizedUsage.input_tokens,
-      'gen_ai.usage.output_tokens': normalizedUsage.output_tokens,
-      'awf.cache_read_tokens':      normalizedUsage.cache_read_tokens,
-      'awf.cache_write_tokens':     normalizedUsage.cache_write_tokens,
-      'awf.streaming':              streaming,
+      'gen_ai.usage.input_tokens':                normalizedUsage.input_tokens,
+      'gen_ai.usage.output_tokens':               normalizedUsage.output_tokens,
+      'gen_ai.usage.cache_read.input_tokens':     normalizedUsage.cache_read_tokens,
+      'gen_ai.usage.cache_creation.input_tokens': normalizedUsage.cache_write_tokens,
+      'gen_ai.usage.reasoning.output_tokens':     normalizedUsage.reasoning_tokens || 0,
+      'gen_ai.request.stream':                    streaming,
     });
     span.addEvent('gen_ai.usage', {
-      'gen_ai.usage.input_tokens':  normalizedUsage.input_tokens,
-      'gen_ai.usage.output_tokens': normalizedUsage.output_tokens,
-      'awf.cache_read_tokens':      normalizedUsage.cache_read_tokens,
-      'awf.cache_write_tokens':     normalizedUsage.cache_write_tokens,
+      'gen_ai.usage.input_tokens':                normalizedUsage.input_tokens,
+      'gen_ai.usage.output_tokens':               normalizedUsage.output_tokens,
+      'gen_ai.usage.cache_read.input_tokens':     normalizedUsage.cache_read_tokens,
+      'gen_ai.usage.cache_creation.input_tokens': normalizedUsage.cache_write_tokens,
+      'gen_ai.usage.reasoning.output_tokens':     normalizedUsage.reasoning_tokens || 0,
     });
   } catch { /* best-effort */ }
 }

containers/api-proxy/otel.test.js

@@ -145,9 +145,10 @@ describe('otel — startRequestSpan', () => {
     expect(s.kind).toBe(SpanKind.CLIENT);
     expect(s.attributes['http.request.method']).toBe('POST');
     expect(s.attributes['url.path']).toBe('/v1/chat/completions');
-    expect(s.attributes['awf.provider']).toBe('openai');
+    expect(s.attributes['gen_ai.provider.name']).toBe('openai');
     expect(s.attributes['awf.request_id']).toBe('req-001');
-    expect(s.attributes['gen_ai.system']).toBe('openai');
+    expect(s.attributes['gen_ai.operation.name']).toBe('chat');
+    expect(s.attributes['gen_ai.request.stream']).toBe(true);
   });
 
   test('span name includes provider name', async () => {
@@ -234,9 +235,9 @@ describe('otel — setTokenAttributes', () => {
     expect(s.attributes['gen_ai.response.model']).toBe('gpt-4o');
     expect(s.attributes['gen_ai.usage.input_tokens']).toBe(1000);
     expect(s.attributes['gen_ai.usage.output_tokens']).toBe(500);
-    expect(s.attributes['awf.cache_read_tokens']).toBe(200);
-    expect(s.attributes['awf.cache_write_tokens']).toBe(50);
-    expect(s.attributes['awf.streaming']).toBe(false);
+    expect(s.attributes['gen_ai.usage.cache_read.input_tokens']).toBe(200);
+    expect(s.attributes['gen_ai.usage.cache_creation.input_tokens']).toBe(50);
+    expect(s.attributes['gen_ai.request.stream']).toBe(false);
 
     const usageEvent = s.events.find(e => e.name === 'gen_ai.usage');
     expect(usageEvent).toBeDefined();