feat: emit errors for deprecated gen_ai.response.text and validate gen_ai.output.messages against a Zod schema (#145)

* feat: emit errors for deprecated `gen_ai.response.text` and validate `gen_ai.output.messages` against a Zod schema

* chore: make manual instrumentation pass schema checks by adding finish_reason

Author: constantinius

package-lock.json

@@ -34,7 +34,8 @@
     "prettier": "^3.8.1",
     "vhtml": "^2.2.0",
     "vite": "^6.0.11",
-    "vite-plugin-singlefile": "^2.0.5"
+    "vite-plugin-singlefile": "^2.0.5",
+    "zod": "^3.25.76"
   },
   "devDependencies": {
     "@types/chai": "^5.2.3",

package.json

@@ -139,8 +139,8 @@ const TOOLS = {{ agent.tools | dump }};
       }
       const toolDefinitionsJson{{ loop.index }} = JSON.stringify(toolDefinitions{{ loop.index }});
 
-      const outputParts{{ loop.index }} = [{ type: "text", text: responseText{{ loop.index }} }, ...outputToolCallParts{{ loop.index }}];
-      const outputMessages{{ loop.index }} = JSON.stringify([{ role: "assistant", parts: outputParts{{ loop.index }} }]);
+      const outputParts{{ loop.index }} = [{ type: "text", content: responseText{{ loop.index }} }, ...outputToolCallParts{{ loop.index }}];
+      const outputMessages{{ loop.index }} = JSON.stringify([{ role: "assistant", parts: outputParts{{ loop.index }}, finish_reason: "stop" }]);
 
       // Agent span (parent)
       const agentDesc{{ loop.index }} = `invoke_agent ${AGENT_NAME}`;

src/runner/templates/agents/node/manual/template.njk

@@ -136,9 +136,9 @@ TOOLS = {{ agent.tools | dump }}
     tool_definitions_json = json.dumps(tool_definitions)
 
     # Build output messages
-    output_parts = [{"type": "text", "text": response_text}]
+    output_parts = [{"type": "text", "content": response_text}]
     output_parts.extend(output_tool_call_parts)
-    output_messages = json.dumps([{"role": "assistant", "parts": output_parts}])
+    output_messages = json.dumps([{"role": "assistant", "parts": output_parts, "finish_reason": "stop"}])
 
     # Agent span (parent)
     agent_desc = f"invoke_agent {AGENT_NAME}"

src/runner/templates/agents/python/manual/template.njk

@@ -117,7 +117,7 @@ function buildSentryMessages(messages) {
 
       const responseText{{ loop.index }} = "This is a simulated response for turn {{ loop.index }}.";
       const outputMessages{{ loop.index }} = JSON.stringify([
-        { role: "assistant", parts: [{ type: "text", text: responseText{{ loop.index }} }] },
+        { role: "assistant", parts: [{ type: "text", content: responseText{{ loop.index }} }], finish_reason: "stop" },
       ]);
 
       const desc{{ loop.index }} = `chat ${model{{ loop.index }}}`;

src/runner/templates/llm/node/manual/template.njk

@@ -110,7 +110,7 @@ def build_sentry_messages(messages):
 
     response_text = "This is a simulated response for turn {{ loop.index }}."
     output_messages = json.dumps([
-        {"role": "assistant", "parts": [{"type": "text", "text": response_text}]}
+        {"role": "assistant", "parts": [{"type": "text", "content": response_text}], "finish_reason": "stop"}
     ])
 
     description = f"chat {model}"

src/runner/templates/llm/python/manual/template.njk

@@ -12,6 +12,8 @@
  * collect or report deprecation warnings.
  */
 
+import { z } from "zod";
+
 import { CapturedSpan, ErrorLocation, Check } from "../types.js";
 import { CheckError } from "../validator.js";
 import {
@@ -152,6 +154,53 @@ function parseInputMessages(
   return { messages: result.value as unknown[], attribute: result.usedAttribute! };
 }
 
+const OutputMessagePartSchema = z.object({ type: z.string() }).passthrough();
+const OutputMessagesSchema = z.array(
+  z.object({
+    role: z.string(),
+    parts: z.array(OutputMessagePartSchema),
+    name: z.string().nullable().optional(),
+    finish_reason: z.string(),
+  }).passthrough(),
+);
+
+function formatZodPath(path: Array<string | number>): string {
+  return path.reduce<string>((formatted, segment) => {
+    if (typeof segment === "number") {
+      return `${formatted}[${segment}]`;
+    }
+    return formatted ? `${formatted}.${segment}` : segment;
+  }, "messages");
+}
+
+/**
+ * Validate gen_ai.output.messages against the OTEL semantic convention schema:
+ * https://opentelemetry.io/docs/specs/semconv/gen-ai/gen-ai-output-messages.json
+ *
+ * The schema's GenericPart branch allows provider-specific part payloads, so this
+ * mirrors the schema envelope instead of requiring stricter fields for each type.
+ */
+function validateOutputMessagesSchema(
+  value: unknown,
+  attribute = "gen_ai.output.messages",
+): string[] {
+  let parsedValue = value;
+  if (typeof value === "string") {
+    try {
+      parsedValue = JSON.parse(value);
+    } catch {
+      return [`Invalid JSON in ${attribute}`];
+    }
+  }
+
+  const result = OutputMessagesSchema.safeParse(parsedValue);
+  if (result.success) return [];
+
+  return result.error.issues.map(
+    (issue) => `${formatZodPath(issue.path)} ${issue.message}`,
+  );
+}
+
 function getMessageText(message: unknown): string | undefined {
   if (typeof message !== "object" || message === null) {
     return undefined;
@@ -329,8 +378,9 @@ function assertOnlyLastInputMessage(
  * - description equals "<gen_ai.operation.name> <gen_ai.request.model>"
  * - gen_ai.operation.name matches AI_CLIENT_OPERATION_NAME_PATTERN
  * - gen_ai.request.model matches expected model
- * - gen_ai.request.messages exists
- * - gen_ai.response.text exists
+ * - gen_ai.input.messages exists (or deprecated gen_ai.request.messages fallback)
+ * - gen_ai.output.messages exists and matches the OTEL output messages schema
+ * - deprecated gen_ai.response.text is not present
  * - gen_ai.usage.input_tokens exists
  * - gen_ai.usage.output_tokens exists
  *
@@ -373,17 +423,23 @@ export const checkChatSpanAttributes: Check = {
         locations.push({ spanId: span.span_id, attribute: "gen_ai.input.messages", message: "Missing messages attribute" });
       }
 
-      const responseResult = getAttributeWithFallback(
-        span,
-        "gen_ai.output.messages",
-        "gen_ai.response.text"
-      );
-      
-      if (responseResult.value === undefined) {
-        const hasToolCalls = !!span.data?.["gen_ai.response.tool_calls"];
-        if (!hasToolCalls) {
-          errors.push("Should have gen_ai.output.messages, gen_ai.response.text, or gen_ai.response.tool_calls");
-          locations.push({ spanId: span.span_id, attribute: "gen_ai.output.messages", message: "Missing response attribute" });
+      if (span.data?.["gen_ai.response.text"] !== undefined) {
+        const msg = 'Deprecated attribute "gen_ai.response.text" is not allowed; use "gen_ai.output.messages" instead';
+        errors.push(msg);
+        locations.push({ spanId: span.span_id, attribute: "gen_ai.response.text", message: msg });
+      }
+
+      const outputMessages = span.data?.["gen_ai.output.messages"];
+
+      if (outputMessages === undefined) {
+        const msg = "Should have gen_ai.output.messages";
+        errors.push(msg);
+        locations.push({ spanId: span.span_id, attribute: "gen_ai.output.messages", message: "Missing output messages attribute" });
+      } else {
+        const schemaErrors = validateOutputMessagesSchema(outputMessages);
+        for (const schemaError of schemaErrors) {
+          errors.push(schemaError);
+          locations.push({ spanId: span.span_id, attribute: "gen_ai.output.messages", message: schemaError });
         }
       }
     }