Skip to content

Commit d54c058

Browse files
api-clients-generation-pipeline[bot]ci.datadog-api-spec
api-clients-generation-pipeline[bot]
and
ci.datadog-api-spec
authored
Document LLM Observability LLM provider integration endpoints (#4220)
Co-authored-by: ci.datadog-api-spec <packages@datadoghq.com>
1 parent ecc0486 commit d54c058

37 files changed

Lines changed: 3725 additions & 0 deletions

‎.generator/schemas/v2/openapi.yaml‎

Lines changed: 870 additions & 0 deletions
Large diffs are not rendered by default.

‎examples/v2/llm-observability/CreateLLMObsIntegrationInference.ts‎

Lines changed: 118 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,118 @@
1+
/**
2+
* Run an LLM inference returns "OK" response
3+
*/
4+
5+
import { client, v2 } from "@datadog/datadog-api-client";
6+
7+
const configuration = client.createConfiguration();
8+
configuration.unstableOperations["v2.createLLMObsIntegrationInference"] = true;
9+
const apiInstance = new v2.LLMObservabilityApi(configuration);
10+
11+
const params: v2.LLMObservabilityApiCreateLLMObsIntegrationInferenceRequest = {
12+
body: {
13+
anthropicMetadata: {
14+
effort: "medium",
15+
thinking: {
16+
budgetTokens: 1024,
17+
type: "enabled",
18+
},
19+
},
20+
azureOpenaiMetadata: {
21+
deploymentId: "my-gpt4-deployment",
22+
modelVersion: "0613",
23+
resourceName: "my-azure-resource",
24+
},
25+
bedrockMetadata: {
26+
region: "us-east-1",
27+
},
28+
frequencyPenalty: 0.0,
29+
jsonSchema: `{"type":"object","properties":{"answer":{"type":"string"}}}`,
30+
maxCompletionTokens: 1024,
31+
maxTokens: 1024,
32+
messages: [
33+
{
34+
content: "What is the capital of France?",
35+
contents: [
36+
{
37+
type: "text",
38+
value: {
39+
text: "Hello, how can I help you?",
40+
toolCall: {
41+
arguments: {
42+
location: "San Francisco",
43+
},
44+
name: "get_weather",
45+
toolId: "call_abc123",
46+
type: "function",
47+
},
48+
toolCallResult: {
49+
name: "get_weather",
50+
result: "The weather in San Francisco is 68°F and sunny.",
51+
toolId: "call_abc123",
52+
type: "function",
53+
},
54+
},
55+
},
56+
],
57+
id: "msg_001",
58+
role: "user",
59+
toolCalls: [
60+
{
61+
arguments: {
62+
location: "San Francisco",
63+
},
64+
name: "get_weather",
65+
toolId: "call_abc123",
66+
type: "function",
67+
},
68+
],
69+
toolResults: [
70+
{
71+
name: "get_weather",
72+
result: "The weather in San Francisco is 68°F and sunny.",
73+
toolId: "call_abc123",
74+
type: "function",
75+
},
76+
],
77+
},
78+
],
79+
modelId: "gpt-4o",
80+
openaiMetadata: {
81+
reasoningEffort: "medium",
82+
reasoningSummary: "auto",
83+
},
84+
presencePenalty: 0.0,
85+
temperature: 0.7,
86+
tools: [
87+
{
88+
_function: {
89+
description: "Get the current weather for a location.",
90+
name: "get_weather",
91+
parameters: {
92+
properties: "{'location': {'type': 'string'}}",
93+
type: "object",
94+
},
95+
},
96+
type: "function",
97+
},
98+
],
99+
topK: 50,
100+
topP: 1.0,
101+
vertexAiMetadata: {
102+
location: "us-central1",
103+
project: "my-gcp-project",
104+
projectIds: ["my-gcp-project"],
105+
},
106+
},
107+
integration: "openai",
108+
accountId: "account_id",
109+
};
110+
111+
apiInstance
112+
.createLLMObsIntegrationInference(params)
113+
.then((data: v2.LLMObsIntegrationInferenceResponse) => {
114+
console.log(
115+
"API called successfully. Returned data: " + JSON.stringify(data)
116+
);
117+
})
118+
.catch((error: any) => console.error(error));

‎examples/v2/llm-observability/ListLLMObsIntegrationAccounts.ts‎

Lines changed: 22 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,22 @@
1+
/**
2+
* List LLM integration accounts returns "OK" response
3+
*/
4+
5+
import { client, v2 } from "@datadog/datadog-api-client";
6+
7+
const configuration = client.createConfiguration();
8+
configuration.unstableOperations["v2.listLLMObsIntegrationAccounts"] = true;
9+
const apiInstance = new v2.LLMObservabilityApi(configuration);
10+
11+
const params: v2.LLMObservabilityApiListLLMObsIntegrationAccountsRequest = {
12+
integration: "openai",
13+
};
14+
15+
apiInstance
16+
.listLLMObsIntegrationAccounts(params)
17+
.then((data: v2.LLMObsIntegrationAccount[]) => {
18+
console.log(
19+
"API called successfully. Returned data: " + JSON.stringify(data)
20+
);
21+
})
22+
.catch((error: any) => console.error(error));

‎examples/v2/llm-observability/ListLLMObsIntegrationModels.ts‎

Lines changed: 23 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,23 @@
1+
/**
2+
* List LLM integration models returns "OK" response
3+
*/
4+
5+
import { client, v2 } from "@datadog/datadog-api-client";
6+
7+
const configuration = client.createConfiguration();
8+
configuration.unstableOperations["v2.listLLMObsIntegrationModels"] = true;
9+
const apiInstance = new v2.LLMObservabilityApi(configuration);
10+
11+
const params: v2.LLMObservabilityApiListLLMObsIntegrationModelsRequest = {
12+
integration: "openai",
13+
accountId: "account_id",
14+
};
15+
16+
apiInstance
17+
.listLLMObsIntegrationModels(params)
18+
.then((data: v2.LLMObsIntegrationModel[]) => {
19+
console.log(
20+
"API called successfully. Returned data: " + JSON.stringify(data)
21+
);
22+
})
23+
.catch((error: any) => console.error(error));

‎features/support/scenarios_model_mapping.ts‎

Lines changed: 33 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2745,6 +2745,39 @@ export const ScenariosModelMappings: {[key: string]: {[key: string]: any}} = {
27452745
},
27462746
"operationResponseType": "{}",
27472747
},
2748+
"v2.ListLLMObsIntegrationAccounts": {
2749+
"integration": {
2750+
"type": "LLMObsIntegrationName",
2751+
"format": "",
2752+
},
2753+
"operationResponseType": "Array<LLMObsIntegrationAccount>",
2754+
},
2755+
"v2.CreateLLMObsIntegrationInference": {
2756+
"integration": {
2757+
"type": "LLMObsIntegrationName",
2758+
"format": "",
2759+
},
2760+
"accountId": {
2761+
"type": "string",
2762+
"format": "",
2763+
},
2764+
"body": {
2765+
"type": "LLMObsIntegrationInferenceRequest",
2766+
"format": "",
2767+
},
2768+
"operationResponseType": "LLMObsIntegrationInferenceResponse",
2769+
},
2770+
"v2.ListLLMObsIntegrationModels": {
2771+
"integration": {
2772+
"type": "LLMObsIntegrationName",
2773+
"format": "",
2774+
},
2775+
"accountId": {
2776+
"type": "string",
2777+
"format": "",
2778+
},
2779+
"operationResponseType": "Array<LLMObsIntegrationModel>",
2780+
},
27482781
"v2.ListLLMObsProjects": {
27492782
"filterId": {
27502783
"type": "string",

‎features/v2/llm_observability.feature‎

Lines changed: 54 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -588,6 +588,40 @@ Feature: LLM Observability
588588
When the request is sent
589589
Then the response status is 200 OK
590590

591+
@generated @skip @team:DataDog/ml-observability
592+
Scenario: List LLM integration accounts returns "Bad Request" response
593+
Given operation "ListLLMObsIntegrationAccounts" enabled
594+
And new "ListLLMObsIntegrationAccounts" request
595+
And request contains "integration" parameter from "REPLACE.ME"
596+
When the request is sent
597+
Then the response status is 400 Bad Request
598+
599+
@generated @skip @team:DataDog/ml-observability
600+
Scenario: List LLM integration accounts returns "OK" response
601+
Given operation "ListLLMObsIntegrationAccounts" enabled
602+
And new "ListLLMObsIntegrationAccounts" request
603+
And request contains "integration" parameter from "REPLACE.ME"
604+
When the request is sent
605+
Then the response status is 200 OK
606+
607+
@generated @skip @team:DataDog/ml-observability
608+
Scenario: List LLM integration models returns "Bad Request" response
609+
Given operation "ListLLMObsIntegrationModels" enabled
610+
And new "ListLLMObsIntegrationModels" request
611+
And request contains "integration" parameter from "REPLACE.ME"
612+
And request contains "account_id" parameter from "REPLACE.ME"
613+
When the request is sent
614+
Then the response status is 400 Bad Request
615+
616+
@generated @skip @team:DataDog/ml-observability
617+
Scenario: List LLM integration models returns "OK" response
618+
Given operation "ListLLMObsIntegrationModels" enabled
619+
And new "ListLLMObsIntegrationModels" request
620+
And request contains "integration" parameter from "REPLACE.ME"
621+
And request contains "account_id" parameter from "REPLACE.ME"
622+
When the request is sent
623+
Then the response status is 200 OK
624+
591625
@generated @skip @team:DataDog/ml-observability
592626
Scenario: List events for an LLM Observability experiment returns "Bad Request" response
593627
Given operation "ListLLMObsExperimentEvents" enabled
@@ -639,6 +673,26 @@ Feature: LLM Observability
639673
When the request is sent
640674
Then the response status is 404 Not Found
641675

676+
@generated @skip @team:DataDog/ml-observability
677+
Scenario: Run an LLM inference returns "Bad Request" response
678+
Given operation "CreateLLMObsIntegrationInference" enabled
679+
And new "CreateLLMObsIntegrationInference" request
680+
And request contains "integration" parameter from "REPLACE.ME"
681+
And request contains "account_id" parameter from "REPLACE.ME"
682+
And body with value {"anthropic_metadata": {"effort": "medium", "thinking": {"budget_tokens": 1024, "type": "enabled"}}, "azure_openai_metadata": {"deployment_id": "my-gpt4-deployment", "model_version": "0613", "resource_name": "my-azure-resource"}, "bedrock_metadata": {"region": "us-east-1"}, "frequency_penalty": 0.0, "json_schema": "{\"type\":\"object\",\"properties\":{\"answer\":{\"type\":\"string\"}}}", "max_completion_tokens": 1024, "max_tokens": 1024, "messages": [{"content": "What is the capital of France?", "contents": [{"type": "text", "value": {"text": "Hello, how can I help you?", "tool_call": {"arguments": {"location": "San Francisco"}, "name": "get_weather", "tool_id": "call_abc123", "type": "function"}, "tool_call_result": {"name": "get_weather", "result": "The weather in San Francisco is 68\u00b0F and sunny.", "tool_id": "call_abc123", "type": "function"}}}], "id": "msg_001", "role": "user", "tool_calls": [{"arguments": {"location": "San Francisco"}, "name": "get_weather", "tool_id": "call_abc123", "type": "function"}], "tool_results": [{"name": "get_weather", "result": "The weather in San Francisco is 68\u00b0F and sunny.", "tool_id": "call_abc123", "type": "function"}]}], "model_id": "gpt-4o", "openai_metadata": {"reasoning_effort": "medium", "reasoning_summary": "auto"}, "presence_penalty": 0.0, "temperature": 0.7, "tools": [{"function": {"description": "Get the current weather for a location.", "name": "get_weather", "parameters": {"properties": {"location": {"type": "string"}}, "type": "object"}}, "type": "function"}], "top_k": 50, "top_p": 1.0, "vertex_ai_metadata": {"location": "us-central1", "project": "my-gcp-project", "project_ids": ["my-gcp-project"]}}
683+
When the request is sent
684+
Then the response status is 400 Bad Request
685+
686+
@generated @skip @team:DataDog/ml-observability
687+
Scenario: Run an LLM inference returns "OK" response
688+
Given operation "CreateLLMObsIntegrationInference" enabled
689+
And new "CreateLLMObsIntegrationInference" request
690+
And request contains "integration" parameter from "REPLACE.ME"
691+
And request contains "account_id" parameter from "REPLACE.ME"
692+
And body with value {"anthropic_metadata": {"effort": "medium", "thinking": {"budget_tokens": 1024, "type": "enabled"}}, "azure_openai_metadata": {"deployment_id": "my-gpt4-deployment", "model_version": "0613", "resource_name": "my-azure-resource"}, "bedrock_metadata": {"region": "us-east-1"}, "frequency_penalty": 0.0, "json_schema": "{\"type\":\"object\",\"properties\":{\"answer\":{\"type\":\"string\"}}}", "max_completion_tokens": 1024, "max_tokens": 1024, "messages": [{"content": "What is the capital of France?", "contents": [{"type": "text", "value": {"text": "Hello, how can I help you?", "tool_call": {"arguments": {"location": "San Francisco"}, "name": "get_weather", "tool_id": "call_abc123", "type": "function"}, "tool_call_result": {"name": "get_weather", "result": "The weather in San Francisco is 68\u00b0F and sunny.", "tool_id": "call_abc123", "type": "function"}}}], "id": "msg_001", "role": "user", "tool_calls": [{"arguments": {"location": "San Francisco"}, "name": "get_weather", "tool_id": "call_abc123", "type": "function"}], "tool_results": [{"name": "get_weather", "result": "The weather in San Francisco is 68\u00b0F and sunny.", "tool_id": "call_abc123", "type": "function"}]}], "model_id": "gpt-4o", "openai_metadata": {"reasoning_effort": "medium", "reasoning_summary": "auto"}, "presence_penalty": 0.0, "temperature": 0.7, "tools": [{"function": {"description": "Get the current weather for a location.", "name": "get_weather", "parameters": {"properties": {"location": {"type": "string"}}, "type": "object"}}, "type": "function"}], "top_k": 50, "top_p": 1.0, "vertex_ai_metadata": {"location": "us-central1", "project": "my-gcp-project", "project_ids": ["my-gcp-project"]}}
693+
When the request is sent
694+
Then the response status is 200 OK
695+
642696
@generated @skip @team:DataDog/ml-observability
643697
Scenario: Search LLM Observability experimentation entities returns "Bad Request" response
644698
Given operation "SearchLLMObsExperimentation" enabled

‎features/v2/undo.json‎

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3797,6 +3797,24 @@
37973797
"type": "unsafe"
37983798
}
37993799
},
3800+
"ListLLMObsIntegrationAccounts": {
3801+
"tag": "LLM Observability",
3802+
"undo": {
3803+
"type": "safe"
3804+
}
3805+
},
3806+
"CreateLLMObsIntegrationInference": {
3807+
"tag": "LLM Observability",
3808+
"undo": {
3809+
"type": "safe"
3810+
}
3811+
},
3812+
"ListLLMObsIntegrationModels": {
3813+
"tag": "LLM Observability",
3814+
"undo": {
3815+
"type": "safe"
3816+
}
3817+
},
38003818
"ListLLMObsProjects": {
38013819
"tag": "LLM Observability",
38023820
"undo": {

‎packages/datadog-api-client-common/configuration.ts‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -247,6 +247,7 @@ export function createConfiguration(
247247
"v2.createLLMObsDatasetRecords": false,
248248
"v2.createLLMObsExperiment": false,
249249
"v2.createLLMObsExperimentEvents": false,
250+
"v2.createLLMObsIntegrationInference": false,
250251
"v2.createLLMObsProject": false,
251252
"v2.deleteLLMObsAnnotationQueue": false,
252253
"v2.deleteLLMObsAnnotationQueueInteractions": false,
@@ -264,6 +265,8 @@ export function createConfiguration(
264265
"v2.listLLMObsDatasets": false,
265266
"v2.listLLMObsExperimentEvents": false,
266267
"v2.listLLMObsExperiments": false,
268+
"v2.listLLMObsIntegrationAccounts": false,
269+
"v2.listLLMObsIntegrationModels": false,
267270
"v2.listLLMObsProjects": false,
268271
"v2.listLLMObsSpans": false,
269272
"v2.searchLLMObsExperimentation": false,

0 commit comments

Comments
 (0)