Skip to content

Commit 541ee36

Browse files
api-clients-generation-pipeline[bot]ci.datadog-api-spec
andauthored
Document LLM Observability LLM provider integration endpoints (#4219)
Co-authored-by: ci.datadog-api-spec <packages@datadoghq.com>
1 parent dc0237a commit 541ee36

33 files changed

Lines changed: 3391 additions & 0 deletions

.generator/schemas/v2/openapi.yaml

Lines changed: 870 additions & 0 deletions
Large diffs are not rendered by default.

features/v2/llm_observability.feature

Lines changed: 54 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -588,6 +588,40 @@ Feature: LLM Observability
588588
When the request is sent
589589
Then the response status is 200 OK
590590

591+
@generated @skip @team:DataDog/ml-observability
592+
Scenario: List LLM integration accounts returns "Bad Request" response
593+
Given operation "ListLLMObsIntegrationAccounts" enabled
594+
And new "ListLLMObsIntegrationAccounts" request
595+
And request contains "integration" parameter from "REPLACE.ME"
596+
When the request is sent
597+
Then the response status is 400 Bad Request
598+
599+
@generated @skip @team:DataDog/ml-observability
600+
Scenario: List LLM integration accounts returns "OK" response
601+
Given operation "ListLLMObsIntegrationAccounts" enabled
602+
And new "ListLLMObsIntegrationAccounts" request
603+
And request contains "integration" parameter from "REPLACE.ME"
604+
When the request is sent
605+
Then the response status is 200 OK
606+
607+
@generated @skip @team:DataDog/ml-observability
608+
Scenario: List LLM integration models returns "Bad Request" response
609+
Given operation "ListLLMObsIntegrationModels" enabled
610+
And new "ListLLMObsIntegrationModels" request
611+
And request contains "integration" parameter from "REPLACE.ME"
612+
And request contains "account_id" parameter from "REPLACE.ME"
613+
When the request is sent
614+
Then the response status is 400 Bad Request
615+
616+
@generated @skip @team:DataDog/ml-observability
617+
Scenario: List LLM integration models returns "OK" response
618+
Given operation "ListLLMObsIntegrationModels" enabled
619+
And new "ListLLMObsIntegrationModels" request
620+
And request contains "integration" parameter from "REPLACE.ME"
621+
And request contains "account_id" parameter from "REPLACE.ME"
622+
When the request is sent
623+
Then the response status is 200 OK
624+
591625
@generated @skip @team:DataDog/ml-observability
592626
Scenario: List events for an LLM Observability experiment returns "Bad Request" response
593627
Given operation "ListLLMObsExperimentEvents" enabled
@@ -639,6 +673,26 @@ Feature: LLM Observability
639673
When the request is sent
640674
Then the response status is 404 Not Found
641675

676+
@generated @skip @team:DataDog/ml-observability
677+
Scenario: Run an LLM inference returns "Bad Request" response
678+
Given operation "CreateLLMObsIntegrationInference" enabled
679+
And new "CreateLLMObsIntegrationInference" request
680+
And request contains "integration" parameter from "REPLACE.ME"
681+
And request contains "account_id" parameter from "REPLACE.ME"
682+
And body with value {"anthropic_metadata": {"effort": "medium", "thinking": {"budget_tokens": 1024, "type": "enabled"}}, "azure_openai_metadata": {"deployment_id": "my-gpt4-deployment", "model_version": "0613", "resource_name": "my-azure-resource"}, "bedrock_metadata": {"region": "us-east-1"}, "frequency_penalty": 0.0, "json_schema": "{\"type\":\"object\",\"properties\":{\"answer\":{\"type\":\"string\"}}}", "max_completion_tokens": 1024, "max_tokens": 1024, "messages": [{"content": "What is the capital of France?", "contents": [{"type": "text", "value": {"text": "Hello, how can I help you?", "tool_call": {"arguments": {"location": "San Francisco"}, "name": "get_weather", "tool_id": "call_abc123", "type": "function"}, "tool_call_result": {"name": "get_weather", "result": "The weather in San Francisco is 68\u00b0F and sunny.", "tool_id": "call_abc123", "type": "function"}}}], "id": "msg_001", "role": "user", "tool_calls": [{"arguments": {"location": "San Francisco"}, "name": "get_weather", "tool_id": "call_abc123", "type": "function"}], "tool_results": [{"name": "get_weather", "result": "The weather in San Francisco is 68\u00b0F and sunny.", "tool_id": "call_abc123", "type": "function"}]}], "model_id": "gpt-4o", "openai_metadata": {"reasoning_effort": "medium", "reasoning_summary": "auto"}, "presence_penalty": 0.0, "temperature": 0.7, "tools": [{"function": {"description": "Get the current weather for a location.", "name": "get_weather", "parameters": {"properties": {"location": {"type": "string"}}, "type": "object"}}, "type": "function"}], "top_k": 50, "top_p": 1.0, "vertex_ai_metadata": {"location": "us-central1", "project": "my-gcp-project", "project_ids": ["my-gcp-project"]}}
683+
When the request is sent
684+
Then the response status is 400 Bad Request
685+
686+
@generated @skip @team:DataDog/ml-observability
687+
Scenario: Run an LLM inference returns "OK" response
688+
Given operation "CreateLLMObsIntegrationInference" enabled
689+
And new "CreateLLMObsIntegrationInference" request
690+
And request contains "integration" parameter from "REPLACE.ME"
691+
And request contains "account_id" parameter from "REPLACE.ME"
692+
And body with value {"anthropic_metadata": {"effort": "medium", "thinking": {"budget_tokens": 1024, "type": "enabled"}}, "azure_openai_metadata": {"deployment_id": "my-gpt4-deployment", "model_version": "0613", "resource_name": "my-azure-resource"}, "bedrock_metadata": {"region": "us-east-1"}, "frequency_penalty": 0.0, "json_schema": "{\"type\":\"object\",\"properties\":{\"answer\":{\"type\":\"string\"}}}", "max_completion_tokens": 1024, "max_tokens": 1024, "messages": [{"content": "What is the capital of France?", "contents": [{"type": "text", "value": {"text": "Hello, how can I help you?", "tool_call": {"arguments": {"location": "San Francisco"}, "name": "get_weather", "tool_id": "call_abc123", "type": "function"}, "tool_call_result": {"name": "get_weather", "result": "The weather in San Francisco is 68\u00b0F and sunny.", "tool_id": "call_abc123", "type": "function"}}}], "id": "msg_001", "role": "user", "tool_calls": [{"arguments": {"location": "San Francisco"}, "name": "get_weather", "tool_id": "call_abc123", "type": "function"}], "tool_results": [{"name": "get_weather", "result": "The weather in San Francisco is 68\u00b0F and sunny.", "tool_id": "call_abc123", "type": "function"}]}], "model_id": "gpt-4o", "openai_metadata": {"reasoning_effort": "medium", "reasoning_summary": "auto"}, "presence_penalty": 0.0, "temperature": 0.7, "tools": [{"function": {"description": "Get the current weather for a location.", "name": "get_weather", "parameters": {"properties": {"location": {"type": "string"}}, "type": "object"}}, "type": "function"}], "top_k": 50, "top_p": 1.0, "vertex_ai_metadata": {"location": "us-central1", "project": "my-gcp-project", "project_ids": ["my-gcp-project"]}}
693+
When the request is sent
694+
Then the response status is 200 OK
695+
642696
@generated @skip @team:DataDog/ml-observability
643697
Scenario: Search LLM Observability experimentation entities returns "Bad Request" response
644698
Given operation "SearchLLMObsExperimentation" enabled

features/v2/undo.json

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3797,6 +3797,24 @@
37973797
"type": "unsafe"
37983798
}
37993799
},
3800+
"ListLLMObsIntegrationAccounts": {
3801+
"tag": "LLM Observability",
3802+
"undo": {
3803+
"type": "safe"
3804+
}
3805+
},
3806+
"CreateLLMObsIntegrationInference": {
3807+
"tag": "LLM Observability",
3808+
"undo": {
3809+
"type": "safe"
3810+
}
3811+
},
3812+
"ListLLMObsIntegrationModels": {
3813+
"tag": "LLM Observability",
3814+
"undo": {
3815+
"type": "safe"
3816+
}
3817+
},
38003818
"ListLLMObsProjects": {
38013819
"tag": "LLM Observability",
38023820
"undo": {

private/bdd_runner/src/support/scenarios_model_mapping.ts

Lines changed: 33 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2755,6 +2755,39 @@ export const ScenariosModelMappings: { [key: string]: OperationMapping } = {
27552755
},
27562756
operationResponseType: "{}",
27572757
},
2758+
"LLMObservabilityApi.V2.ListLLMObsIntegrationAccounts": {
2759+
integration: {
2760+
type: "LLMObsIntegrationName",
2761+
format: "",
2762+
},
2763+
operationResponseType: "Array<LLMObsIntegrationAccount>",
2764+
},
2765+
"LLMObservabilityApi.V2.CreateLLMObsIntegrationInference": {
2766+
integration: {
2767+
type: "LLMObsIntegrationName",
2768+
format: "",
2769+
},
2770+
accountId: {
2771+
type: "string",
2772+
format: "",
2773+
},
2774+
body: {
2775+
type: "LLMObsIntegrationInferenceRequest",
2776+
format: "",
2777+
},
2778+
operationResponseType: "LLMObsIntegrationInferenceResponse",
2779+
},
2780+
"LLMObservabilityApi.V2.ListLLMObsIntegrationModels": {
2781+
integration: {
2782+
type: "LLMObsIntegrationName",
2783+
format: "",
2784+
},
2785+
accountId: {
2786+
type: "string",
2787+
format: "",
2788+
},
2789+
operationResponseType: "Array<LLMObsIntegrationModel>",
2790+
},
27582791
"LLMObservabilityApi.V2.ListLLMObsProjects": {
27592792
filterId: {
27602793
type: "string",

0 commit comments

Comments
 (0)