Skip to content

Commit 1094864

Browse files
api-clients-generation-pipeline[bot]ci.datadog-api-spec
api-clients-generation-pipeline[bot]
and
ci.datadog-api-spec
authored
Add buffer support to remaining Observability Pipelines destinations (#4232)
Co-authored-by: ci.datadog-api-spec <packages@datadoghq.com>
1 parent c916eba commit 1094864

9 files changed

Lines changed: 185 additions & 0 deletions

File tree

‎.generator/schemas/v2/openapi.yaml‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -53206,6 +53206,8 @@ components:
5320653206
description: S3 bucket name.
5320753207
example: "my-bucket"
5320853208
type: string
53209+
buffer:
53210+
$ref: "#/components/schemas/ObservabilityPipelineBufferOptions"
5320953211
compression:
5321053212
$ref: "#/components/schemas/ObservabilityPipelineAmazonS3GenericCompression"
5321153213
encoding:
@@ -53455,6 +53457,8 @@ components:
5345553457

5345653458
**Supported pipeline types:** logs
5345753459
properties:
53460+
buffer:
53461+
$ref: "#/components/schemas/ObservabilityPipelineBufferOptions"
5345853462
endpoint_url_key:
5345953463
description: Name of the environment variable or secret that holds the CloudPrem endpoint URL.
5346053464
example: CLOUDPREM_ENDPOINT_URL
@@ -55105,6 +55109,8 @@ components:
5510555109
properties:
5510655110
auth_strategy:
5510755111
$ref: "#/components/schemas/ObservabilityPipelineHttpClientDestinationAuthStrategy"
55112+
buffer:
55113+
$ref: "#/components/schemas/ObservabilityPipelineBufferOptions"
5510855114
compression:
5510955115
$ref: "#/components/schemas/ObservabilityPipelineHttpClientDestinationCompression"
5511055116
custom_key:
@@ -55389,6 +55395,8 @@ components:
5538955395
description: Name of the environment variable or secret that holds the Kafka bootstrap servers list.
5539055396
example: KAFKA_BOOTSTRAP_SERVERS
5539155397
type: string
55398+
buffer:
55399+
$ref: "#/components/schemas/ObservabilityPipelineBufferOptions"
5539255400
compression:
5539355401
$ref: "#/components/schemas/ObservabilityPipelineKafkaDestinationCompression"
5539455402
encoding:

‎cassettes/v2/Observability-Pipelines_4170000189/Validate-an-observability-pipeline-with-cloud_prem-destination-buffer-returns-OK-response_952838987/frozen.json‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
"2026-05-21T09:58:44.439Z"

‎cassettes/v2/Observability-Pipelines_4170000189/Validate-an-observability-pipeline-with-cloud_prem-destination-buffer-returns-OK-response_952838987/recording.har‎

Lines changed: 67 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,67 @@
1+
{
2+
"log": {
3+
"_recordingName": "Observability Pipelines/Validate an observability pipeline with cloud_prem destination buffer returns \"OK\" response",
4+
"creator": {
5+
"comment": "persister:fs",
6+
"name": "Polly.JS",
7+
"version": "6.0.5"
8+
},
9+
"entries": [
10+
{
11+
"_id": "a8aaebafd72ecdfc887a408c7c6e0f8f",
12+
"_order": 0,
13+
"cache": {},
14+
"request": {
15+
"bodySize": 554,
16+
"cookies": [],
17+
"headers": [
18+
{
19+
"_fromType": "array",
20+
"name": "accept",
21+
"value": "application/json"
22+
},
23+
{
24+
"_fromType": "array",
25+
"name": "content-type",
26+
"value": "application/json"
27+
}
28+
],
29+
"headersSize": 583,
30+
"httpVersion": "HTTP/1.1",
31+
"method": "POST",
32+
"postData": {
33+
"mimeType": "application/json",
34+
"params": [],
35+
"text": "{\"data\":{\"attributes\":{\"config\":{\"destinations\":[{\"buffer\":{\"max_size\":1073741824,\"type\":\"disk\",\"when_full\":\"block\"},\"id\":\"cloud-prem-destination\",\"inputs\":[\"my-processor-group\"],\"type\":\"cloud_prem\"}],\"processor_groups\":[{\"enabled\":true,\"id\":\"my-processor-group\",\"include\":\"service:my-service\",\"inputs\":[\"datadog-agent-source\"],\"processors\":[{\"enabled\":true,\"id\":\"filter-processor\",\"include\":\"status:error\",\"type\":\"filter\"}]}],\"sources\":[{\"id\":\"datadog-agent-source\",\"type\":\"datadog_agent\"}]},\"name\":\"Pipeline with CloudPrem Buffer\"},\"type\":\"pipelines\"}}"
36+
},
37+
"queryString": [],
38+
"url": "https://api.datadoghq.com/api/v2/obs-pipelines/pipelines/validate"
39+
},
40+
"response": {
41+
"bodySize": 14,
42+
"content": {
43+
"mimeType": "application/vnd.api+json",
44+
"size": 14,
45+
"text": "{\"errors\":[]}\n"
46+
},
47+
"cookies": [],
48+
"headers": [
49+
{
50+
"name": "content-type",
51+
"value": "application/vnd.api+json"
52+
}
53+
],
54+
"headersSize": 370,
55+
"httpVersion": "HTTP/1.1",
56+
"redirectURL": "",
57+
"status": 200,
58+
"statusText": "OK"
59+
},
60+
"startedDateTime": "2026-05-21T09:58:44.444Z",
61+
"time": 372
62+
}
63+
],
64+
"pages": [],
65+
"version": "1.2"
66+
}
67+
}

‎examples/v2/observability-pipelines/ValidatePipeline_1070998129.ts‎

Lines changed: 65 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,65 @@
1+
/**
2+
* Validate an observability pipeline with cloud_prem destination buffer returns "OK" response
3+
*/
4+
5+
import { client, v2 } from "@datadog/datadog-api-client";
6+
7+
const configuration = client.createConfiguration();
8+
const apiInstance = new v2.ObservabilityPipelinesApi(configuration);
9+
10+
const params: v2.ObservabilityPipelinesApiValidatePipelineRequest = {
11+
body: {
12+
data: {
13+
attributes: {
14+
config: {
15+
destinations: [
16+
{
17+
id: "cloud-prem-destination",
18+
inputs: ["my-processor-group"],
19+
type: "cloud_prem",
20+
endpointUrlKey: "CLOUDPREM_ENDPOINT_URL",
21+
buffer: {
22+
type: "disk",
23+
maxSize: 1073741824,
24+
whenFull: "block",
25+
},
26+
},
27+
],
28+
processorGroups: [
29+
{
30+
enabled: true,
31+
id: "my-processor-group",
32+
include: "service:my-service",
33+
inputs: ["datadog-agent-source"],
34+
processors: [
35+
{
36+
enabled: true,
37+
id: "filter-processor",
38+
include: "status:error",
39+
type: "filter",
40+
},
41+
],
42+
},
43+
],
44+
sources: [
45+
{
46+
id: "datadog-agent-source",
47+
type: "datadog_agent",
48+
},
49+
],
50+
},
51+
name: "Pipeline with CloudPrem Buffer",
52+
},
53+
type: "pipelines",
54+
},
55+
},
56+
};
57+
58+
apiInstance
59+
.validatePipeline(params)
60+
.then((data: v2.ValidationResponse) => {
61+
console.log(
62+
"API called successfully. Returned data: " + JSON.stringify(data)
63+
);
64+
})
65+
.catch((error: any) => console.error(error));

‎features/v2/observability_pipelines.feature‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -247,6 +247,14 @@ Feature: Observability Pipelines
247247
Then the response status is 200 OK
248248
And the response "errors" has length 0
249249

250+
@skip @team:DataDog/observability-pipelines
251+
Scenario: Validate an observability pipeline with cloud_prem destination buffer returns "OK" response
252+
Given new "ValidatePipeline" request
253+
And body with value {"data": {"attributes": {"config": {"destinations": [{"id": "cloud-prem-destination", "inputs": ["my-processor-group"], "type": "cloud_prem", "endpoint_url_key": "CLOUDPREM_ENDPOINT_URL", "buffer": {"type": "disk", "max_size": 1073741824, "when_full": "block"}}], "processor_groups": [{"enabled": true, "id": "my-processor-group", "include": "service:my-service", "inputs": ["datadog-agent-source"], "processors": [{"enabled": true, "id": "filter-processor", "include": "status:error", "type": "filter"}]}], "sources": [{"id": "datadog-agent-source", "type": "datadog_agent"}]}, "name": "Pipeline with CloudPrem Buffer"}, "type": "pipelines"}}
254+
When the request is sent
255+
Then the response status is 200 OK
256+
And the response "errors" has length 0
257+
250258
@team:DataDog/observability-pipelines
251259
Scenario: Validate an observability pipeline with destination secret key returns "OK" response
252260
Given new "ValidatePipeline" request

‎packages/datadog-api-client-v2/models/ObservabilityPipelineAmazonS3GenericDestination.ts‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,7 @@ import { ObservabilityPipelineAmazonS3GenericCompression } from "./Observability
99
import { ObservabilityPipelineAmazonS3GenericDestinationType } from "./ObservabilityPipelineAmazonS3GenericDestinationType";
1010
import { ObservabilityPipelineAmazonS3GenericEncoding } from "./ObservabilityPipelineAmazonS3GenericEncoding";
1111
import { ObservabilityPipelineAwsAuth } from "./ObservabilityPipelineAwsAuth";
12+
import { ObservabilityPipelineBufferOptions } from "./ObservabilityPipelineBufferOptions";
1213

1314
import { AttributeTypeMap } from "../../datadog-api-client-common/util";
1415

@@ -31,6 +32,10 @@ export class ObservabilityPipelineAmazonS3GenericDestination {
3132
* S3 bucket name.
3233
*/
3334
"bucket": string;
35+
/**
36+
* Configuration for buffer settings on destination components.
37+
*/
38+
"buffer"?: ObservabilityPipelineBufferOptions;
3439
/**
3540
* Compression algorithm applied to encoded logs.
3641
*/
@@ -93,6 +98,10 @@ export class ObservabilityPipelineAmazonS3GenericDestination {
9398
type: "string",
9499
required: true,
95100
},
101+
buffer: {
102+
baseName: "buffer",
103+
type: "ObservabilityPipelineBufferOptions",
104+
},
96105
compression: {
97106
baseName: "compression",
98107
type: "ObservabilityPipelineAmazonS3GenericCompression",

‎packages/datadog-api-client-v2/models/ObservabilityPipelineCloudPremDestination.ts‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,7 @@
33
* This product includes software developed at Datadog (https://www.datadoghq.com/).
44
* Copyright 2020-Present Datadog, Inc.
55
*/
6+
import { ObservabilityPipelineBufferOptions } from "./ObservabilityPipelineBufferOptions";
67
import { ObservabilityPipelineCloudPremDestinationType } from "./ObservabilityPipelineCloudPremDestinationType";
78

89
import { AttributeTypeMap } from "../../datadog-api-client-common/util";
@@ -13,6 +14,10 @@ import { AttributeTypeMap } from "../../datadog-api-client-common/util";
1314
* **Supported pipeline types:** logs
1415
*/
1516
export class ObservabilityPipelineCloudPremDestination {
17+
/**
18+
* Configuration for buffer settings on destination components.
19+
*/
20+
"buffer"?: ObservabilityPipelineBufferOptions;
1621
/**
1722
* Name of the environment variable or secret that holds the CloudPrem endpoint URL.
1823
*/
@@ -46,6 +51,10 @@ export class ObservabilityPipelineCloudPremDestination {
4651
* @ignore
4752
*/
4853
static readonly attributeTypeMap: AttributeTypeMap = {
54+
buffer: {
55+
baseName: "buffer",
56+
type: "ObservabilityPipelineBufferOptions",
57+
},
4958
endpointUrlKey: {
5059
baseName: "endpoint_url_key",
5160
type: "string",

‎packages/datadog-api-client-v2/models/ObservabilityPipelineHttpClientDestination.ts‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,7 @@
33
* This product includes software developed at Datadog (https://www.datadoghq.com/).
44
* Copyright 2020-Present Datadog, Inc.
55
*/
6+
import { ObservabilityPipelineBufferOptions } from "./ObservabilityPipelineBufferOptions";
67
import { ObservabilityPipelineHttpClientDestinationAuthStrategy } from "./ObservabilityPipelineHttpClientDestinationAuthStrategy";
78
import { ObservabilityPipelineHttpClientDestinationCompression } from "./ObservabilityPipelineHttpClientDestinationCompression";
89
import { ObservabilityPipelineHttpClientDestinationEncoding } from "./ObservabilityPipelineHttpClientDestinationEncoding";
@@ -21,6 +22,10 @@ export class ObservabilityPipelineHttpClientDestination {
2122
* HTTP authentication strategy.
2223
*/
2324
"authStrategy"?: ObservabilityPipelineHttpClientDestinationAuthStrategy;
25+
/**
26+
* Configuration for buffer settings on destination components.
27+
*/
28+
"buffer"?: ObservabilityPipelineBufferOptions;
2429
/**
2530
* Compression configuration for HTTP requests.
2631
*/
@@ -86,6 +91,10 @@ export class ObservabilityPipelineHttpClientDestination {
8691
baseName: "auth_strategy",
8792
type: "ObservabilityPipelineHttpClientDestinationAuthStrategy",
8893
},
94+
buffer: {
95+
baseName: "buffer",
96+
type: "ObservabilityPipelineBufferOptions",
97+
},
8998
compression: {
9099
baseName: "compression",
91100
type: "ObservabilityPipelineHttpClientDestinationCompression",

‎packages/datadog-api-client-v2/models/ObservabilityPipelineKafkaDestination.ts‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,7 @@
33
* This product includes software developed at Datadog (https://www.datadoghq.com/).
44
* Copyright 2020-Present Datadog, Inc.
55
*/
6+
import { ObservabilityPipelineBufferOptions } from "./ObservabilityPipelineBufferOptions";
67
import { ObservabilityPipelineKafkaDestinationCompression } from "./ObservabilityPipelineKafkaDestinationCompression";
78
import { ObservabilityPipelineKafkaDestinationEncoding } from "./ObservabilityPipelineKafkaDestinationEncoding";
89
import { ObservabilityPipelineKafkaDestinationType } from "./ObservabilityPipelineKafkaDestinationType";
@@ -22,6 +23,10 @@ export class ObservabilityPipelineKafkaDestination {
2223
* Name of the environment variable or secret that holds the Kafka bootstrap servers list.
2324
*/
2425
"bootstrapServersKey"?: string;
26+
/**
27+
* Configuration for buffer settings on destination components.
28+
*/
29+
"buffer"?: ObservabilityPipelineBufferOptions;
2530
/**
2631
* Compression codec for Kafka messages.
2732
*/
@@ -103,6 +108,10 @@ export class ObservabilityPipelineKafkaDestination {
103108
baseName: "bootstrap_servers_key",
104109
type: "string",
105110
},
111+
buffer: {
112+
baseName: "buffer",
113+
type: "ObservabilityPipelineBufferOptions",
114+
},
106115
compression: {
107116
baseName: "compression",
108117
type: "ObservabilityPipelineKafkaDestinationCompression",

0 commit comments

Comments
 (0)