Skip to content

Commit 99a5572

Browse files
api-clients-generation-pipeline[bot]ci.datadog-api-spec
api-clients-generation-pipeline[bot]
and
ci.datadog-api-spec
authored
Add buffer support to remaining Observability Pipelines destinations (#4231)
Co-authored-by: ci.datadog-api-spec <packages@datadoghq.com>
1 parent 7f09a30 commit 99a5572

8 files changed

Lines changed: 120 additions & 0 deletions

File tree

‎.generator/schemas/v2/openapi.yaml‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -53206,6 +53206,8 @@ components:
5320653206
description: S3 bucket name.
5320753207
example: "my-bucket"
5320853208
type: string
53209+
buffer:
53210+
$ref: "#/components/schemas/ObservabilityPipelineBufferOptions"
5320953211
compression:
5321053212
$ref: "#/components/schemas/ObservabilityPipelineAmazonS3GenericCompression"
5321153213
encoding:
@@ -53455,6 +53457,8 @@ components:
5345553457

5345653458
**Supported pipeline types:** logs
5345753459
properties:
53460+
buffer:
53461+
$ref: "#/components/schemas/ObservabilityPipelineBufferOptions"
5345853462
endpoint_url_key:
5345953463
description: Name of the environment variable or secret that holds the CloudPrem endpoint URL.
5346053464
example: CLOUDPREM_ENDPOINT_URL
@@ -55105,6 +55109,8 @@ components:
5510555109
properties:
5510655110
auth_strategy:
5510755111
$ref: "#/components/schemas/ObservabilityPipelineHttpClientDestinationAuthStrategy"
55112+
buffer:
55113+
$ref: "#/components/schemas/ObservabilityPipelineBufferOptions"
5510855114
compression:
5510955115
$ref: "#/components/schemas/ObservabilityPipelineHttpClientDestinationCompression"
5511055116
custom_key:
@@ -55389,6 +55395,8 @@ components:
5538955395
description: Name of the environment variable or secret that holds the Kafka bootstrap servers list.
5539055396
example: KAFKA_BOOTSTRAP_SERVERS
5539155397
type: string
55398+
buffer:
55399+
$ref: "#/components/schemas/ObservabilityPipelineBufferOptions"
5539255400
compression:
5539355401
$ref: "#/components/schemas/ObservabilityPipelineKafkaDestinationCompression"
5539455402
encoding:

‎cassettes/v2/Observability-Pipelines_4170000189/Validate-an-observability-pipeline-with-cloud_prem-destination-buffer-returns-OK-response_952838987/frozen.json‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
"2026-05-21T09:58:44.439Z"

‎cassettes/v2/Observability-Pipelines_4170000189/Validate-an-observability-pipeline-with-cloud_prem-destination-buffer-returns-OK-response_952838987/recording.har‎

Lines changed: 67 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,67 @@
1+
{
2+
"log": {
3+
"_recordingName": "Observability Pipelines/Validate an observability pipeline with cloud_prem destination buffer returns \"OK\" response",
4+
"creator": {
5+
"comment": "persister:fs",
6+
"name": "Polly.JS",
7+
"version": "6.0.5"
8+
},
9+
"entries": [
10+
{
11+
"_id": "a8aaebafd72ecdfc887a408c7c6e0f8f",
12+
"_order": 0,
13+
"cache": {},
14+
"request": {
15+
"bodySize": 554,
16+
"cookies": [],
17+
"headers": [
18+
{
19+
"_fromType": "array",
20+
"name": "accept",
21+
"value": "application/json"
22+
},
23+
{
24+
"_fromType": "array",
25+
"name": "content-type",
26+
"value": "application/json"
27+
}
28+
],
29+
"headersSize": 583,
30+
"httpVersion": "HTTP/1.1",
31+
"method": "POST",
32+
"postData": {
33+
"mimeType": "application/json",
34+
"params": [],
35+
"text": "{\"data\":{\"attributes\":{\"config\":{\"destinations\":[{\"buffer\":{\"max_size\":1073741824,\"type\":\"disk\",\"when_full\":\"block\"},\"id\":\"cloud-prem-destination\",\"inputs\":[\"my-processor-group\"],\"type\":\"cloud_prem\"}],\"processor_groups\":[{\"enabled\":true,\"id\":\"my-processor-group\",\"include\":\"service:my-service\",\"inputs\":[\"datadog-agent-source\"],\"processors\":[{\"enabled\":true,\"id\":\"filter-processor\",\"include\":\"status:error\",\"type\":\"filter\"}]}],\"sources\":[{\"id\":\"datadog-agent-source\",\"type\":\"datadog_agent\"}]},\"name\":\"Pipeline with CloudPrem Buffer\"},\"type\":\"pipelines\"}}"
36+
},
37+
"queryString": [],
38+
"url": "https://api.datadoghq.com/api/v2/obs-pipelines/pipelines/validate"
39+
},
40+
"response": {
41+
"bodySize": 14,
42+
"content": {
43+
"mimeType": "application/vnd.api+json",
44+
"size": 14,
45+
"text": "{\"errors\":[]}\n"
46+
},
47+
"cookies": [],
48+
"headers": [
49+
{
50+
"name": "content-type",
51+
"value": "application/vnd.api+json"
52+
}
53+
],
54+
"headersSize": 370,
55+
"httpVersion": "HTTP/1.1",
56+
"redirectURL": "",
57+
"status": 200,
58+
"statusText": "OK"
59+
},
60+
"startedDateTime": "2026-05-21T09:58:44.444Z",
61+
"time": 372
62+
}
63+
],
64+
"pages": [],
65+
"version": "1.2"
66+
}
67+
}

‎features/v2/observability_pipelines.feature‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -247,6 +247,14 @@ Feature: Observability Pipelines
247247
Then the response status is 200 OK
248248
And the response "errors" has length 0
249249

250+
@skip @team:DataDog/observability-pipelines
251+
Scenario: Validate an observability pipeline with cloud_prem destination buffer returns "OK" response
252+
Given new "ValidatePipeline" request
253+
And body with value {"data": {"attributes": {"config": {"destinations": [{"id": "cloud-prem-destination", "inputs": ["my-processor-group"], "type": "cloud_prem", "endpoint_url_key": "CLOUDPREM_ENDPOINT_URL", "buffer": {"type": "disk", "max_size": 1073741824, "when_full": "block"}}], "processor_groups": [{"enabled": true, "id": "my-processor-group", "include": "service:my-service", "inputs": ["datadog-agent-source"], "processors": [{"enabled": true, "id": "filter-processor", "include": "status:error", "type": "filter"}]}], "sources": [{"id": "datadog-agent-source", "type": "datadog_agent"}]}, "name": "Pipeline with CloudPrem Buffer"}, "type": "pipelines"}}
254+
When the request is sent
255+
Then the response status is 200 OK
256+
And the response "errors" has length 0
257+
250258
@team:DataDog/observability-pipelines
251259
Scenario: Validate an observability pipeline with destination secret key returns "OK" response
252260
Given new "ValidatePipeline" request

‎services/observability_pipelines/src/v2/models/ObservabilityPipelineAmazonS3GenericDestination.ts‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,7 @@ import { ObservabilityPipelineAmazonS3GenericCompression } from "./Observability
66
import { ObservabilityPipelineAmazonS3GenericDestinationType } from "./ObservabilityPipelineAmazonS3GenericDestinationType";
77
import { ObservabilityPipelineAmazonS3GenericEncoding } from "./ObservabilityPipelineAmazonS3GenericEncoding";
88
import { ObservabilityPipelineAwsAuth } from "./ObservabilityPipelineAwsAuth";
9+
import { ObservabilityPipelineBufferOptions } from "./ObservabilityPipelineBufferOptions";
910

1011
/**
1112
* The `amazon_s3_generic` destination sends your logs to an Amazon S3 bucket.
@@ -26,6 +27,10 @@ export class ObservabilityPipelineAmazonS3GenericDestination {
2627
* S3 bucket name.
2728
*/
2829
"bucket": string;
30+
/**
31+
* Configuration for buffer settings on destination components.
32+
*/
33+
"buffer"?: ObservabilityPipelineBufferOptions;
2934
/**
3035
* Compression algorithm applied to encoded logs.
3136
*/
@@ -86,6 +91,10 @@ export class ObservabilityPipelineAmazonS3GenericDestination {
8691
type: "string",
8792
required: true,
8893
},
94+
buffer: {
95+
baseName: "buffer",
96+
type: "ObservabilityPipelineBufferOptions",
97+
},
8998
compression: {
9099
baseName: "compression",
91100
type: "ObservabilityPipelineAmazonS3GenericCompression",

‎services/observability_pipelines/src/v2/models/ObservabilityPipelineCloudPremDestination.ts‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,6 @@
11
import { AttributeTypeMap } from "@datadog/datadog-api-client";
22

3+
import { ObservabilityPipelineBufferOptions } from "./ObservabilityPipelineBufferOptions";
34
import { ObservabilityPipelineCloudPremDestinationType } from "./ObservabilityPipelineCloudPremDestinationType";
45

56
/**
@@ -8,6 +9,10 @@ import { ObservabilityPipelineCloudPremDestinationType } from "./ObservabilityPi
89
* **Supported pipeline types:** logs
910
*/
1011
export class ObservabilityPipelineCloudPremDestination {
12+
/**
13+
* Configuration for buffer settings on destination components.
14+
*/
15+
"buffer"?: ObservabilityPipelineBufferOptions;
1116
/**
1217
* Name of the environment variable or secret that holds the CloudPrem endpoint URL.
1318
*/
@@ -39,6 +44,10 @@ export class ObservabilityPipelineCloudPremDestination {
3944
* @ignore
4045
*/
4146
static readonly attributeTypeMap: AttributeTypeMap = {
47+
buffer: {
48+
baseName: "buffer",
49+
type: "ObservabilityPipelineBufferOptions",
50+
},
4251
endpointUrlKey: {
4352
baseName: "endpoint_url_key",
4453
type: "string",

‎services/observability_pipelines/src/v2/models/ObservabilityPipelineHttpClientDestination.ts‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,6 @@
11
import { AttributeTypeMap } from "@datadog/datadog-api-client";
22

3+
import { ObservabilityPipelineBufferOptions } from "./ObservabilityPipelineBufferOptions";
34
import { ObservabilityPipelineHttpClientDestinationAuthStrategy } from "./ObservabilityPipelineHttpClientDestinationAuthStrategy";
45
import { ObservabilityPipelineHttpClientDestinationCompression } from "./ObservabilityPipelineHttpClientDestinationCompression";
56
import { ObservabilityPipelineHttpClientDestinationEncoding } from "./ObservabilityPipelineHttpClientDestinationEncoding";
@@ -16,6 +17,10 @@ export class ObservabilityPipelineHttpClientDestination {
1617
* HTTP authentication strategy.
1718
*/
1819
"authStrategy"?: ObservabilityPipelineHttpClientDestinationAuthStrategy;
20+
/**
21+
* Configuration for buffer settings on destination components.
22+
*/
23+
"buffer"?: ObservabilityPipelineBufferOptions;
1924
/**
2025
* Compression configuration for HTTP requests.
2126
*/
@@ -79,6 +84,10 @@ export class ObservabilityPipelineHttpClientDestination {
7984
baseName: "auth_strategy",
8085
type: "ObservabilityPipelineHttpClientDestinationAuthStrategy",
8186
},
87+
buffer: {
88+
baseName: "buffer",
89+
type: "ObservabilityPipelineBufferOptions",
90+
},
8291
compression: {
8392
baseName: "compression",
8493
type: "ObservabilityPipelineHttpClientDestinationCompression",

‎services/observability_pipelines/src/v2/models/ObservabilityPipelineKafkaDestination.ts‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,6 @@
11
import { AttributeTypeMap } from "@datadog/datadog-api-client";
22

3+
import { ObservabilityPipelineBufferOptions } from "./ObservabilityPipelineBufferOptions";
34
import { ObservabilityPipelineKafkaDestinationCompression } from "./ObservabilityPipelineKafkaDestinationCompression";
45
import { ObservabilityPipelineKafkaDestinationEncoding } from "./ObservabilityPipelineKafkaDestinationEncoding";
56
import { ObservabilityPipelineKafkaDestinationType } from "./ObservabilityPipelineKafkaDestinationType";
@@ -17,6 +18,10 @@ export class ObservabilityPipelineKafkaDestination {
1718
* Name of the environment variable or secret that holds the Kafka bootstrap servers list.
1819
*/
1920
"bootstrapServersKey"?: string;
21+
/**
22+
* Configuration for buffer settings on destination components.
23+
*/
24+
"buffer"?: ObservabilityPipelineBufferOptions;
2025
/**
2126
* Compression codec for Kafka messages.
2227
*/
@@ -96,6 +101,10 @@ export class ObservabilityPipelineKafkaDestination {
96101
baseName: "bootstrap_servers_key",
97102
type: "string",
98103
},
104+
buffer: {
105+
baseName: "buffer",
106+
type: "ObservabilityPipelineBufferOptions",
107+
},
99108
compression: {
100109
baseName: "compression",
101110
type: "ObservabilityPipelineKafkaDestinationCompression",

0 commit comments

Comments
 (0)