Skip to content

Commit b3dbd38

Browse files
api-clients-generation-pipeline[bot]ci.datadog-api-spec
api-clients-generation-pipeline[bot]
and
ci.datadog-api-spec
authored
Adding compression optional field to Amazon S3 source (#3887)
Co-authored-by: ci.datadog-api-spec <packages@datadoghq.com>
1 parent 6f113a0 commit b3dbd38

File tree

9 files changed

+193
-2
lines changed

9 files changed

+193
-2
lines changed

‎.generator/schemas/v2/openapi.yaml‎

Lines changed: 17 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -43045,12 +43045,14 @@ components:
4304543045
ObservabilityPipelineAmazonS3Source:
4304643046
description: |-
4304743047
The `amazon_s3` source ingests logs from an Amazon S3 bucket.
43048-
It supports AWS authentication and TLS encryption.
43048+
It supports AWS authentication, TLS encryption, and configurable compression.
4304943049

4305043050
**Supported pipeline types:** logs
4305143051
properties:
4305243052
auth:
4305343053
$ref: "#/components/schemas/ObservabilityPipelineAwsAuth"
43054+
compression:
43055+
$ref: "#/components/schemas/ObservabilityPipelineAmazonS3SourceCompression"
4305443056
id:
4305543057
description: The unique identifier for this component. Used in other parts of the pipeline to reference this component (for example, as the `input` to downstream components).
4305643058
example: aws-s3-source
@@ -43073,6 +43075,20 @@ components:
4307343075
- region
4307443076
type: object
4307543077
x-pipeline-types: [logs]
43078+
ObservabilityPipelineAmazonS3SourceCompression:
43079+
description: Compression format for objects retrieved from the S3 bucket. Use `auto` to detect compression from the object's Content-Encoding header or file extension.
43080+
enum:
43081+
- auto
43082+
- none
43083+
- gzip
43084+
- zstd
43085+
example: gzip
43086+
type: string
43087+
x-enum-varnames:
43088+
- AUTO
43089+
- NONE
43090+
- GZIP
43091+
- ZSTD
4307643092
ObservabilityPipelineAmazonS3SourceType:
4307743093
default: amazon_s3
4307843094
description: The source type. Always `amazon_s3`.

‎cassettes/v2/Observability-Pipelines_4170000189/Validate-an-observability-pipeline-with-amazon-S3-source-compression-returns-OK-response_4232921620/frozen.json‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
"2026-04-08T12:44:25.060Z"

‎cassettes/v2/Observability-Pipelines_4170000189/Validate-an-observability-pipeline-with-amazon-S3-source-compression-returns-OK-response_4232921620/recording.har‎

Lines changed: 67 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,67 @@
1+
{
2+
"log": {
3+
"_recordingName": "Observability Pipelines/Validate an observability pipeline with amazon S3 source compression returns \"OK\" response",
4+
"creator": {
5+
"comment": "persister:fs",
6+
"name": "Polly.JS",
7+
"version": "6.0.5"
8+
},
9+
"entries": [
10+
{
11+
"_id": "558c65f8ee8c961fc37ffd1c9d896964",
12+
"_order": 0,
13+
"cache": {},
14+
"request": {
15+
"bodySize": 532,
16+
"cookies": [],
17+
"headers": [
18+
{
19+
"_fromType": "array",
20+
"name": "accept",
21+
"value": "application/json"
22+
},
23+
{
24+
"_fromType": "array",
25+
"name": "content-type",
26+
"value": "application/json"
27+
}
28+
],
29+
"headersSize": 583,
30+
"httpVersion": "HTTP/1.1",
31+
"method": "POST",
32+
"postData": {
33+
"mimeType": "application/json",
34+
"params": [],
35+
"text": "{\"data\":{\"attributes\":{\"config\":{\"destinations\":[{\"id\":\"datadog-logs-destination\",\"inputs\":[\"my-processor-group\"],\"type\":\"datadog_logs\"}],\"processor_groups\":[{\"enabled\":true,\"id\":\"my-processor-group\",\"include\":\"service:my-service\",\"inputs\":[\"amazon-s3-source\"],\"processors\":[{\"enabled\":true,\"id\":\"filter-processor\",\"include\":\"service:my-service\",\"type\":\"filter\"}]}],\"sources\":[{\"compression\":\"gzip\",\"id\":\"amazon-s3-source\",\"region\":\"us-east-1\",\"type\":\"amazon_s3\"}]},\"name\":\"Pipeline with S3 Source Compression\"},\"type\":\"pipelines\"}}"
36+
},
37+
"queryString": [],
38+
"url": "https://api.datadoghq.com/api/v2/obs-pipelines/pipelines/validate"
39+
},
40+
"response": {
41+
"bodySize": 14,
42+
"content": {
43+
"mimeType": "application/vnd.api+json",
44+
"size": 14,
45+
"text": "{\"errors\":[]}\n"
46+
},
47+
"cookies": [],
48+
"headers": [
49+
{
50+
"name": "content-type",
51+
"value": "application/vnd.api+json"
52+
}
53+
],
54+
"headersSize": 370,
55+
"httpVersion": "HTTP/1.1",
56+
"redirectURL": "",
57+
"status": 200,
58+
"statusText": "OK"
59+
},
60+
"startedDateTime": "2026-04-08T12:44:25.066Z",
61+
"time": 389
62+
}
63+
],
64+
"pages": [],
65+
"version": "1.2"
66+
}
67+
}

‎examples/v2/observability-pipelines/ValidatePipeline_99164570.ts‎

Lines changed: 61 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,61 @@
1+
/**
2+
* Validate an observability pipeline with amazon S3 source compression returns "OK" response
3+
*/
4+
5+
import { client, v2 } from "@datadog/datadog-api-client";
6+
7+
const configuration = client.createConfiguration();
8+
const apiInstance = new v2.ObservabilityPipelinesApi(configuration);
9+
10+
const params: v2.ObservabilityPipelinesApiValidatePipelineRequest = {
11+
body: {
12+
data: {
13+
attributes: {
14+
config: {
15+
destinations: [
16+
{
17+
id: "datadog-logs-destination",
18+
inputs: ["my-processor-group"],
19+
type: "datadog_logs",
20+
},
21+
],
22+
processorGroups: [
23+
{
24+
enabled: true,
25+
id: "my-processor-group",
26+
include: "service:my-service",
27+
inputs: ["amazon-s3-source"],
28+
processors: [
29+
{
30+
enabled: true,
31+
id: "filter-processor",
32+
include: "service:my-service",
33+
type: "filter",
34+
},
35+
],
36+
},
37+
],
38+
sources: [
39+
{
40+
id: "amazon-s3-source",
41+
type: "amazon_s3",
42+
region: "us-east-1",
43+
compression: "gzip",
44+
},
45+
],
46+
},
47+
name: "Pipeline with S3 Source Compression",
48+
},
49+
type: "pipelines",
50+
},
51+
},
52+
};
53+
54+
apiInstance
55+
.validatePipeline(params)
56+
.then((data: v2.ValidationResponse) => {
57+
console.log(
58+
"API called successfully. Returned data: " + JSON.stringify(data)
59+
);
60+
})
61+
.catch((error: any) => console.error(error));

‎features/v2/observability_pipelines.feature‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -207,6 +207,14 @@ Feature: Observability Pipelines
207207
Then the response status is 200 OK
208208
And the response "errors" has length 0
209209

210+
@team:DataDog/observability-pipelines
211+
Scenario: Validate an observability pipeline with amazon S3 source compression returns "OK" response
212+
Given new "ValidatePipeline" request
213+
And body with value {"data": {"attributes": {"config": {"destinations": [{"id": "datadog-logs-destination", "inputs": ["my-processor-group"], "type": "datadog_logs"}], "processor_groups": [{"enabled": true, "id": "my-processor-group", "include": "service:my-service", "inputs": ["amazon-s3-source"], "processors": [{"enabled": true, "id": "filter-processor", "include": "service:my-service", "type": "filter"}]}], "sources": [{"id": "amazon-s3-source", "type": "amazon_s3", "region": "us-east-1", "compression": "gzip"}]}, "name": "Pipeline with S3 Source Compression"}, "type": "pipelines"}}
214+
When the request is sent
215+
Then the response status is 200 OK
216+
And the response "errors" has length 0
217+
210218
@team:DataDog/observability-pipelines
211219
Scenario: Validate an observability pipeline with destination secret key returns "OK" response
212220
Given new "ValidatePipeline" request

‎packages/datadog-api-client-v2/index.ts‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -4005,6 +4005,7 @@ export { ObservabilityPipelineAmazonS3GenericEncodingJsonType } from "./models/O
40054005
export { ObservabilityPipelineAmazonS3GenericEncodingParquet } from "./models/ObservabilityPipelineAmazonS3GenericEncodingParquet";
40064006
export { ObservabilityPipelineAmazonS3GenericEncodingParquetType } from "./models/ObservabilityPipelineAmazonS3GenericEncodingParquetType";
40074007
export { ObservabilityPipelineAmazonS3Source } from "./models/ObservabilityPipelineAmazonS3Source";
4008+
export { ObservabilityPipelineAmazonS3SourceCompression } from "./models/ObservabilityPipelineAmazonS3SourceCompression";
40084009
export { ObservabilityPipelineAmazonS3SourceType } from "./models/ObservabilityPipelineAmazonS3SourceType";
40094010
export { ObservabilityPipelineAmazonSecurityLakeDestination } from "./models/ObservabilityPipelineAmazonSecurityLakeDestination";
40104011
export { ObservabilityPipelineAmazonSecurityLakeDestinationType } from "./models/ObservabilityPipelineAmazonSecurityLakeDestinationType";

‎packages/datadog-api-client-v2/models/ObjectSerializer.ts‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -4642,6 +4642,12 @@ const enumsMap: { [key: string]: any[] } = {
46424642
ObservabilityPipelineAmazonS3GenericDestinationType: ["amazon_s3_generic"],
46434643
ObservabilityPipelineAmazonS3GenericEncodingJsonType: ["json"],
46444644
ObservabilityPipelineAmazonS3GenericEncodingParquetType: ["parquet"],
4645+
ObservabilityPipelineAmazonS3SourceCompression: [
4646+
"auto",
4647+
"none",
4648+
"gzip",
4649+
"zstd",
4650+
],
46454651
ObservabilityPipelineAmazonS3SourceType: ["amazon_s3"],
46464652
ObservabilityPipelineAmazonSecurityLakeDestinationType: [
46474653
"amazon_security_lake",

‎packages/datadog-api-client-v2/models/ObservabilityPipelineAmazonS3Source.ts‎

Lines changed: 10 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,7 @@
33
* This product includes software developed at Datadog (https://www.datadoghq.com/).
44
* Copyright 2020-Present Datadog, Inc.
55
*/
6+
import { ObservabilityPipelineAmazonS3SourceCompression } from "./ObservabilityPipelineAmazonS3SourceCompression";
67
import { ObservabilityPipelineAmazonS3SourceType } from "./ObservabilityPipelineAmazonS3SourceType";
78
import { ObservabilityPipelineAwsAuth } from "./ObservabilityPipelineAwsAuth";
89
import { ObservabilityPipelineTls } from "./ObservabilityPipelineTls";
@@ -11,7 +12,7 @@ import { AttributeTypeMap } from "../../datadog-api-client-common/util";
1112

1213
/**
1314
* The `amazon_s3` source ingests logs from an Amazon S3 bucket.
14-
* It supports AWS authentication and TLS encryption.
15+
* It supports AWS authentication, TLS encryption, and configurable compression.
1516
*
1617
* **Supported pipeline types:** logs
1718
*/
@@ -21,6 +22,10 @@ export class ObservabilityPipelineAmazonS3Source {
2122
* If omitted, the system’s default credentials are used (for example, the IAM role and environment variables).
2223
*/
2324
"auth"?: ObservabilityPipelineAwsAuth;
25+
/**
26+
* Compression format for objects retrieved from the S3 bucket. Use `auto` to detect compression from the object's Content-Encoding header or file extension.
27+
*/
28+
"compression"?: ObservabilityPipelineAmazonS3SourceCompression;
2429
/**
2530
* The unique identifier for this component. Used in other parts of the pipeline to reference this component (for example, as the `input` to downstream components).
2631
*/
@@ -62,6 +67,10 @@ export class ObservabilityPipelineAmazonS3Source {
6267
baseName: "auth",
6368
type: "ObservabilityPipelineAwsAuth",
6469
},
70+
compression: {
71+
baseName: "compression",
72+
type: "ObservabilityPipelineAmazonS3SourceCompression",
73+
},
6574
id: {
6675
baseName: "id",
6776
type: "string",

‎packages/datadog-api-client-v2/models/ObservabilityPipelineAmazonS3SourceCompression.ts‎

Lines changed: 22 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,22 @@
1+
/**
2+
* Unless explicitly stated otherwise all files in this repository are licensed under the Apache-2.0 License.
3+
* This product includes software developed at Datadog (https://www.datadoghq.com/).
4+
* Copyright 2020-Present Datadog, Inc.
5+
*/
6+
7+
import { UnparsedObject } from "../../datadog-api-client-common/util";
8+
9+
/**
10+
* Compression format for objects retrieved from the S3 bucket. Use `auto` to detect compression from the object's Content-Encoding header or file extension.
11+
*/
12+
13+
export type ObservabilityPipelineAmazonS3SourceCompression =
14+
| typeof AUTO
15+
| typeof NONE
16+
| typeof GZIP
17+
| typeof ZSTD
18+
| UnparsedObject;
19+
export const AUTO = "auto";
20+
export const NONE = "none";
21+
export const GZIP = "gzip";
22+
export const ZSTD = "zstd";

0 commit comments

Comments
 (0)