Adding compression optional field to Amazon S3 source (#3886)

Co-authored-by: ci.datadog-api-spec <packages@datadoghq.com>

Author: api-clients-generation-pipeline[bot]

.generator/schemas/v2/openapi.yaml

@@ -43045,12 +43045,14 @@ components:
     ObservabilityPipelineAmazonS3Source:
       description: |-
         The `amazon_s3` source ingests logs from an Amazon S3 bucket.
-        It supports AWS authentication and TLS encryption.
+        It supports AWS authentication, TLS encryption, and configurable compression.
 
         **Supported pipeline types:** logs
       properties:
         auth:
           $ref: "#/components/schemas/ObservabilityPipelineAwsAuth"
+        compression:
+          $ref: "#/components/schemas/ObservabilityPipelineAmazonS3SourceCompression"
         id:
           description: The unique identifier for this component. Used in other parts of the pipeline to reference this component (for example, as the `input` to downstream components).
           example: aws-s3-source
@@ -43073,6 +43075,20 @@ components:
         - region
       type: object
       x-pipeline-types: [logs]
+    ObservabilityPipelineAmazonS3SourceCompression:
+      description: Compression format for objects retrieved from the S3 bucket. Use `auto` to detect compression from the object's Content-Encoding header or file extension.
+      enum:
+        - auto
+        - none
+        - gzip
+        - zstd
+      example: gzip
+      type: string
+      x-enum-varnames:
+        - AUTO
+        - NONE
+        - GZIP
+        - ZSTD
     ObservabilityPipelineAmazonS3SourceType:
       default: amazon_s3
       description: The source type. Always `amazon_s3`.

cassettes/v2/Observability-Pipelines_4170000189/Validate-an-observability-pipeline-with-amazon-S3-source-compression-returns-OK-response_4232921620/frozen.json

@@ -0,0 +1 @@
+"2026-04-08T12:44:25.060Z"

cassettes/v2/Observability-Pipelines_4170000189/Validate-an-observability-pipeline-with-amazon-S3-source-compression-returns-OK-response_4232921620/recording.har

@@ -0,0 +1,67 @@
+{
+  "log": {
+    "_recordingName": "Observability Pipelines/Validate an observability pipeline with amazon S3 source compression returns \"OK\" response",
+    "creator": {
+      "comment": "persister:fs",
+      "name": "Polly.JS",
+      "version": "6.0.5"
+    },
+    "entries": [
+      {
+        "_id": "558c65f8ee8c961fc37ffd1c9d896964",
+        "_order": 0,
+        "cache": {},
+        "request": {
+          "bodySize": 532,
+          "cookies": [],
+          "headers": [
+            {
+              "_fromType": "array",
+              "name": "accept",
+              "value": "application/json"
+            },
+            {
+              "_fromType": "array",
+              "name": "content-type",
+              "value": "application/json"
+            }
+          ],
+          "headersSize": 583,
+          "httpVersion": "HTTP/1.1",
+          "method": "POST",
+          "postData": {
+            "mimeType": "application/json",
+            "params": [],
+            "text": "{\"data\":{\"attributes\":{\"config\":{\"destinations\":[{\"id\":\"datadog-logs-destination\",\"inputs\":[\"my-processor-group\"],\"type\":\"datadog_logs\"}],\"processor_groups\":[{\"enabled\":true,\"id\":\"my-processor-group\",\"include\":\"service:my-service\",\"inputs\":[\"amazon-s3-source\"],\"processors\":[{\"enabled\":true,\"id\":\"filter-processor\",\"include\":\"service:my-service\",\"type\":\"filter\"}]}],\"sources\":[{\"compression\":\"gzip\",\"id\":\"amazon-s3-source\",\"region\":\"us-east-1\",\"type\":\"amazon_s3\"}]},\"name\":\"Pipeline with S3 Source Compression\"},\"type\":\"pipelines\"}}"
+          },
+          "queryString": [],
+          "url": "https://api.datadoghq.com/api/v2/obs-pipelines/pipelines/validate"
+        },
+        "response": {
+          "bodySize": 14,
+          "content": {
+            "mimeType": "application/vnd.api+json",
+            "size": 14,
+            "text": "{\"errors\":[]}\n"
+          },
+          "cookies": [],
+          "headers": [
+            {
+              "name": "content-type",
+              "value": "application/vnd.api+json"
+            }
+          ],
+          "headersSize": 370,
+          "httpVersion": "HTTP/1.1",
+          "redirectURL": "",
+          "status": 200,
+          "statusText": "OK"
+        },
+        "startedDateTime": "2026-04-08T12:44:25.066Z",
+        "time": 389
+      }
+    ],
+    "pages": [],
+    "version": "1.2"
+  }
+}

features/v2/observability_pipelines.feature

@@ -207,6 +207,14 @@ Feature: Observability Pipelines
     Then the response status is 200 OK
     And the response "errors" has length 0
 
+  @team:DataDog/observability-pipelines
+  Scenario: Validate an observability pipeline with amazon S3 source compression returns "OK" response
+    Given new "ValidatePipeline" request
+    And body with value {"data": {"attributes": {"config": {"destinations": [{"id": "datadog-logs-destination", "inputs": ["my-processor-group"], "type": "datadog_logs"}], "processor_groups": [{"enabled": true, "id": "my-processor-group", "include": "service:my-service", "inputs": ["amazon-s3-source"], "processors": [{"enabled": true, "id": "filter-processor", "include": "service:my-service", "type": "filter"}]}], "sources": [{"id": "amazon-s3-source", "type": "amazon_s3", "region": "us-east-1", "compression": "gzip"}]}, "name": "Pipeline with S3 Source Compression"}, "type": "pipelines"}}
+    When the request is sent
+    Then the response status is 200 OK
+    And the response "errors" has length 0
+
   @team:DataDog/observability-pipelines
   Scenario: Validate an observability pipeline with destination secret key returns "OK" response
     Given new "ValidatePipeline" request

services/observability_pipelines/src/v2/index.ts

@@ -48,6 +48,7 @@ export { ObservabilityPipelineAmazonS3GenericEncodingJsonType } from "./models/O
 export { ObservabilityPipelineAmazonS3GenericEncodingParquet } from "./models/ObservabilityPipelineAmazonS3GenericEncodingParquet";
 export { ObservabilityPipelineAmazonS3GenericEncodingParquetType } from "./models/ObservabilityPipelineAmazonS3GenericEncodingParquetType";
 export { ObservabilityPipelineAmazonS3Source } from "./models/ObservabilityPipelineAmazonS3Source";
+export { ObservabilityPipelineAmazonS3SourceCompression } from "./models/ObservabilityPipelineAmazonS3SourceCompression";
 export { ObservabilityPipelineAmazonS3SourceType } from "./models/ObservabilityPipelineAmazonS3SourceType";
 export { ObservabilityPipelineAmazonSecurityLakeDestination } from "./models/ObservabilityPipelineAmazonSecurityLakeDestination";
 export { ObservabilityPipelineAmazonSecurityLakeDestinationType } from "./models/ObservabilityPipelineAmazonSecurityLakeDestinationType";

services/observability_pipelines/src/v2/models/ObservabilityPipelineAmazonS3Source.ts

@@ -1,12 +1,13 @@
 import { AttributeTypeMap } from "@datadog/datadog-api-client";
 
+import { ObservabilityPipelineAmazonS3SourceCompression } from "./ObservabilityPipelineAmazonS3SourceCompression";
 import { ObservabilityPipelineAmazonS3SourceType } from "./ObservabilityPipelineAmazonS3SourceType";
 import { ObservabilityPipelineAwsAuth } from "./ObservabilityPipelineAwsAuth";
 import { ObservabilityPipelineTls } from "./ObservabilityPipelineTls";
 
 /**
  * The `amazon_s3` source ingests logs from an Amazon S3 bucket.
- * It supports AWS authentication and TLS encryption.
+ * It supports AWS authentication, TLS encryption, and configurable compression.
  *
  * **Supported pipeline types:** logs
  */
@@ -16,6 +17,10 @@ export class ObservabilityPipelineAmazonS3Source {
    * If omitted, the system’s default credentials are used (for example, the IAM role and environment variables).
    */
   "auth"?: ObservabilityPipelineAwsAuth;
+  /**
+   * Compression format for objects retrieved from the S3 bucket. Use `auto` to detect compression from the object's Content-Encoding header or file extension.
+   */
+  "compression"?: ObservabilityPipelineAmazonS3SourceCompression;
   /**
    * The unique identifier for this component. Used in other parts of the pipeline to reference this component (for example, as the `input` to downstream components).
    */
@@ -55,6 +60,10 @@ export class ObservabilityPipelineAmazonS3Source {
       baseName: "auth",
       type: "ObservabilityPipelineAwsAuth",
     },
+    compression: {
+      baseName: "compression",
+      type: "ObservabilityPipelineAmazonS3SourceCompression",
+    },
     id: {
       baseName: "id",
       type: "string",

services/observability_pipelines/src/v2/models/ObservabilityPipelineAmazonS3SourceCompression.ts

@@ -0,0 +1,15 @@
+import { UnparsedObject } from "@datadog/datadog-api-client";
+
+/**
+ * Compression format for objects retrieved from the S3 bucket. Use `auto` to detect compression from the object's Content-Encoding header or file extension.
+ */
+export type ObservabilityPipelineAmazonS3SourceCompression =
+  | typeof AUTO
+  | typeof NONE
+  | typeof GZIP
+  | typeof ZSTD
+  | UnparsedObject;
+export const AUTO = "auto";
+export const NONE = "none";
+export const GZIP = "gzip";
+export const ZSTD = "zstd";

services/observability_pipelines/src/v2/models/TypingInfo.ts

@@ -182,6 +182,12 @@ export const TypingInfo: ModelTypingInfo = {
     ObservabilityPipelineAmazonS3GenericDestinationType: ["amazon_s3_generic"],
     ObservabilityPipelineAmazonS3GenericEncodingJsonType: ["json"],
     ObservabilityPipelineAmazonS3GenericEncodingParquetType: ["parquet"],
+    ObservabilityPipelineAmazonS3SourceCompression: [
+      "auto",
+      "none",
+      "gzip",
+      "zstd",
+    ],
     ObservabilityPipelineAmazonS3SourceType: ["amazon_s3"],
     ObservabilityPipelineAmazonSecurityLakeDestinationType: [
       "amazon_security_lake",