Adding compression optional field to Amazon S3 source (#3205)

Co-authored-by: ci.datadog-api-spec <packages@datadoghq.com>

Author: api-clients-generation-pipeline[bot]

.generator/schemas/v2/openapi.yaml

@@ -43045,12 +43045,14 @@ components:
     ObservabilityPipelineAmazonS3Source:
       description: |-
         The `amazon_s3` source ingests logs from an Amazon S3 bucket.
-        It supports AWS authentication and TLS encryption.
+        It supports AWS authentication, TLS encryption, and configurable compression.
 
         **Supported pipeline types:** logs
       properties:
         auth:
           $ref: "#/components/schemas/ObservabilityPipelineAwsAuth"
+        compression:
+          $ref: "#/components/schemas/ObservabilityPipelineAmazonS3SourceCompression"
         id:
           description: The unique identifier for this component. Used in other parts of the pipeline to reference this component (for example, as the `input` to downstream components).
           example: aws-s3-source
@@ -43073,6 +43075,20 @@ components:
         - region
       type: object
       x-pipeline-types: [logs]
+    ObservabilityPipelineAmazonS3SourceCompression:
+      description: Compression format for objects retrieved from the S3 bucket. Use `auto` to detect compression from the object's Content-Encoding header or file extension.
+      enum:
+        - auto
+        - none
+        - gzip
+        - zstd
+      example: gzip
+      type: string
+      x-enum-varnames:
+        - AUTO
+        - NONE
+        - GZIP
+        - ZSTD
     ObservabilityPipelineAmazonS3SourceType:
       default: amazon_s3
       description: The source type. Always `amazon_s3`.

cassettes/features/v2/observability_pipelines/Validate-an-observability-pipeline-with-amazon-S3-source-compression-returns-OK-response.frozen

@@ -0,0 +1 @@
+2026-04-08T12:44:25.060Z

cassettes/features/v2/observability_pipelines/Validate-an-observability-pipeline-with-amazon-S3-source-compression-returns-OK-response.yml

@@ -0,0 +1,51 @@
+# Validate an observability pipeline with amazon S3 source compression returns "OK" response
+
+require "datadog_api_client"
+api_instance = DatadogAPIClient::V2::ObservabilityPipelinesAPI.new
+
+body = DatadogAPIClient::V2::ObservabilityPipelineSpec.new({
+  data: DatadogAPIClient::V2::ObservabilityPipelineSpecData.new({
+    attributes: DatadogAPIClient::V2::ObservabilityPipelineDataAttributes.new({
+      config: DatadogAPIClient::V2::ObservabilityPipelineConfig.new({
+        destinations: [
+          DatadogAPIClient::V2::ObservabilityPipelineDatadogLogsDestination.new({
+            id: "datadog-logs-destination",
+            inputs: [
+              "my-processor-group",
+            ],
+            type: DatadogAPIClient::V2::ObservabilityPipelineDatadogLogsDestinationType::DATADOG_LOGS,
+          }),
+        ],
+        processor_groups: [
+          DatadogAPIClient::V2::ObservabilityPipelineConfigProcessorGroup.new({
+            enabled: true,
+            id: "my-processor-group",
+            include: "service:my-service",
+            inputs: [
+              "amazon-s3-source",
+            ],
+            processors: [
+              DatadogAPIClient::V2::ObservabilityPipelineFilterProcessor.new({
+                enabled: true,
+                id: "filter-processor",
+                include: "service:my-service",
+                type: DatadogAPIClient::V2::ObservabilityPipelineFilterProcessorType::FILTER,
+              }),
+            ],
+          }),
+        ],
+        sources: [
+          DatadogAPIClient::V2::ObservabilityPipelineAmazonS3Source.new({
+            id: "amazon-s3-source",
+            type: DatadogAPIClient::V2::ObservabilityPipelineAmazonS3SourceType::AMAZON_S3,
+            region: "us-east-1",
+            compression: DatadogAPIClient::V2::ObservabilityPipelineAmazonS3SourceCompression::GZIP,
+          }),
+        ],
+      }),
+      name: "Pipeline with S3 Source Compression",
+    }),
+    type: "pipelines",
+  }),
+})
+p api_instance.validate_pipeline(body)

examples/v2/observability-pipelines/ValidatePipeline_99164570.rb

@@ -207,6 +207,14 @@ Feature: Observability Pipelines
     Then the response status is 200 OK
     And the response "errors" has length 0
 
+  @team:DataDog/observability-pipelines
+  Scenario: Validate an observability pipeline with amazon S3 source compression returns "OK" response
+    Given new "ValidatePipeline" request
+    And body with value {"data": {"attributes": {"config": {"destinations": [{"id": "datadog-logs-destination", "inputs": ["my-processor-group"], "type": "datadog_logs"}], "processor_groups": [{"enabled": true, "id": "my-processor-group", "include": "service:my-service", "inputs": ["amazon-s3-source"], "processors": [{"enabled": true, "id": "filter-processor", "include": "service:my-service", "type": "filter"}]}], "sources": [{"id": "amazon-s3-source", "type": "amazon_s3", "region": "us-east-1", "compression": "gzip"}]}, "name": "Pipeline with S3 Source Compression"}, "type": "pipelines"}}
+    When the request is sent
+    Then the response status is 200 OK
+    And the response "errors" has length 0
+
   @team:DataDog/observability-pipelines
   Scenario: Validate an observability pipeline with destination secret key returns "OK" response
     Given new "ValidatePipeline" request

features/v2/observability_pipelines.feature

@@ -3758,6 +3758,7 @@ def overrides
           "v2.observability_pipeline_amazon_s3_generic_encoding_parquet" => "ObservabilityPipelineAmazonS3GenericEncodingParquet",
           "v2.observability_pipeline_amazon_s3_generic_encoding_parquet_type" => "ObservabilityPipelineAmazonS3GenericEncodingParquetType",
           "v2.observability_pipeline_amazon_s3_source" => "ObservabilityPipelineAmazonS3Source",
+          "v2.observability_pipeline_amazon_s3_source_compression" => "ObservabilityPipelineAmazonS3SourceCompression",
           "v2.observability_pipeline_amazon_s3_source_type" => "ObservabilityPipelineAmazonS3SourceType",
           "v2.observability_pipeline_amazon_security_lake_destination" => "ObservabilityPipelineAmazonSecurityLakeDestination",
           "v2.observability_pipeline_amazon_security_lake_destination_type" => "ObservabilityPipelineAmazonSecurityLakeDestinationType",

lib/datadog_api_client/inflector.rb

@@ -18,7 +18,7 @@
 
 module DatadogAPIClient::V2
   # The `amazon_s3` source ingests logs from an Amazon S3 bucket.
-  # It supports AWS authentication and TLS encryption.
+  # It supports AWS authentication, TLS encryption, and configurable compression.
   # 
   # **Supported pipeline types:** logs
   class ObservabilityPipelineAmazonS3Source
@@ -28,6 +28,9 @@ class ObservabilityPipelineAmazonS3Source
     # If omitted, the system’s default credentials are used (for example, the IAM role and environment variables).
     attr_accessor :auth
 
+    # Compression format for objects retrieved from the S3 bucket. Use `auto` to detect compression from the object's Content-Encoding header or file extension.
+    attr_accessor :compression
+
     # The unique identifier for this component. Used in other parts of the pipeline to reference this component (for example, as the `input` to downstream components).
     attr_reader :id
 
@@ -50,6 +53,7 @@ class ObservabilityPipelineAmazonS3Source
     def self.attribute_map
       {
         :'auth' => :'auth',
+        :'compression' => :'compression',
         :'id' => :'id',
         :'region' => :'region',
         :'tls' => :'tls',
@@ -63,6 +67,7 @@ def self.attribute_map
     def self.openapi_types
       {
         :'auth' => :'ObservabilityPipelineAwsAuth',
+        :'compression' => :'ObservabilityPipelineAmazonS3SourceCompression',
         :'id' => :'String',
         :'region' => :'String',
         :'tls' => :'ObservabilityPipelineTls',
@@ -93,6 +98,10 @@ def initialize(attributes = {})
         self.auth = attributes[:'auth']
       end
 
+      if attributes.key?(:'compression')
+        self.compression = attributes[:'compression']
+      end
+
       if attributes.key?(:'id')
         self.id = attributes[:'id']
       end
@@ -181,6 +190,7 @@ def ==(o)
       return true if self.equal?(o)
       self.class == o.class &&
           auth == o.auth &&
+          compression == o.compression &&
           id == o.id &&
           region == o.region &&
           tls == o.tls &&
@@ -193,7 +203,7 @@ def ==(o)
     # @return [Integer] Hash code
     # @!visibility private
     def hash
-      [auth, id, region, tls, type, url_key, additional_properties].hash
+      [auth, compression, id, region, tls, type, url_key, additional_properties].hash
     end
   end
 end

lib/datadog_api_client/v2/models/observability_pipeline_amazon_s3_source.rb

@@ -0,0 +1,29 @@
+=begin
+#Datadog API V2 Collection
+
+#Collection of all Datadog Public endpoints.
+
+The version of the OpenAPI document: 1.0
+Contact: support@datadoghq.com
+Generated by: https://github.com/DataDog/datadog-api-client-ruby/tree/master/.generator
+
+ Unless explicitly stated otherwise all files in this repository are licensed under the Apache-2.0 License.
+ This product includes software developed at Datadog (https://www.datadoghq.com/).
+ Copyright 2020-Present Datadog, Inc.
+
+=end
+
+require 'date'
+require 'time'
+
+module DatadogAPIClient::V2
+  # Compression format for objects retrieved from the S3 bucket. Use `auto` to detect compression from the object's Content-Encoding header or file extension.
+  class ObservabilityPipelineAmazonS3SourceCompression
+    include BaseEnumModel
+
+    AUTO = "auto".freeze
+    NONE = "none".freeze
+    GZIP = "gzip".freeze
+    ZSTD = "zstd".freeze
+  end
+end