[pull] master from DataDog:master

.github/workflows/flaky-tests.yml

@@ -19,18 +19,16 @@ jobs:
       repo: core
 
       # Options
-      # Because -m starts with a dash, 
-      # click (the library used to build the CLI) will 
+      # Because -m starts with a dash,
+      # click (the library used to build the CLI) will
       # interpret it as an option, not as an argument.
-      # To avoid this, using -- syntax, 
-      # which tells the command that everything following it 
+      # To avoid this, using -- syntax,
+      # which tells the command that everything following it
       # should be treated as positional arguments, not options
       pytest-args: '-m flaky'
     secrets: inherit
 
     permissions:
-       # needed for codecov in test-target.yml, allows the action to get a JWT signed by Github
-       id-token: write
        # needed for compute-matrix in test-target.yml
        contents: read
 
@@ -45,7 +43,7 @@ jobs:
       checks: write # Needed for test-results-master
       contents: write # Needed for test-results-master
       pull-requests: write # Needed for test-results-master
-      
+
     uses: ./.github/workflows/test-results-master.yml
     secrets: inherit
 

.github/workflows/master.yml

@@ -18,23 +18,21 @@ jobs:
       repo: core
 
       # Options
-      # Because -m starts with a dash, 
-      # click (the library used to build the CLI) will 
+      # Because -m starts with a dash,
+      # click (the library used to build the CLI) will
       # interpret it as an option, not as an argument.
-      # To avoid this, using -- syntax, 
-      # which tells the command that everything following it 
+      # To avoid this, using -- syntax,
+      # which tells the command that everything following it
       # should be treated as positional arguments, not options
       pytest-args: '-m "not flaky"'
     secrets: inherit
 
     permissions:
-       # needed for codecov in test-target.yml, allows the action to get a JWT signed by Github
-       id-token: write
        # needed for compute-matrix in test-target.yml
        contents: read
 
   publish-test-results:
-    
+
     needs:
     - test
 

.github/workflows/nightly-base-package.yml

@@ -15,8 +15,6 @@ jobs:
     uses: ./.github/workflows/test-all.yml
 
     permissions:
-       # needed for codecov in test-target.yml, allows the action to get a JWT signed by Github
-       id-token: write
        # needed for compute-matrix in test-target.yml
        contents: read
 

.github/workflows/pr-all.yml

@@ -17,8 +17,6 @@ jobs:
     uses: ./.github/workflows/test-all.yml
 
     permissions:
-       # needed for codecov in test-target.yml, allows the action to get a JWT signed by Github
-       id-token: write
        # needed for compute-matrix in test-target.yml
        contents: read
 

.github/workflows/pr.yml

@@ -24,7 +24,5 @@ jobs:
     secrets: inherit
 
     permissions:
-       # needed for codecov in test-target.yml, allows the action to get a JWT signed by Github
-       id-token: write
        # needed for compute-matrix in test-target.yml
        contents: read

.github/workflows/test-agent.yml

@@ -50,8 +50,6 @@ jobs:
       skip-ddev-tests: true
     secrets: inherit
     permissions:
-       # needed for codecov in test-target.yml, allows the action to get a JWT signed by Github
-       id-token: write
        # needed for compute-matrix in test-target.yml
        contents: read
 

.github/workflows/test-fips-e2e.yml

@@ -46,8 +46,6 @@ jobs:
       TRACE_CAPTURE_LOG: "trace-captures/output.log"
 
     permissions:
-       # needed for codecov in test-target.yml, allows the action to get a JWT signed by Github
-       id-token: write
        # needed for compute-matrix in test-target.yml
        contents: read
 
@@ -151,12 +149,3 @@ jobs:
         name: "test-results-${{ inputs.target || 'tls' }}"
         path: "${{ env.TEST_RESULTS_BASE_DIR }}"
 
-    - name: Upload coverage data
-      if: >
-        !github.event.repository.private &&
-        always()
-      uses: codecov/codecov-action@15559ed290fa727036809b67ab0f646ffa6c5158
-      with:
-        use_oidc: true
-        files: "${{ inputs.target || 'tls' }}/coverage.xml"
-        flags: "${{ inputs.target || 'tls' }}"

.github/workflows/test-target.yml

@@ -15,7 +15,7 @@ on:
         description: "Specific target env to run"
         required: false
         default: ""
-        type: string      
+        type: string
       platform:
         required: true
         type: string
@@ -114,8 +114,6 @@ jobs:
       TRACE_CAPTURE_LOG: "trace-captures/output.log"
 
     permissions:
-       # needed for codecov in test-target.yml, allows the action to get a JWT signed by Github
-       id-token: write
        # needed for compute-matrix in test-target.yml
        contents: read
     steps:
@@ -413,15 +411,3 @@ jobs:
       with:
         name: "test-results-${{ inputs.target }}${{ inputs.target-env && format('-{0}', inputs.target-env) || '' }}-${{ inputs.platform }}"
         path: "${{ env.TEST_RESULTS_BASE_DIR }}"
-
-    - name: Upload coverage data
-      if: >
-        !github.event.repository.private &&
-        always() &&
-        inputs.pytest-args != '-m flaky'
-      # Flaky tests will have low coverage, don't upload it to avoid pipeline failure
-      uses: codecov/codecov-action@15559ed290fa727036809b67ab0f646ffa6c5158
-      with:
-        use_oidc: true
-        files: "${{ inputs.target }}/coverage.xml"
-        flags: "${{ inputs.target }}"

.github/workflows/weekly-latest.yml

@@ -15,8 +15,6 @@ jobs:
       latest: true
     secrets: inherit
     permissions:
-       # needed for codecov in test-target.yml, allows the action to get a JWT signed by Github
-       id-token: write
        # needed for compute-matrix in test-target.yml
        contents: read
 

LICENSE-3rdparty.csv

@@ -22,6 +22,7 @@ cryptography,PyPI,BSD-3-Clause,Copyright (c) Individual contributors.
 cryptography,PyPI,PSF,Copyright (c) Individual contributors.
 ddtrace,PyPI,BSD-3-Clause,"Copyright 2016 Datadog, Inc."
 dnspython,PyPI,ISC,Copyright (C) Dnspython Contributors
+fastavro,PyPI,MIT,Copyright (c) 2011 Miki Tebeka
 flup,Vendor,BSD-3-Clause,Copyright (c) 2005 Allan Saddi. All Rights Reserved.
 flup-py3,Vendor,BSD-3-Clause,"Copyright (c) 2005, 2006 Allan Saddi <allan@saddi.com> All rights reserved."
 foundationdb,PyPI,Apache-2.0,Copyright 2017 FoundationDB

agent_requirements.in

@@ -13,6 +13,7 @@ confluent-kafka==2.8.0
 cryptography==45.0.4
 ddtrace==3.9.3
 dnspython==2.7.0
+fastavro==1.11.1
 foundationdb==6.3.25
 hazelcast-python-client==5.5.0
 in-toto==2.0.0

kafka_consumer/changelog.d/20862.added

@@ -0,0 +1 @@
+Add support for Avro and Protobuf formats for Data Streams messages feature.

kafka_consumer/datadog_checks/kafka_consumer/config.py

@@ -148,19 +148,47 @@ def _validate_live_messages_configs(self):
             ):
                 self.log.debug('Data Streams live messages configuration missing required kafka parameters.', kafka)
                 continue
-            # Only json format is supported for Data Streams live messages
+
+            # Validate value format
             if kafka.get('value_format', '') == '':
                 kafka['value_format'] = 'json'
-            if kafka['value_format'] != 'json':
+            value_format = kafka['value_format']
+            if value_format not in ['json', 'avro', 'protobuf']:
                 self.log.debug(
-                    'Only json format is supported for Data Streams live messages, got %s', kafka['value_format']
+                    'Unsupported value format for Data Streams live messages, got %s. '
+                    'Supported formats: json, avro, protobuf',
+                    value_format,
                 )
+                continue
+
+            # Validate key format
             if kafka.get('key_format', '') == '':
                 kafka['key_format'] = 'json'
-            if kafka['key_format'] != 'json':
+            key_format = kafka['key_format']
+            if key_format not in ['json', 'avro', 'protobuf']:
                 self.log.debug(
-                    'Only json format is supported for Data Streams live messages, got %s', kafka['key_format']
+                    'Unsupported key format for Data Streams live messages, got %s. '
+                    'Supported formats: json, avro, protobuf',
+                    key_format,
                 )
+                continue
+
+            # Validate schemas for non-JSON formats
+            if value_format in ['avro', 'protobuf']:
+                if 'value_schema' not in kafka or not kafka['value_schema']:
+                    self.log.debug(
+                        'Value schema is required for %s format in Data Streams live messages configuration',
+                        value_format,
+                    )
+                    continue
+
+            if key_format in ['avro', 'protobuf']:
+                if 'key_schema' not in kafka or not kafka['key_schema']:
+                    self.log.debug(
+                        'Key schema is required for %s format in Data Streams live messages configuration', key_format
+                    )
+                    continue
+
             live_messages_configs.append(config)
         self.live_messages_configs = live_messages_configs