[pull] master from DataDog:master

fluentd/assets/configuration/spec.yaml

@@ -12,6 +12,7 @@ files:
       fleet_configurable: true
       value:
         type: string
+        require_trusted_provider: true
         example: fluentd
     - template: init_config/http
     - template: init_config/default
@@ -31,6 +32,7 @@ files:
       fleet_configurable: true
       value:
         type: string
+        require_trusted_provider: true
         example: fluentd
     - name: plugin_ids
       description: Enter your Plugin IDs to monitor a specific scope of plugins.

fluentd/changelog.d/23629.added

@@ -0,0 +1 @@
+Add support for security validation in models for the `fluentd` configuration option.

fluentd/datadog_checks/fluentd/config_models/instance.py

@@ -22,7 +22,7 @@
 
 
 SECURE_FIELD_NAMES = frozenset(
-    ['auth_token', 'kerberos_cache', 'kerberos_keytab', 'tls_ca_cert', 'tls_cert', 'tls_private_key']
+    ['auth_token', 'fluentd', 'kerberos_cache', 'kerberos_keytab', 'tls_ca_cert', 'tls_cert', 'tls_private_key']
 )
 
 

fluentd/datadog_checks/fluentd/config_models/shared.py

@@ -19,6 +19,9 @@
 from . import defaults, validators
 
 
+SECURE_FIELD_NAMES = frozenset(['fluentd'])
+
+
 class Proxy(BaseModel):
     model_config = ConfigDict(
         arbitrary_types_allowed=True,
@@ -51,6 +54,11 @@ def _validate(cls, value, info):
         field_name = field.alias or info.field_name
         if field_name in info.context['configured_fields']:
             value = getattr(validators, f'shared_{info.field_name}', identity)(value, field=field)
+
+            if info.field_name in SECURE_FIELD_NAMES:
+                validation.security.check_field_trusted_provider(
+                    info.field_name, value, info.context.get('security_config')
+                )
         else:
             value = getattr(defaults, f'shared_{info.field_name}', lambda: value)()
 

mysql/changelog.d/23562.added

@@ -0,0 +1 @@
+Add collection of global wait event metrics from performance_schema for DBM-enabled instances.

mysql/datadog_checks/mysql/mysql.py

@@ -67,6 +67,7 @@
     QUERY_DEADLOCKS,
     QUERY_ERRORS_RAISED,
     QUERY_USER_CONNECTIONS,
+    QUERY_WAIT_EVENT_SUMMARY,
     SQL_95TH_PERCENTILE,
     SQL_AVG_QUERY_RUN_TIME,
     SQL_GROUP_REPLICATION_MEMBER,
@@ -468,8 +469,10 @@ def _get_runtime_queries(self, db):
 
         if self.global_variables.performance_schema_enabled:
             queries.extend([QUERY_USER_CONNECTIONS])
-            if not self.is_mariadb and self.version.version_compatible((8, 0, 0)) and self._config.dbm_enabled:
-                queries.extend([QUERY_ERRORS_RAISED])
+            if self._config.dbm_enabled:
+                queries.extend([QUERY_WAIT_EVENT_SUMMARY])
+                if not self.is_mariadb and self.version.version_compatible((8, 0, 0)):
+                    queries.extend([QUERY_ERRORS_RAISED])
         if self._index_metrics.include_index_metrics:
             queries.extend(self._index_metrics.queries)
         self._runtime_queries_cached = self._new_query_executor(queries)

mysql/datadog_checks/mysql/queries.py

@@ -262,6 +262,29 @@
     ],
 }
 
+QUERY_WAIT_EVENT_SUMMARY = {
+    'name': 'performance_schema.events_waits_summary_global_by_event_name',
+    'query': """
+        SELECT
+            event_name,
+            count_star,
+            sum_timer_wait / 1000,
+            avg_timer_wait / 1000,
+            max_timer_wait / 1000
+        FROM performance_schema.events_waits_summary_global_by_event_name
+        WHERE count_star > 0
+        ORDER BY sum_timer_wait DESC
+        LIMIT 200
+    """.strip(),
+    'columns': [
+        {'name': 'wait_event', 'type': 'tag'},
+        {'name': 'mysql.performance.wait_event.count', 'type': 'monotonic_count'},
+        {'name': 'mysql.performance.wait_event.time', 'type': 'monotonic_count'},
+        {'name': 'mysql.performance.wait_event.avg_time', 'type': 'gauge'},
+        {'name': 'mysql.performance.wait_event.max_time', 'type': 'gauge'},
+    ],
+}
+
 
 def show_replica_status_query(version, is_mariadb: bool, channel: str = '') -> tuple[str, tuple[str, ...]]:
     if version.version_compatible((10, 5, 1)) or not is_mariadb and version.version_compatible((8, 0, 22)):

mysql/metadata.csv

@@ -215,6 +215,10 @@ mysql.performance.threads_created,count,,thread,,"The number of threads created
 mysql.performance.threads_running,gauge,,thread,,The number of threads that are not sleeping.,0,mysql,threads running,
 mysql.performance.user_connections,gauge,,connection,,"The number of user connections. Tags: `processlist_db`, `processlist_host`, `processlist_state`, `processlist_user`",0,mysql,user conns,
 mysql.performance.user_time,gauge,,percent,,Percentage of CPU time spent in user space by MySQL.,-1,mysql,cpu user,
+mysql.performance.wait_event.avg_time,gauge,,nanosecond,,Average wait time per occurrence of a wait event (DBM only). Tagged by `wait_event`.,0,mysql,wait event avg time,
+mysql.performance.wait_event.count,count,,event,,Total number of occurrences of a wait event (DBM only). Tagged by `wait_event`.,0,mysql,wait event count,
+mysql.performance.wait_event.max_time,gauge,,nanosecond,,Maximum single-occurrence wait time for a wait event (DBM only). Tagged by `wait_event`.,0,mysql,wait event max time,
+mysql.performance.wait_event.time,count,,nanosecond,,Total accumulated wait time for a wait event (DBM only). Tagged by `wait_event`.,0,mysql,wait event time,
 mysql.queries.count,count,,query,,The total count of executed queries per normalized query and schema. (DBM only),0,mysql,mysql queries count,cpu
 mysql.queries.created_tmp_disk_tables,count,,table,,The total count of temporary tables that exceeded tmp_table_size and were written to disk per normalized query and schema. (DBM only),0,mysql,mysql queries created tmp disk tables,
 mysql.queries.created_tmp_tables,count,,table,,The total count of in-memory temporary tables created during execution per normalized query and schema. (DBM only),0,mysql,mysql queries created tmp tables,

mysql/tests/test_mysql.py

@@ -1065,3 +1065,44 @@ def test_errors_raised_metric_with_dbm(aggregator, dd_run_check, instance_basic,
     else:
         # In all other cases the metric should not be present
         aggregator.assert_metric('mysql.performance.errors_raised', count=0)
+
+
+@pytest.mark.integration
+@pytest.mark.usefixtures('dd_environment')
+@pytest.mark.parametrize(
+    'dbm_enabled',
+    [
+        pytest.param(True, id="dbm_enabled"),
+        pytest.param(False, id="dbm_disabled"),
+    ],
+)
+def test_wait_event_summary_metrics_with_dbm(aggregator, dd_run_check, instance_basic, dbm_enabled):
+    instance_basic['dbm'] = dbm_enabled
+    if dbm_enabled:
+        instance_basic['collect_settings'] = {'enabled': False}
+        instance_basic['query_activity'] = {'enabled': False}
+        instance_basic['query_samples'] = {'enabled': False}
+        instance_basic['query_metrics'] = {'enabled': False}
+
+    mysql_check = MySql(common.CHECK_NAME, {}, [instance_basic])
+    dd_run_check(mysql_check)
+
+    aggregator.assert_service_check('mysql.can_connect', status=MySql.OK, count=1)
+
+    wait_event_metrics = [
+        'mysql.performance.wait_event.count',
+        'mysql.performance.wait_event.time',
+        'mysql.performance.wait_event.avg_time',
+        'mysql.performance.wait_event.max_time',
+    ]
+    if dbm_enabled:
+        for metric in wait_event_metrics:
+            aggregator.assert_metric(metric, at_least=1)
+        # Verify wait_event tag is present on submitted metrics
+        for metric in wait_event_metrics:
+            for m in aggregator.metrics(metric):
+                wait_event_tags = [t for t in m.tags if t.startswith('wait_event:')]
+                assert len(wait_event_tags) == 1, "Expected exactly one wait_event tag on {}".format(metric)
+    else:
+        for metric in wait_event_metrics:
+            aggregator.assert_metric(metric, count=0)

postgres/assets/configuration/spec.yaml

@@ -953,7 +953,7 @@ files:
             Enable collection of database schemas. Requires `dbm: true`.
           value:
             type: boolean
-            example: false
+            example: true
         - name: max_tables
           fleet_configurable: true
           description: |
@@ -1189,9 +1189,11 @@ files:
           description: |
             Configuration section used for Azure AD Authentication.
 
-            This supports using System, User assigned managed identities or
+            This supports using System or User assigned managed identities, as well as
             workload identity federation (e.g. on AKS).
 
+            If enabled, the `password` field will be ignored.
+
             For more information on configuration, see
             https://docs.datadoghq.com/database_monitoring/guide/managed_authentication
 
@@ -1201,17 +1203,15 @@ files:
             For more information on Workload Identity, see the Azure docs
             https://learn.microsoft.com/en-us/azure/aks/workload-identity-overview
 
-            To enable Azure AD Authentication, set `azure.managed_authentication.enabled` to `true`.
-            Additionally set `azure.managed_authentication.auth_type` to `managed_identity` (default)
-            or `workload_identity`.
-            For managed identity, `client_id` is required.
-            For workload identity, `client_id` and `tenant_id` are optional overrides. By default they are
-            read from the `AZURE_CLIENT_ID` and `AZURE_TENANT_ID` environment variables, which are
-            automatically injected by the AKS workload identity webhook along with `AZURE_FEDERATED_TOKEN_FILE`.
-            If `azure.managed_authentication.enabled` is set, then the `password` fields will be ignored.
+            The following fields are supported:
 
-            For more information on scopes, see the Azure docs
-            https://learn.microsoft.com/en-us/azure/active-directory/develop/scopes-oidc
+            enabled: Set to `true` to enable Azure AD authentication.
+            auth_type: The authentication method. Use `managed_identity` (default) or `workload_identity` for AKS.
+            client_id: The client ID of the managed identity or app registration. Required for `managed_identity`.
+              Optional for `workload_identity` (defaults to the `AZURE_CLIENT_ID` environment variable).
+            tenant_id: The Azure AD tenant ID. Only used for `workload_identity`
+              (defaults to the `AZURE_TENANT_ID` environment variable).
+            identity_scope: The permission scope for the identity token.
           value:
             type: object
             properties:
@@ -1221,17 +1221,22 @@ files:
             - name: auth_type
               type: string
               example: managed_identity
-              default: managed_identity
+              description: |
+                The authentication method. Use `managed_identity` (default) or `workload_identity` for AKS.
             - name: client_id
               description: |
+                The client ID of the managed identity or application registration.
                 Required for `managed_identity` auth. Optional for `workload_identity`,
                 where it defaults to the `AZURE_CLIENT_ID` environment variable.
               type: string
             - name: tenant_id
               description: |
-                Only used for `workload_identity` auth.
+                The Azure AD tenant ID. Only used for `workload_identity` auth.
+                Defaults to the `AZURE_TENANT_ID` environment variable.
               type: string
             - name: identity_scope
+              description: |
+                The permission scope from where to access the identity token.
               type: string
               example: https://ossrdbms-aad.database.windows.net/.default
     - name: obfuscator_options

postgres/changelog.d/23625.fixed

@@ -0,0 +1 @@
+Cleanup config field descriptions for Azure Cloud authentication mechanisms

postgres/changelog.d/23626.added

@@ -0,0 +1 @@
+Enable schema collection for Postgres by default. This functionality still requires `dbm:true` or `data_observability.enabled:true`. To disable, set `collect_schemas.enabled: false`.

postgres/datadog_checks/postgres/config_models/dict_defaults.py

@@ -86,7 +86,7 @@ def instance_collect_settings():
 
 def instance_collect_schemas():
     return instance.CollectSchemas(
-        enabled=False,
+        enabled=True,
         max_tables=300,
         max_columns=50,
         collection_interval=600,

postgres/datadog_checks/postgres/config_models/instance.py

@@ -47,14 +47,25 @@ class ManagedAuthentication1(BaseModel):
         arbitrary_types_allowed=True,
         frozen=True,
     )
-    auth_type: Optional[str] = Field('managed_identity', examples=['managed_identity'])
+    auth_type: Optional[str] = Field(
+        None,
+        description='The authentication method. Use `managed_identity` (default) or `workload_identity` for AKS.\n',
+        examples=['managed_identity'],
+    )
     client_id: Optional[str] = Field(
         None,
-        description='Required for `managed_identity` auth. Optional for `workload_identity`,\nwhere it defaults to the `AZURE_CLIENT_ID` environment variable.\n',
+        description='The client ID of the managed identity or application registration.\nRequired for `managed_identity` auth. Optional for `workload_identity`,\nwhere it defaults to the `AZURE_CLIENT_ID` environment variable.\n',
     )
     enabled: Optional[bool] = Field(None, examples=[False])
-    identity_scope: Optional[str] = Field(None, examples=['https://ossrdbms-aad.database.windows.net/.default'])
-    tenant_id: Optional[str] = Field(None, description='Only used for `workload_identity` auth.\n')
+    identity_scope: Optional[str] = Field(
+        None,
+        description='The permission scope from where to access the identity token.\n',
+        examples=['https://ossrdbms-aad.database.windows.net/.default'],
+    )
+    tenant_id: Optional[str] = Field(
+        None,
+        description='The Azure AD tenant ID. Only used for `workload_identity` auth.\nDefaults to the `AZURE_TENANT_ID` environment variable.\n',
+    )
 
 
 class Azure(BaseModel):

postgres/datadog_checks/postgres/data/conf.yaml.example

@@ -593,10 +593,10 @@ instances:
     #
     # collect_schemas:
 
-        ## @param enabled - boolean - optional - default: false
+        ## @param enabled - boolean - optional - default: true
         ## Enable collection of database schemas. Requires `dbm: true`.
         #
-        # enabled: false
+        # enabled: true
 
         ## @param max_tables - number - optional - default: 300
         ## Maximum amount of tables the Agent collects from the instance.
@@ -771,9 +771,11 @@ instances:
         ## @param managed_authentication - mapping - optional
         ## Configuration section used for Azure AD Authentication.
         ##
-        ## This supports using System, User assigned managed identities or
+        ## This supports using System or User assigned managed identities, as well as
         ## workload identity federation (e.g. on AKS).
         ##
+        ## If enabled, the `password` field will be ignored.
+        ##
         ## For more information on configuration, see
         ## https://docs.datadoghq.com/database_monitoring/guide/managed_authentication
         ##
@@ -783,17 +785,15 @@ instances:
         ## For more information on Workload Identity, see the Azure docs
         ## https://learn.microsoft.com/en-us/azure/aks/workload-identity-overview
         ##
-        ## To enable Azure AD Authentication, set `azure.managed_authentication.enabled` to `true`.
-        ## Additionally set `azure.managed_authentication.auth_type` to `managed_identity` (default)
-        ## or `workload_identity`.
-        ## For managed identity, `client_id` is required.
-        ## For workload identity, `client_id` and `tenant_id` are optional overrides. By default they are
-        ## read from the `AZURE_CLIENT_ID` and `AZURE_TENANT_ID` environment variables, which are
-        ## automatically injected by the AKS workload identity webhook along with `AZURE_FEDERATED_TOKEN_FILE`.
-        ## If `azure.managed_authentication.enabled` is set, then the `password` fields will be ignored.
+        ## The following fields are supported:
         ##
-        ## For more information on scopes, see the Azure docs
-        ## https://learn.microsoft.com/en-us/azure/active-directory/develop/scopes-oidc
+        ## enabled: Set to `true` to enable Azure AD authentication.
+        ## auth_type: The authentication method. Use `managed_identity` (default) or `workload_identity` for AKS.
+        ## client_id: The client ID of the managed identity or app registration. Required for `managed_identity`.
+        ##   Optional for `workload_identity` (defaults to the `AZURE_CLIENT_ID` environment variable).
+        ## tenant_id: The Azure AD tenant ID. Only used for `workload_identity`
+        ##   (defaults to the `AZURE_TENANT_ID` environment variable).
+        ## identity_scope: The permission scope for the identity token.
         #
         # managed_authentication: {}
 

postgres/tests/test_config_defaults.py

@@ -115,7 +115,7 @@
     },
     # === DBM: Schema collection ===
     'collect_schemas': {
-        'enabled': False,
+        'enabled': True,
         'max_tables': 300,
         'max_columns': 50,
         'collection_interval': 600,