From cbfd974b3047621d96bfb2174b8ba86b37d672c0 Mon Sep 17 00:00:00 2001 From: Zhengda Lu Date: Wed, 16 Jul 2025 09:46:27 -0400 Subject: [PATCH] deprecate schemas_collection, deadlock_collection and xe_collection config option (#20599) * deprecate schemas_collection, deadlock_collection and xe_collection config option * add changelog * sync config * sync config * fix tests * fix tests * fix deprecation version --- sqlserver/assets/configuration/spec.yaml | 135 +++++++++++++++++- sqlserver/changelog.d/20599.added | 1 + sqlserver/datadog_checks/sqlserver/config.py | 7 +- .../sqlserver/config_models/deprecations.py | 11 ++ .../sqlserver/config_models/instance.py | 81 ++++++++--- .../sqlserver/data/conf.yaml.example | 13 +- sqlserver/tests/test_deadlocks.py | 12 ++ sqlserver/tests/test_metadata.py | 12 ++ sqlserver/tests/test_xe_collection.py | 18 +++ 9 files changed, 256 insertions(+), 34 deletions(-) create mode 100644 sqlserver/changelog.d/20599.added create mode 100644 sqlserver/datadog_checks/sqlserver/config_models/deprecations.py diff --git a/sqlserver/assets/configuration/spec.yaml b/sqlserver/assets/configuration/spec.yaml index e43bc42eba8ab..d558b6f299fae 100644 --- a/sqlserver/assets/configuration/spec.yaml +++ b/sqlserver/assets/configuration/spec.yaml @@ -907,8 +907,8 @@ files: - name: collect_raw_query_statement description: | Configure the collection of raw query statements in query activity, and XE events. - To collect raw query statements from XE events, set `xe_collection.query_completions.enabled` and - `xe_collection.query_errors.enabled` to `true`. + To collect raw query statements from XE events, set `collect_xe.query_completions.enabled` and + `collect_xe.query_errors.enabled` to `true`. Raw query statements and execution plans may contain sensitive information (e.g., passwords) or personally identifiable information in query text. Enabling this option will allow the collection and ingestion of raw query statements and @@ -987,9 +987,37 @@ files: type: number example: 1800 display_default: 300 + - name: collect_schemas + description: | + Configure collection of schemas. If `database_autodiscovery` is not enabled, data is collected + only for the database configured with `database` parameter. + options: + - name: enabled + description: | + Enable schema collection. Requires `dbm: true`. Defaults to false. + value: + type: boolean + example: false + - name: collection_interval + description: | + Set the database schema collection interval (in seconds). Defaults to 600 seconds. + value: + type: number + example: 600 + - name: max_execution_time + description: | + Set the maximum time for schema collection (in seconds). Defaults to 10 seconds. + Capped by `collect_schemas.collection_interval` + value: + type: number + example: 10 - name: schemas_collection + hidden: true + deprecation: + Agent version: 7.69.0 + Migration: Use `collect_schemas` instead. description: | - Available for Agent 7.56 and newer. + DEPRECATED: Use `collect_schemas` instead. Configure collection of schemas. If `database_autodiscovery` is not enabled, data is collected only for the database configured with `database` parameter. options: @@ -1020,8 +1048,82 @@ files: value: example: false type: boolean + - name: collect_xe + description: | + Configure the collection of events from XE (Extended Events) sessions. Requires `dbm: true`. + + Set `collect_raw_query_statement.enabled` to `true` to collect the raw query statements for each event. + options: + - name: debug_sample_events + description: | + Set the maximum number of XE events to log in debug mode per collection. Used for troubleshooting. + This only affects logging when debug mode is enabled. Defaults to 3. + hidden: true + value: + type: integer + example: 3 + display_default: 3 + - name: query_completions + description: | + Configure the collection of completed queries from the `datadog_query_completions` XE session. + + Set `query_completions.enabled` to `true` to enable the collection of query completion events. + + Use `query_completions.collection_interval` to set the interval (in seconds) for the collection of + query completion events. Defaults to 10 seconds. If you intend on updating this value, + it is strongly recommended to use a consistent value throughout all SQL Server agent deployments. + + Use `query_completions.max_events` to set the maximum number of query completion events to process + per collection. Note that SQL Server's ring buffer has a maximum of 1000 events per query, + so values above 1000 will still be capped at 1000 by the database engine. Defaults to 1000. + value: + type: object + properties: + - name: enabled + type: boolean + example: false + - name: collection_interval + type: number + example: 10 + display_default: 10 + - name: max_events + type: integer + example: 1000 + display_default: 1000 + - name: query_errors + description: | + Configure the collection of query errors from the `datadog_query_errors` XE session. + + Set `query_errors.enabled` to `true` to enable the collection of query error events. + + Use `query_errors.collection_interval` to set the interval (in seconds) for the collection of + query error events. Defaults to 10 seconds. If you intend on updating this value, + it is strongly recommended to use a consistent value throughout all SQL Server agent deployments. + + Use `query_errors.max_events` to set the maximum number of query error events to process + per collection. Note that SQL Server's ring buffer has a maximum of 1000 events per query, + so values above 1000 will still be capped at 1000 by the database engine. Defaults to 1000. + value: + type: object + properties: + - name: enabled + type: boolean + example: false + - name: collection_interval + type: number + example: 10 + display_default: 10 + - name: max_events + type: integer + example: 1000 + display_default: 1000 - name: xe_collection + hidden: true + deprecation: + Agent version: 7.69.0 + Migration: Use `collect_xe` instead. description: | + DEPRECATED: Use `collect_xe` instead. Configure the collection of events from XE (Extended Events) sessions. Requires `dbm: true`. Set `collect_raw_query_statement.enabled` to `true` to collect the raw query statements for each event. @@ -1089,8 +1191,35 @@ files: type: integer example: 1000 display_default: 1000 + - name: collect_deadlocks + description: | + Configure the collection of deadlock data. + options: + - name: enabled + description: | + Enable the collection of deadlock data. Requires `dbm: true`. Disabled by default. + value: + type: boolean + example: false + - name: collection_interval + description: | + Set the interval for collecting deadlock data, in seconds. Defaults to 600 seconds. + value: + type: number + example: 600 + - name: max_deadlocks + description: | + Set the maximum number of deadlocks to retrieve per collection. + value: + type: number + example: 100 - name: deadlocks_collection + hidden: true + deprecation: + Agent version: 7.69.0 + Migration: Use `collect_deadlocks` instead. description: | + DEPRECATED: Use `collect_deadlocks` instead. Configure the collection of deadlock data. options: - name: enabled diff --git a/sqlserver/changelog.d/20599.added b/sqlserver/changelog.d/20599.added new file mode 100644 index 0000000000000..188e2bae0a270 --- /dev/null +++ b/sqlserver/changelog.d/20599.added @@ -0,0 +1 @@ +Add new collect_* configuration options (collect_schemas, collect_deadlocks, collect_xe) to replace deprecated *_collection options while maintaining backward compatibility. diff --git a/sqlserver/datadog_checks/sqlserver/config.py b/sqlserver/datadog_checks/sqlserver/config.py index dc5743721320f..6bdf3fc38dbc0 100644 --- a/sqlserver/datadog_checks/sqlserver/config.py +++ b/sqlserver/datadog_checks/sqlserver/config.py @@ -55,9 +55,10 @@ def __init__(self, init_config, instance, log): self.procedure_metrics_config: dict = instance.get('procedure_metrics', {}) or {} self.settings_config: dict = instance.get('collect_settings', {}) or {} self.activity_config: dict = instance.get('query_activity', {}) or {} - self.schema_config: dict = instance.get('schemas_collection', {}) or {} - self.deadlocks_config: dict = instance.get('deadlocks_collection', {}) or {} - self.xe_collection_config: dict = instance.get('xe_collection', {}) or {} + # Backward compatibility: check new names first, then fall back to old names + self.schema_config: dict = instance.get('collect_schemas', instance.get('schemas_collection', {})) or {} + self.deadlocks_config: dict = instance.get('collect_deadlocks', instance.get('deadlocks_collection', {})) or {} + self.xe_collection_config: dict = instance.get('collect_xe', instance.get('xe_collection', {})) or {} self.cloud_metadata: dict = {} aws: dict = instance.get('aws', {}) or {} gcp: dict = instance.get('gcp', {}) or {} diff --git a/sqlserver/datadog_checks/sqlserver/config_models/deprecations.py b/sqlserver/datadog_checks/sqlserver/config_models/deprecations.py new file mode 100644 index 0000000000000..eb453178b75d2 --- /dev/null +++ b/sqlserver/datadog_checks/sqlserver/config_models/deprecations.py @@ -0,0 +1,11 @@ +# (C) Datadog, Inc. 2025-present +# All rights reserved +# Licensed under a 3-clause BSD style license (see LICENSE) + + +def instance(): + return { + 'deadlocks_collection': {'Agent version': '7.69.0', 'Migration': 'Use `collect_deadlocks` instead.'}, + 'schemas_collection': {'Agent version': '7.69.0', 'Migration': 'Use `collect_schemas` instead.'}, + 'xe_collection': {'Agent version': '7.69.0', 'Migration': 'Use `collect_xe` instead.'}, + } diff --git a/sqlserver/datadog_checks/sqlserver/config_models/instance.py b/sqlserver/datadog_checks/sqlserver/config_models/instance.py index 3ede062134625..2f646a6d87b80 100644 --- a/sqlserver/datadog_checks/sqlserver/config_models/instance.py +++ b/sqlserver/datadog_checks/sqlserver/config_models/instance.py @@ -17,7 +17,7 @@ from datadog_checks.base.utils.functions import identity from datadog_checks.base.utils.models import validation -from . import defaults, validators +from . import defaults, deprecations, validators class AgentJobs(BaseModel): @@ -48,6 +48,16 @@ class Azure(BaseModel): fully_qualified_domain_name: Optional[str] = None +class CollectDeadlocks(BaseModel): + model_config = ConfigDict( + arbitrary_types_allowed=True, + frozen=True, + ) + collection_interval: Optional[float] = None + enabled: Optional[bool] = None + max_deadlocks: Optional[float] = None + + class CollectRawQueryStatement(BaseModel): model_config = ConfigDict( arbitrary_types_allowed=True, @@ -56,6 +66,16 @@ class CollectRawQueryStatement(BaseModel): enabled: Optional[bool] = None +class CollectSchemas(BaseModel): + model_config = ConfigDict( + arbitrary_types_allowed=True, + frozen=True, + ) + collection_interval: Optional[float] = None + enabled: Optional[bool] = None + max_execution_time: Optional[float] = None + + class CollectSettings(BaseModel): model_config = ConfigDict( arbitrary_types_allowed=True, @@ -65,6 +85,36 @@ class CollectSettings(BaseModel): enabled: Optional[bool] = None +class QueryCompletions(BaseModel): + model_config = ConfigDict( + arbitrary_types_allowed=True, + frozen=True, + ) + collection_interval: Optional[float] = Field(None, examples=[10]) + enabled: Optional[bool] = Field(None, examples=[False]) + max_events: Optional[int] = Field(None, examples=[1000]) + + +class QueryErrors(BaseModel): + model_config = ConfigDict( + arbitrary_types_allowed=True, + frozen=True, + ) + collection_interval: Optional[float] = Field(None, examples=[10]) + enabled: Optional[bool] = Field(None, examples=[False]) + max_events: Optional[int] = Field(None, examples=[1000]) + + +class CollectXe(BaseModel): + model_config = ConfigDict( + arbitrary_types_allowed=True, + frozen=True, + ) + debug_sample_events: Optional[int] = None + query_completions: Optional[QueryCompletions] = None + query_errors: Optional[QueryErrors] = None + + class CustomQuery(BaseModel): model_config = ConfigDict( arbitrary_types_allowed=True, @@ -357,26 +407,6 @@ class SchemasCollection(BaseModel): max_execution_time: Optional[float] = None -class QueryCompletions(BaseModel): - model_config = ConfigDict( - arbitrary_types_allowed=True, - frozen=True, - ) - collection_interval: Optional[float] = Field(None, examples=[10]) - enabled: Optional[bool] = Field(None, examples=[False]) - max_events: Optional[int] = Field(None, examples=[1000]) - - -class QueryErrors(BaseModel): - model_config = ConfigDict( - arbitrary_types_allowed=True, - frozen=True, - ) - collection_interval: Optional[float] = Field(None, examples=[10]) - enabled: Optional[bool] = Field(None, examples=[False]) - max_events: Optional[int] = Field(None, examples=[1000]) - - class XeCollection(BaseModel): model_config = ConfigDict( arbitrary_types_allowed=True, @@ -400,8 +430,11 @@ class InstanceConfig(BaseModel): autodiscovery_include: Optional[tuple[str, ...]] = None aws: Optional[Aws] = None azure: Optional[Azure] = None + collect_deadlocks: Optional[CollectDeadlocks] = None collect_raw_query_statement: Optional[CollectRawQueryStatement] = None + collect_schemas: Optional[CollectSchemas] = None collect_settings: Optional[CollectSettings] = None + collect_xe: Optional[CollectXe] = None command_timeout: Optional[int] = None connection_string: Optional[str] = None connector: Optional[str] = None @@ -448,6 +481,12 @@ class InstanceConfig(BaseModel): username: Optional[str] = None xe_collection: Optional[XeCollection] = None + @model_validator(mode='before') + def _handle_deprecations(cls, values, info): + fields = info.context['configured_fields'] + validation.utils.handle_deprecations('instances', deprecations.instance(), fields, info.context) + return values + @model_validator(mode='before') def _initial_validation(cls, values): return validation.core.initialize_config(getattr(validators, 'initialize_instance', identity)(values)) diff --git a/sqlserver/datadog_checks/sqlserver/data/conf.yaml.example b/sqlserver/datadog_checks/sqlserver/data/conf.yaml.example index 1c8a7b17dfd4f..b1c0baf98bf60 100644 --- a/sqlserver/datadog_checks/sqlserver/data/conf.yaml.example +++ b/sqlserver/datadog_checks/sqlserver/data/conf.yaml.example @@ -657,8 +657,8 @@ instances: # keep_identifier_quotation: false ## Configure the collection of raw query statements in query activity, and XE events. - ## To collect raw query statements from XE events, set `xe_collection.query_completions.enabled` and - ## `xe_collection.query_errors.enabled` to `true`. + ## To collect raw query statements from XE events, set `collect_xe.query_completions.enabled` and + ## `collect_xe.query_errors.enabled` to `true`. ## Raw query statements and execution plans may contain sensitive information (e.g., passwords) ## or personally identifiable information in query text. ## Enabling this option will allow the collection and ingestion of raw query statements and @@ -783,11 +783,10 @@ instances: # # ignore_missing_database: false - ## Available for Agent 7.56 and newer. ## Configure collection of schemas. If `database_autodiscovery` is not enabled, data is collected ## only for the database configured with `database` parameter. # - # schemas_collection: + # collect_schemas: ## @param enabled - boolean - optional - default: false ## Enable schema collection. Requires `dbm: true`. Defaults to false. @@ -801,7 +800,7 @@ instances: ## @param max_execution_time - number - optional - default: 10 ## Set the maximum time for schema collection (in seconds). Defaults to 10 seconds. - ## Capped by `schemas_collection.collection_interval` + ## Capped by `collect_schemas.collection_interval` # # max_execution_time: 10 @@ -816,7 +815,7 @@ instances: ## ## Set `collect_raw_query_statement.enabled` to `true` to collect the raw query statements for each event. # - # xe_collection: + # collect_xe: ## @param query_completions - mapping - optional ## Configure the collection of completed queries from the `datadog_query_completions` XE session. @@ -850,7 +849,7 @@ instances: ## Configure the collection of deadlock data. # - # deadlocks_collection: + # collect_deadlocks: ## @param enabled - boolean - optional - default: false ## Enable the collection of deadlock data. Requires `dbm: true`. Disabled by default. diff --git a/sqlserver/tests/test_deadlocks.py b/sqlserver/tests/test_deadlocks.py index 62aebeb26e7c8..8aa7fe68b4a8b 100644 --- a/sqlserver/tests/test_deadlocks.py +++ b/sqlserver/tests/test_deadlocks.py @@ -364,3 +364,15 @@ def test_deadlock_calls_obfuscator(deadlocks_collection_instance): result_string = result_string.replace('\t', '').replace('\n', '') result_string = re.sub(r'\s{2,}', ' ', result_string) assert expected_xml_string == result_string + + +@pytest.mark.unit +def test_collect_deadlocks_config(dbm_instance): + dbm_instance['collect_deadlocks'] = {"enabled": True, 'collection_interval': 0.2} + check = SQLServer(CHECK_NAME, {}, [dbm_instance]) + assert check._config.deadlocks_config == {"enabled": True, 'collection_interval': 0.2} + + dbm_instance.pop('collect_deadlocks') + dbm_instance['deadlocks_collection'] = {"enabled": True, 'collection_interval': 0.3} + check = SQLServer(CHECK_NAME, {}, [dbm_instance]) + assert check._config.deadlocks_config == {"enabled": True, 'collection_interval': 0.3} diff --git a/sqlserver/tests/test_metadata.py b/sqlserver/tests/test_metadata.py index 9b6b52798f0c0..dc3886e7a8e53 100644 --- a/sqlserver/tests/test_metadata.py +++ b/sqlserver/tests/test_metadata.py @@ -403,3 +403,15 @@ def test_schemas_collection_truncated(aggregator, dd_run_check, dbm_instance): ): found = True assert found + + +@pytest.mark.unit +def test_collect_schemas_config(dbm_instance): + dbm_instance['collect_schemas'] = {"enabled": True, "max_execution_time": 0} + check = SQLServer(CHECK_NAME, {}, [dbm_instance]) + assert check._config.schema_config == {"enabled": True, "max_execution_time": 0} + + dbm_instance.pop('collect_schemas') + dbm_instance['schemas_collection'] = {"enabled": True, "max_execution_time": 0} + check = SQLServer(CHECK_NAME, {}, [dbm_instance]) + assert check._config.schema_config == {"enabled": True, "max_execution_time": 0} diff --git a/sqlserver/tests/test_xe_collection.py b/sqlserver/tests/test_xe_collection.py index 8eb431448e3ad..62ee1dac3cdc6 100644 --- a/sqlserver/tests/test_xe_collection.py +++ b/sqlserver/tests/test_xe_collection.py @@ -1271,3 +1271,21 @@ def capture_payload(payload, **kwargs): # Verify the batch exists and contains multiple events assert batch_key in original_payload, f"Missing '{batch_key}' array in payload" assert len(original_payload[batch_key]) > 1, "Expected multiple events in the batch" + + +@pytest.mark.unit +def test_collect_xe_config(instance_docker): + instance_docker['collect_xe'] = {"query_completions": {"enabled": True}, "query_errors": {"enabled": True}} + check = SQLServer(CHECK_NAME, {}, [instance_docker]) + assert check._config.xe_collection_config == { + "query_completions": {"enabled": True}, + "query_errors": {"enabled": True}, + } + + instance_docker.pop('collect_xe') + instance_docker['xe_collection'] = {"query_completions": {"enabled": True}, "query_errors": {"enabled": True}} + check = SQLServer(CHECK_NAME, {}, [instance_docker]) + assert check._config.xe_collection_config == { + "query_completions": {"enabled": True}, + "query_errors": {"enabled": True}, + }