[pull] master from DataDog:master

.deps/builder_inputs.toml

@@ -11,7 +11,7 @@
 # Hash inputs are defined in .builders/inputs_hash.py (SHARED_INPUTS,
 # RESOLUTION_INPUTS).
 [resolution]
-hash = "779028a5905d9bd45a2a151aa0402a5e464166eeba8ecbc149d625235d5c0a63"
+hash = "49b911b71bf751d20b0253d733ffe78f0d7bb8ee26d2f7913246288b94a2d2db"
 
 [images]
 linux-aarch64 = "f1b68d5fadd431f289254a30a6c0c71cac8170fe968c5a7ab3836f6f452d411d"

active_directory/changelog.d/24321.added

@@ -0,0 +1 @@
+Update dependencies

active_directory/pyproject.toml

@@ -36,7 +36,7 @@ dynamic = [
 
 [project.optional-dependencies]
 deps = [
-    "pywin32==311; sys_platform == 'win32'",
+    "pywin32==312; sys_platform == 'win32'",
 ]
 
 [project.urls]

agent_requirements.in

@@ -2,20 +2,20 @@ aerospike==7.1.1; sys_platform != 'win32' and sys_platform != 'darwin'
 aws-msk-iam-sasl-signer-python==1.0.2
 aws-requests-auth==0.4.3
 azure-identity==1.25.3
-beautifulsoup4==4.14.3
+beautifulsoup4==4.15.0
 binary==1.0.2
-boto3==1.42.72
-botocore==1.42.72
-cachetools==7.0.5
-clickhouse-connect==0.14.1
+boto3==1.43.40
+botocore==1.43.40
+cachetools==7.1.4
+clickhouse-connect==1.4.1
 cm-client==45.0.4
 confluent-kafka==2.13.2
 cryptography==48.0.1
 ddtrace==3.19.5
 dnspython==2.8.0
-fastavro==1.12.1
+fastavro==1.12.2
 foundationdb==6.3.25
-hazelcast-python-client==5.6.0
+hazelcast-python-client==5.7.0
 in-toto==2.0.0
 jellyfish==1.2.1
 kubernetes==35.0.0
@@ -25,16 +25,16 @@ lxml==6.1.0
 lz4==4.4.5
 mmh3==5.2.1
 oauthlib==3.3.1
-openstacksdk==4.10.0
-orjson==3.11.7
-packaging==26.0
-paramiko==4.0.0
-prometheus-client==0.24.1
-protobuf==7.34.0
+openstacksdk==4.17.0
+orjson==3.11.9
+packaging==26.2
+paramiko==5.0.0
+prometheus-client==0.25.0
+protobuf==7.35.1
 psutil==6.0.0
-psycopg[c,pool]==3.3.3
+psycopg[c,pool]==3.3.4
 pyasn1==0.6.3
-pydantic==2.12.5
+pydantic==2.13.4
 pyjwt==2.13.0
 pymongo[srv]==4.8.0; python_version >= '3.9'
 pymqi==1.12.13
@@ -44,26 +44,26 @@ pyopenssl==26.2.0
 pysnmp-mibs==0.1.6
 pysnmp==7.1.22
 pysocks==1.7.1
-python-binary-memcached==0.31.4; sys_platform != 'win32'
+python-binary-memcached==0.32.0; sys_platform != 'win32'
 python-dateutil==2.9.0.post0
 python3-gearman==0.1.0; sys_platform != 'win32'
 pyvmomi==8.0.3.0.1
-pywin32==311; sys_platform == 'win32'
+pywin32==312; sys_platform == 'win32'
 pyyaml==6.0.3
 redis==7.3.0
 requests-kerberos==0.15.0
 requests-ntlm==1.3.0
 requests-oauthlib==2.0.0
 requests-toolbelt==1.0.0
 requests-unixsocket2==1.0.1
-requests==2.33.0
+requests==2.34.2
 rethinkdb==2.4.10.post1
 securesystemslib[crypto,pynacl]==0.28.0
 semver==3.0.4
-service-identity[idna]==24.2.0
-simplejson==3.20.2
+service-identity[idna]==26.1.0
+simplejson==4.1.1
 supervisor==4.3.0
 tuf==4.0.0
 urllib3==2.7.0
 vertica-python==1.4.0
-wrapt==2.1.2
+wrapt==2.2.2

amazon_msk/changelog.d/24321.added

@@ -0,0 +1 @@
+Update dependencies

amazon_msk/pyproject.toml

@@ -36,7 +36,7 @@ license = "BSD-3-Clause"
 
 [project.optional-dependencies]
 deps = [
-    "boto3==1.42.72",
+    "boto3==1.43.40",
 ]
 
 [project.urls]

aspdotnet/changelog.d/24321.added

@@ -0,0 +1 @@
+Update dependencies

aspdotnet/pyproject.toml

@@ -36,7 +36,7 @@ dynamic = [
 
 [project.optional-dependencies]
 deps = [
-    "pywin32==311; sys_platform == 'win32'",
+    "pywin32==312; sys_platform == 'win32'",
 ]
 
 [project.urls]

clickhouse/changelog.d/24266.fixed

@@ -0,0 +1 @@
+Collect standard system table metrics per node via clusterAllReplicas in single endpoint mode so per-node counters no longer produce phantom failures (e.g. clickhouse.query.failed.count).

clickhouse/changelog.d/24321.added

@@ -0,0 +1 @@
+Update dependencies

clickhouse/datadog_checks/clickhouse/clickhouse.py

@@ -23,7 +23,7 @@
 from .statement_samples import ClickhouseStatementSamples
 from .statements import ClickhouseStatementMetrics
 from .table_metrics import ClickhouseTableMetrics
-from .utils import ErrorSanitizer
+from .utils import ErrorSanitizer, cluster_aware_query
 
 try:
     import datadog_agent
@@ -279,14 +279,19 @@ def check(self, _):
 
     def get_queries(self) -> list[dict]:
         query_list = []
+        single = self._config.single_endpoint_mode
+
+        def pick(query: dict) -> dict:
+            """In single endpoint mode, read all replicas and tag each row per node."""
+            return cluster_aware_query(query) if single else query
 
         if self._config.use_legacy_queries:
             query_list.extend(
                 [
-                    queries.SystemMetrics,
-                    queries.SystemEventsToDeprecate,
-                    queries.SystemEvents,
-                    queries.SystemAsynchronousMetrics,
+                    pick(queries.SystemMetrics),
+                    pick(queries.SystemEventsToDeprecate),
+                    pick(queries.SystemEvents),
+                    pick(queries.SystemAsynchronousMetrics),
                     queries.SystemParts,
                     queries.SystemReplicas,
                     queries.SystemDictionaries,
@@ -296,13 +301,13 @@ def get_queries(self) -> list[dict]:
         if self._config.use_advanced_queries:
             query_list.extend(
                 [
-                    advanced_queries.SystemMetrics,
-                    advanced_queries.SystemEvents,
-                    advanced_queries.SystemAsynchronousMetrics,
+                    pick(advanced_queries.SystemMetrics),
+                    pick(advanced_queries.SystemEvents),
+                    pick(advanced_queries.SystemAsynchronousMetrics),
                 ]
             )
             if self.version_ge('21.3'):
-                query_list.append(advanced_queries.SystemErrors)
+                query_list.append(pick(advanced_queries.SystemErrors))
 
         return query_list
 

clickhouse/datadog_checks/clickhouse/utils.py

@@ -27,5 +27,27 @@ def compact_query(query):
     return re.sub(r'\n\s+', ' ', query.strip())
 
 
+# Tag added to per-node metrics when collecting from all replicas in single endpoint mode.
+CLUSTER_NODE_TAG = 'clickhouse_node'
+
+
+def cluster_aware_query(base: dict) -> dict:
+    """Build a cluster-aware variant that reads all replicas and tags each row per node.
+
+    Derives the SELECT list and table from the base query, whose shape is always
+    ``SELECT <cols> FROM system.<table>[ <trailing clause>]``.
+    """
+    select, _, tail = base['query'].partition(' FROM system.')
+    table, sep, trailing = tail.partition(' ')
+    return {
+        'name': base['name'],
+        'query': (
+            f"{select}, hostName() AS {CLUSTER_NODE_TAG} "
+            f"FROM clusterAllReplicas('default', system.{table}){sep}{trailing}"
+        ),
+        'columns': [*base['columns'], {'name': CLUSTER_NODE_TAG, 'type': 'tag'}],
+    }
+
+
 def parse_version(version: str) -> list[int]:
     return [int(v) for v in version.split('.')]

clickhouse/pyproject.toml

@@ -36,7 +36,7 @@ license = "BSD-3-Clause"
 
 [project.optional-dependencies]
 deps = [
-    "clickhouse-connect==0.14.1",
+    "clickhouse-connect==1.4.1",
     "lz4==4.4.5",
 ]
 

clickhouse/tests/test_unit.py

@@ -6,7 +6,8 @@
 from clickhouse_connect.driver.exceptions import Error, OperationalError
 
 from datadog_checks.base import ConfigurationError
-from datadog_checks.clickhouse import ClickhouseCheck, advanced_queries
+from datadog_checks.clickhouse import ClickhouseCheck, advanced_queries, queries
+from datadog_checks.clickhouse.utils import cluster_aware_query
 
 from .utils import ensure_csv_safe, parse_described_metrics, raise_error
 
@@ -381,3 +382,76 @@ def test_database_hostname_ignores_reported_hostname_override(reported_hostname,
         # reported_hostname honors the override when configured, otherwise the resolved host
         assert check.reported_hostname == expected_reported_hostname
         mock_resolve.assert_called_with(BASE_INSTANCE['server'])
+
+
+def test_cluster_aware_query_bulk_match_query():
+    """The cluster-aware variant reads all replicas and tags system.events per node."""
+    variant = cluster_aware_query(advanced_queries.SystemEvents)
+
+    assert variant['query'] == (
+        "SELECT value, event, hostName() AS clickhouse_node FROM clusterAllReplicas('default', system.events)"
+    )
+    assert variant['columns'][-1] == {'name': 'clickhouse_node', 'type': 'tag'}
+    # The base query dict must not be mutated by building the variant.
+    assert advanced_queries.SystemEvents['query'] == 'SELECT value, event FROM system.events'
+    assert all(column['name'] != 'clickhouse_node' for column in advanced_queries.SystemEvents['columns'])
+
+
+def test_cluster_aware_query_preserves_where_clause():
+    """system.errors carries a WHERE clause that must survive in the cluster-aware variant."""
+    variant = cluster_aware_query(advanced_queries.SystemErrors)
+
+    assert variant['query'] == (
+        "SELECT value, name, code, remote, hostName() AS clickhouse_node "
+        "FROM clusterAllReplicas('default', system.errors) WHERE value > 0"
+    )
+    assert variant['columns'][-1] == {'name': 'clickhouse_node', 'type': 'tag'}
+
+
+def test_cluster_aware_query_legacy_query():
+    """The helper builds a cluster-aware variant for a legacy query too."""
+    variant = cluster_aware_query(queries.SystemMetrics)
+
+    assert variant['query'] == (
+        "SELECT value, metric, hostName() AS clickhouse_node FROM clusterAllReplicas('default', system.metrics)"
+    )
+    assert variant['columns'][-1] == {'name': 'clickhouse_node', 'type': 'tag'}
+    assert queries.SystemMetrics['query'] == 'SELECT value, metric FROM system.metrics'
+
+
+@pytest.mark.parametrize('use_advanced_queries', [True, False])
+def test_get_queries_tags_system_tables_per_node_in_single_endpoint_mode(instance, use_advanced_queries):
+    instance = {
+        **instance,
+        'single_endpoint_mode': True,
+        'use_advanced_queries': use_advanced_queries,
+        'use_legacy_queries': not use_advanced_queries,
+    }
+    check = ClickhouseCheck('clickhouse', {}, [instance])
+    check._server_version = '24.8'
+
+    cluster_aware = [q for q in check.get_queries() if 'clusterAllReplicas' in q['query']]
+
+    # system.events, system.metrics, system.asynchronous_metrics (+ system.errors / events-to-deprecate)
+    assert cluster_aware
+    for query in cluster_aware:
+        assert 'hostName() AS clickhouse_node' in query['query']
+        assert query['columns'][-1] == {'name': 'clickhouse_node', 'type': 'tag'}
+
+    # system.parts/replicas/dictionaries use GROUP BY and are intentionally left untouched here.
+    if not use_advanced_queries:
+        assert any(q is queries.SystemParts for q in check.get_queries())
+
+
+@pytest.mark.parametrize('use_advanced_queries', [True, False])
+def test_get_queries_uses_base_queries_for_direct_connection(instance, use_advanced_queries):
+    instance = {
+        **instance,
+        'single_endpoint_mode': False,
+        'use_advanced_queries': use_advanced_queries,
+        'use_legacy_queries': not use_advanced_queries,
+    }
+    check = ClickhouseCheck('clickhouse', {}, [instance])
+    check._server_version = '24.8'
+
+    assert all('clusterAllReplicas' not in q['query'] for q in check.get_queries())

datadog_checks_base/changelog.d/24321.added

@@ -0,0 +1 @@
+Update dependencies

datadog_checks_base/pyproject.toml

@@ -35,27 +35,27 @@ db = [
 ]
 deps = [
     "binary==1.0.2",
-    "cachetools==7.0.5",
+    "cachetools==7.1.4",
     "cryptography==48.0.1",
     "ddtrace==3.19.5",
     "jellyfish==1.2.1",
     "lazy-loader==0.5",
-    "prometheus-client==0.24.1",
-    "protobuf==7.34.0",
-    "pydantic==2.12.5",
+    "prometheus-client==0.25.0",
+    "protobuf==7.35.1",
+    "pydantic==2.13.4",
     "python-dateutil==2.9.0.post0",
-    "pywin32==311; sys_platform == 'win32'",
+    "pywin32==312; sys_platform == 'win32'",
     "pyyaml==6.0.3",
     "requests-toolbelt==1.0.0",
     "requests-unixsocket2==1.0.1",
-    "requests==2.33.0",
-    "simplejson==3.20.2",
+    "requests==2.34.2",
+    "simplejson==4.1.1",
     "urllib3==2.7.0",
-    "wrapt==2.1.2",
+    "wrapt==2.2.2",
 ]
 http = [
     "aws-requests-auth==0.4.3",
-    "botocore==1.42.72",
+    "botocore==1.43.40",
     "oauthlib==3.3.1",
     "pyjwt==2.13.0",
     "pyopenssl==26.2.0",
@@ -65,7 +65,7 @@ http = [
     "requests-oauthlib==2.0.0",
 ]
 json = [
-    "orjson==3.11.7",
+    "orjson==3.11.9",
 ]
 kube = [
     "kubernetes==35.0.0",

datadog_checks_downloader/changelog.d/24321.added

@@ -0,0 +1 @@
+Update dependencies

datadog_checks_downloader/pyproject.toml

@@ -34,7 +34,7 @@ license = "BSD-3-Clause"
 [project.optional-dependencies]
 deps = [
     "in-toto==2.0.0",
-    "packaging==26.0",
+    "packaging==26.2",
     "securesystemslib[crypto,pynacl]==0.28.0",
     "tuf==4.0.0",
 ]

do_query_actions/changelog.d/24321.added

@@ -0,0 +1 @@
+Update dependencies

do_query_actions/pyproject.toml

@@ -38,8 +38,8 @@ dynamic = [
 [project.optional-dependencies]
 deps = [
     "azure-identity==1.25.3",
-    "boto3==1.42.72",
-    "psycopg[c,pool]==3.3.3",
+    "boto3==1.43.40",
+    "psycopg[c,pool]==3.3.4",
 ]
 
 [project.urls]

dotnetclr/changelog.d/24321.added

@@ -0,0 +1 @@
+Update dependencies

dotnetclr/pyproject.toml

@@ -36,7 +36,7 @@ dynamic = [
 
 [project.optional-dependencies]
 deps = [
-    "pywin32==311; sys_platform == 'win32'",
+    "pywin32==312; sys_platform == 'win32'",
 ]
 
 [project.urls]