[DBMON-6490] Fix ClickHouse TLS setting in the connection pool manager (#23450)

sangeetashivaji · claude · web-flow · commit f110c25421b1 · 2026-04-24T19:01:18.000Z
* Fix ClickHouse TLS setting in the connection pool manager

* Fix CI

* Update change log

* Fixing TLS files

* Fix TLS container crash: remove init.sql that requires missing macros

The init.sql uses ReplicatedMergeTree with {layer}/{shard}/{replica}
macros not present in the single-node TLS container, causing the
entrypoint to exit with code 139 and the container to never start.
The TLS integration tests only verify connection behaviour (verify=True/False)
and do not need the test tables created by init.sql.

Co-Authored-By: Claude Sonnet 4.6 &lt;noreply@anthropic.com&gt;

* Skip E2E test in TLS flavor

The E2E test connects to the non-TLS HTTP port (8128) which does not
exist in the TLS flavor environment. TLS-specific behaviour is already
covered by the integration tests in test_tls.py.

Co-Authored-By: Claude Sonnet 4.6 &lt;noreply@anthropic.com&gt;

* Adding a few more tests

* Supporting TLS for v22.7+ and folding TLS containers to main docker-compose

* Update

* Cleaning up TLS compose profiles

---------

Co-authored-by: Claude Sonnet 4.6 &lt;noreply@anthropic.com&gt;
diff --git a/clickhouse/changelog.d/23450.fixed b/clickhouse/changelog.d/23450.fixed
@@ -0,0 +1 @@
+Fix SSL certificate verification being ignored when `verify: false` is set. The shared connection pool was created without TLS settings, causing `verify=False` to be silently ignored when clickhouse-connect skips its own TLS pool creation for a pre-supplied pool manager.
diff --git a/clickhouse/datadog_checks/clickhouse/clickhouse.py b/clickhouse/datadog_checks/clickhouse/clickhouse.py
@@ -77,10 +77,16 @@ def __init__(self, name, init_config, instances):
         # We'll connect on the first check run
         self._client = None
 
-        # Shared HTTP connection pool for all ClickHouse clients (main + DBM jobs)
-        # This reduces connection overhead while maintaining client isolation
-        # See: https://clickhouse.com/docs/integrations/language-clients/python/advanced-usage#customizing-the-http-connection-pool
-        self._pool_manager = httputil.get_pool_manager(maxsize=8, num_pools=4)
+        # Shared HTTP connection pool for all ClickHouse clients (main + DBM jobs).
+        # TLS settings must be baked in here: when pool_mgr is provided to get_client(),
+        # clickhouse-connect assigns it immediately and skips its own TLS pool creation,
+        # so verify=False would be silently ignored if the pool was created without it.
+        self._pool_manager = httputil.get_pool_manager(
+            maxsize=8,
+            num_pools=4,
+            verify=self._config.verify,
+            ca_cert=self._config.tls_ca_cert,
+        )
 
         self._query_manager = QueryManager(
             self,
diff --git a/clickhouse/tests/common.py b/clickhouse/tests/common.py
@@ -6,11 +6,13 @@
 from datadog_checks.dev import get_docker_hostname, get_here
 
 HERE = get_here()
-COMPOSE_FILE = os.path.join(HERE, 'docker', 'docker-compose.yaml')
+COMPOSE_FILE_PATH = os.path.join(HERE, 'docker', 'docker-compose.yaml')
+SERVER_CERT_PATH = os.path.join(HERE, 'docker', 'certs', 'server.crt')
 
 HOST = get_docker_hostname()
 HTTP_START_PORT = 8128
 TCP_START_PORT = 9001
+HTTPS_PORT = 8134
 CLICKHOUSE_VERSION = os.environ['CLICKHOUSE_VERSION']
 
 CONFIG = {
@@ -20,3 +22,12 @@
     'password': 'Datadog123!',
     'tags': ['foo:bar'],
 }
+
+TLS_CONFIG = {
+    'server': HOST,
+    'port': HTTPS_PORT,
+    'username': 'datadog',
+    'password': 'Datadog123!',
+    'tls_verify': True,
+    'tags': ['foo:bar'],
+}
diff --git a/clickhouse/tests/conftest.py b/clickhouse/tests/conftest.py
@@ -35,7 +35,19 @@ def dd_environment():
         )
     )
 
-    with docker_run(common.COMPOSE_FILE, conditions=conditions, sleep=10, attempts=2):
+    conditions.append(
+        WaitFor(
+            ping_clickhouse(
+                common.TLS_CONFIG['server'],
+                common.TLS_CONFIG['port'],
+                common.TLS_CONFIG['username'],
+                common.TLS_CONFIG['password'],
+                secure=True,
+            )
+        )
+    )
+
+    with docker_run(common.COMPOSE_FILE_PATH, conditions=conditions, sleep=10, attempts=2):
         yield common.CONFIG
 
 
@@ -44,13 +56,20 @@ def instance():
     return deepcopy(common.CONFIG)
 
 
-def ping_clickhouse(host, port, username, password):
+@pytest.fixture
+def tls_instance():
+    return deepcopy(common.TLS_CONFIG)
+
+
+def ping_clickhouse(host, port, username, password, secure=False):
     def _ping_clickhouse():
         client = clickhouse_connect.get_client(
             host=host,
             port=port,
             username=username,
             password=password,
+            secure=secure,
+            verify=False,
         )
         return client.ping()
 
diff --git a/clickhouse/tests/docker/certs/dhparam.pem b/clickhouse/tests/docker/certs/dhparam.pem
@@ -0,0 +1,8 @@
+-----BEGIN DH PARAMETERS-----
+MIIBDAKCAQEA6NjHG4vP50LpyuzuyREzF6dpBzQ6rieSCdT5zU4Ku2Cb5hGzsTfp
+q9TLROgeua2WgRE949/ObBOdqRgc+uU4xSI/h7BRvyakvxx482iWKIefQB3kf+gZ
+wlQo0EZRnjARDMVVapgUxsiMmQHaj+w2trrsrn14hvcCjR9mi9j/VGXTwt36RzUS
+CN9jB8wEankSA2FGlicARjCSPz+X839as7Q8l9NgxM2aFY0ddSkoBCQ7ruxGOB+6
+qBd3YaWCO5CNy1ITq7ZjcqXmlLAbLD70xoIknhahDoDHetsGfuDG18I1tS1OzsS7
+3p2C+FQwPJ0Fl0v37+KweBNHQiR60i5UfwIBAgICAOE=
+-----END DH PARAMETERS-----
diff --git a/clickhouse/tests/docker/certs/server.crt b/clickhouse/tests/docker/certs/server.crt
@@ -0,0 +1,20 @@
+-----BEGIN CERTIFICATE-----
+MIIDUzCCAjugAwIBAgIUPQgoFbWsXK5ce7Eq11vx96y87WIwDQYJKoZIhvcNAQEL
+BQAwKzESMBAGA1UEAwwJbG9jYWxob3N0MRUwEwYDVQQKDAxEYXRhZG9nIFRlc3Qw
+HhcNMjYwNDIzMjAxMTE4WhcNMzYwNDIwMjAxMTE4WjArMRIwEAYDVQQDDAlsb2Nh
+bGhvc3QxFTATBgNVBAoMDERhdGFkb2cgVGVzdDCCASIwDQYJKoZIhvcNAQEBBQAD
+ggEPADCCAQoCggEBAIm3vNOye0w3gZj+NJ4spjk1SmL2wxqfFow/G7jKUxeIoF+X
+2UbfDgrkLBd64mkQqPXlfvzYPuLu+0R+4cSkoM/Pfkf55lm/ekTKPrv2B9gR28q9
+J/HNLsNufGfxs5XRD9YHyF0xms5MoZrSaMmpA6LsYemRet8tCcI1PgVn2ugeDVQo
+wZ/Eko6dToHgrxT59OPwenyQrKmkW/xVLj3yN7+XYZY93fvlpyb9RzOLV6IiErrw
+RgyMrwBZyWltY4bkfluNAmAi0dyoAhSt4NQgaDEYOtXdcdpd0Q6bcJIsVtnx/C+f
+yXlbSxtMHY1cjQmNv4ShralNuxU/Ap7YNyletGECAwEAAaNvMG0wHQYDVR0OBBYE
+FEjXZiVh3FKPFAwPYHD1IYYEjAJxMB8GA1UdIwQYMBaAFEjXZiVh3FKPFAwPYHD1
+IYYEjAJxMA8GA1UdEwEB/wQFMAMBAf8wGgYDVR0RBBMwEYIJbG9jYWxob3N0hwR/
+AAABMA0GCSqGSIb3DQEBCwUAA4IBAQAQJ1tugGV8FlFriqhxW1lUCjo+3J74jIKl
+xbI762sPZ3sZYufzxq9eE3RtZlbnrEAyfxlW2+EqDTxSuzhdezcaWSWeryLjriY0
+pHTFZ2vQ9y6vS+xKa54qlict6YUZjeiskN8XM0IP5TmpO+Kd6YYgh5oz5+mVZ0Jp
+zijXZt4zg27SqzX5SBpMBKi8B1yR+eyGgO4Jen3DEc0HXa+yB94wa5KSa1KQpR6B
+2bOQ0mBa9NkIF859YNNBdw+pCTAzmai3X5ptzrVBy8PEk+cH9La+UYl1guq43cap
+uLyZlwBpFBtx2IBNBIc91b0jR3rhI9DsnNzANtrDKFHE/NWj1Pzr
+-----END CERTIFICATE-----
diff --git a/clickhouse/tests/docker/certs/server.key b/clickhouse/tests/docker/certs/server.key
@@ -0,0 +1,28 @@
+-----BEGIN PRIVATE KEY-----
+MIIEvwIBADANBgkqhkiG9w0BAQEFAASCBKkwggSlAgEAAoIBAQCJt7zTsntMN4GY
+/jSeLKY5NUpi9sManxaMPxu4ylMXiKBfl9lG3w4K5CwXeuJpEKj15X782D7i7vtE
+fuHEpKDPz35H+eZZv3pEyj679gfYEdvKvSfxzS7Dbnxn8bOV0Q/WB8hdMZrOTKGa
+0mjJqQOi7GHpkXrfLQnCNT4FZ9roHg1UKMGfxJKOnU6B4K8U+fTj8Hp8kKyppFv8
+VS498je/l2GWPd375acm/Uczi1eiIhK68EYMjK8AWclpbWOG5H5bjQJgItHcqAIU
+reDUIGgxGDrV3XHaXdEOm3CSLFbZ8fwvn8l5W0sbTB2NXI0Jjb+Eoa2pTbsVPwKe
+2DcpXrRhAgMBAAECggEAAVbDOyY3Kfy4g1/WBZm2AMFbvjIx5i1yZ4Kh6TPN3dRk
+9aldHVuKYdkAFDC1/wfTsoBrG3StT737Y0bhK+nRrq2KZJmf6sLVRevDszNk77Qu
+PqAkl8nEqCJuV+FQodHvutc2GtC/KLbtfT42Reu+APiWFw80SmAzW5msCk5J/knw
+KCH0gWyKESeuSsradac3YEY6o9zJ7gDkIw1ScOFHStBA1MlJ0o6IKwyXvy9JQ9Mb
+mPcbz78CO42BLuHs+07Zl697NghEBhrM0C0dZWea9NSEnDkru6s1SvtCA6y5eQa1
+mzA0ZqC7TefDWY2NbWTlrvjaXbP9lCO3Jz18uO+G0QKBgQC+vHhPxBryPyLXvrPd
+cvLIUrUpdmdxbTH9/MK0t3zryRVP9d8oBRGbgXg6ob/3WjC+StQ4KsJfiD3HufvJ
+lQBPYMid+wfD2PmSBDL8qItwvngd33isbQmam5N8rFJ/BdTABjbzoknOExAiFuRA
+cg+5u2GgnjP6TIdSaLlisb/E0QKBgQC41x2eVrCqgL9n3K9aELAzqFEnDSaHDOM2
+GWINho0DUvb9RoDy93G8EkHodBHHQ2YA1UFsmFJKtBXdcJTd9TvG0tcyabSSq1rw
+vo+GGbEoOtfngkqc114aE+ORbaHlY26HB5SF+RYLtb8i/yPT25wGSmiClQEfA8qz
+U/YnklMakQKBgQCpOok4pJMP6gqdBvuI+KsGJ7PK1N4U2kBnzHJoawBwJxo8trbH
+SDAGHAozYz3yVdg3jibaqr2Ii1XdNlt60DPVOdblrEuIbaOI9SbT2yrU3nmub48e
+1SOXRH/INxB+2QqZUSpCYfQ/3dQLmunrcAgZz3JgUt1lL69E49vQAEtYcQKBgQCD
+zjm7qBuwacCTgoW0FsRfAb5kt3bqxKDc+fq2pGS3oxinAPiAyTxcHTTMWsA2tHPv
+UiKmGnRL+uDAmEA/ilkG7WAw/bmbAQAHPgvAV1/BzTwaRQmJj/X1aItVL7VLirzN
+IvWhFz7GOOLEROW6dBZtUe/2deRnxl9rwwp7q1cBUQKBgQCtXlEAQBBSJDnLggRo
+aoCCvnqyVEOj6iG9Cr+6QKVlczf68Qvn23IAFRIveI+fSIw3W/UqEVEnAPZTKNji
+ZuqC8CoOWWKq8reTFuGfPVaJ1SumQYO/erp4/7qrt7dTOirMZdFk39naOp9ldOEG
+Jnx53Ea7KGbeKmcuZumGwUYWfw==
+-----END PRIVATE KEY-----
diff --git a/clickhouse/tests/docker/docker-compose.yaml b/clickhouse/tests/docker/docker-compose.yaml
@@ -15,6 +15,7 @@ services:
         hard: 262144
     ports:
       - "8128:8123"
+      - "8134:8443"
       - "9001:9000"
     volumes:
       - ./override_configs/zookeeper.xml:/etc/clickhouse-server/config.d/zookeeper.xml
@@ -26,6 +27,10 @@ services:
       - ./init.sql:/docker-entrypoint-initdb.d/init.sql
       - ./test_dictionary.xml:/etc/clickhouse-server/test_dictionary.xml
       - ./test.csv:/opt/dictionaries/test.csv
+      - ./tls_config.xml:/etc/clickhouse-server/config.d/tls.xml
+      - ./certs/server.crt:/etc/clickhouse-server/ssl/server.crt
+      - ./certs/server.key:/etc/clickhouse-server/ssl/server.key
+      - ./certs/dhparam.pem:/etc/clickhouse-server/dhparam.pem
     depends_on:
       clickhouse-zookeeper:
         condition: service_healthy
diff --git a/clickhouse/tests/docker/tls_config.xml b/clickhouse/tests/docker/tls_config.xml
@@ -0,0 +1,12 @@
+<yandex>
+    <https_port>8443</https_port>
+    <openSSL>
+        <server>
+            <privateKeyFile>/etc/clickhouse-server/ssl/server.key</privateKeyFile>
+            <certificateFile>/etc/clickhouse-server/ssl/server.crt</certificateFile>
+            <dhParamsFile>/etc/clickhouse-server/dhparam.pem</dhParamsFile>
+            <verificationMode>none</verificationMode>
+            <cacheSessions>true</cacheSessions>
+        </server>
+    </openSSL>
+</yandex>
diff --git a/clickhouse/tests/test_tls.py b/clickhouse/tests/test_tls.py
@@ -0,0 +1,50 @@
+# (C) Datadog, Inc. 2026-present
+# All rights reserved
+# Licensed under a 3-clause BSD style license (see LICENSE)
+import pytest
+
+from datadog_checks.clickhouse import ClickhouseCheck
+
+from . import common
+
+pytestmark = [pytest.mark.integration, pytest.mark.usefixtures('dd_environment')]
+
+
+def test_connect_with_verify_false(aggregator, tls_instance, dd_run_check):
+    """Regression: verify: false must allow connection to a self-signed cert.
+
+    Integration-level proof that the pool manager fix works end-to-end: the shared
+    pool must be created with verify=False so clickhouse-connect doesn't override it
+    with a cert_reqs=CERT_REQUIRED pool when pool_mgr is pre-supplied.
+    """
+    tls_instance['verify'] = False
+    check = ClickhouseCheck('clickhouse', {}, [tls_instance])
+    dd_run_check(check)
+    aggregator.assert_service_check('clickhouse.can_connect', status=ClickhouseCheck.OK)
+
+
+def test_connect_ssl_verify_true_fails(tls_instance, dd_run_check):
+    """Sanity: verify=True (default) must reject a self-signed cert.
+
+    Confirms TLS is actually active — without this passing, test_connect_with_verify_false
+    would be vacuous (the server might just be accepting plain HTTP).
+    """
+    tls_instance['verify'] = True
+    check = ClickhouseCheck('clickhouse', {}, [tls_instance])
+    with pytest.raises(Exception):
+        dd_run_check(check)
+
+
+def test_connect_verify_true_with_ca_cert(aggregator, tls_instance, dd_run_check):
+    """Production path: verify=True + tls_ca_cert pointing at a trusted CA must succeed.
+
+    Most TLS-using DBM customers configure the integration this way — TLS on, cert
+    validation on, with a custom CA bundle that trusts the server's cert. Pins that
+    path against the self-signed server so the ca_cert plumbing in the shared pool
+    manager doesn't silently regress.
+    """
+    tls_instance['verify'] = True
+    tls_instance['tls_ca_cert'] = common.SERVER_CERT_PATH
+    check = ClickhouseCheck('clickhouse', {}, [tls_instance])
+    dd_run_check(check)
+    aggregator.assert_service_check('clickhouse.can_connect', status=ClickhouseCheck.OK)
diff --git a/clickhouse/tests/test_unit.py b/clickhouse/tests/test_unit.py
@@ -40,6 +40,32 @@ def test_config(instance):
         )
 
 
+def test_config_verify_false(instance):
+    """Regression: verify: false must be forwarded to the shared pool manager.
+
+    When pool_mgr is provided to clickhouse-connect, it skips creating its own
+    TLS-aware pool, so TLS settings must be baked into the pool at creation time.
+    """
+    instance = {**instance, 'verify': False}
+    with mock.patch('clickhouse_connect.driver.httputil.get_pool_manager') as mock_pool:
+        mock_pool.return_value = mock.MagicMock()
+        ClickhouseCheck('clickhouse', {}, [instance])
+        mock_pool.assert_called_once_with(maxsize=8, num_pools=4, verify=False, ca_cert=None)
+
+
+def test_config_tls_ca_cert_forwarded_to_pool_manager(instance):
+    """Regression: tls_ca_cert must be forwarded to the shared pool manager.
+
+    Same failure mode as verify=False: if ca_cert isn't baked into the pre-supplied
+    pool manager, clickhouse-connect's get_client can't apply it later.
+    """
+    instance = {**instance, 'tls_ca_cert': '/path/to/ca.pem'}
+    with mock.patch('clickhouse_connect.driver.httputil.get_pool_manager') as mock_pool:
+        mock_pool.return_value = mock.MagicMock()
+        ClickhouseCheck('clickhouse', {}, [instance])
+        mock_pool.assert_called_once_with(maxsize=8, num_pools=4, verify=True, ca_cert='/path/to/ca.pem')
+
+
 def test_error_query(instance, dd_run_check):
     check = ClickhouseCheck('clickhouse', {}, [instance])
     check.log = mock.MagicMock()

Original file line number	Diff line number	Diff line change
`@@ -0,0 +1 @@`
	`1`	+Fix SSL certificate verification being ignored when `verify: false` is set. The shared connection pool was created without TLS settings, causing `verify=False` to be silently ignored when clickhouse-connect skips its own TLS pool creation for a pre-supplied pool manager.