feat(qwp): support for PEM CA files (#75)

tls_roots now accepts a PEM certificate file:

QuestDB.connect("wss::addr=db.example.com:9000;tls_roots=/etc/ssl/ca.pem;")

The file may contain one certificate or a bundle, and no longer has to be imported into a password-protected JKS with keytool first.

Existing JKS/PKCS#12 trust stores keep working as before by supplying tls_roots_password, and PKCS#12 now also loads on Java 8.

Author: jerrinot

README.md

@@ -364,6 +364,19 @@ try (QuestDB db = QuestDB.connect("wss::addr=localhost:9000;username=admin;passw
 }
 ```
 
+**Custom PEM certificate authority:**
+
+```java
+try (QuestDB db = QuestDB.connect(
+        "wss::addr=localhost:9000;tls_roots=/path/to/ca.pem;")) {
+    // ... use db ...
+}
+```
+
+`tls_roots` accepts a PEM certificate or bundle directly. For an existing JKS
+or PKCS#12 trust store, also set `tls_roots_password`; the password switches the
+file interpretation from PEM to a Java trust store.
+
 **Disable certificate validation (not for production):**
 
 ```java
@@ -412,8 +425,8 @@ schema::key1=value1;key2=value2;
 | `password` / `pass`  |              | Basic-auth password                                                 |
 | `token`              |              | Bearer token (sent as an `Authorization` header on the WS upgrade)  |
 | `tls_verify`         | `on`         | TLS certificate validation (`on` or `unsafe_off`)                   |
-| `tls_roots`          |              | Path to a custom truststore                                         |
-| `tls_roots_password` |              | Truststore password                                                 |
+| `tls_roots`          |              | Path to a PEM certificate/bundle, or a JKS/PKCS#12 trust store       |
+| `tls_roots_password` |              | Optional JKS/PKCS#12 password; omit when `tls_roots` is PEM          |
 | `connect_timeout`    | _(OS)_       | TCP connect + TLS handshake timeout, in milliseconds                |
 | `auth_timeout_ms`    | `15000`      | Authentication/upgrade request timeout, in milliseconds             |
 

core/src/main/java/io/questdb/client/ClientTlsConfiguration.java

@@ -33,6 +33,9 @@ public class ClientTlsConfiguration {
     private final String trustStorePath;
 
     public ClientTlsConfiguration(String trustStorePath, char[] trustStorePassword, int tlsValidationMode) {
+        if (trustStorePath != null && tlsValidationMode == TLS_VALIDATION_MODE_NONE) {
+            throw new IllegalArgumentException("custom trust store cannot be combined with disabled TLS validation");
+        }
         this.trustStorePath = trustStorePath;
         this.trustStorePassword = trustStorePassword;
         this.tlsValidationMode = tlsValidationMode;

core/src/main/java/io/questdb/client/Sender.java

@@ -934,6 +934,7 @@ enum Transport {
      */
     final class LineSenderBuilder {
         private static final int AUTO_FLUSH_DISABLED = 0;
+        private static final String TLS_ROOTS_INSECURE_CONFIG_ERROR = "tls_roots cannot be combined with tls_verify=unsafe_off; remove tls_verify to use custom roots, or remove tls_roots to disable certificate validation";
         // close() drain timeout. Default applied at build() time. 0 or -1
         // means "fast close" (skip the drain entirely); any positive value
         // bounds the wait for ackedFsn to catch up to publishedFsn. Uses
@@ -1415,7 +1416,7 @@ public Sender build() {
                 }
                 ClientTlsConfiguration tlsConfig = null;
                 if (tlsEnabled) {
-                    assert (trustStorePath == null) == (trustStorePassword == null); //either both null or both non-null
+                    assert trustStorePassword == null || trustStorePath != null;
                     tlsConfig = new ClientTlsConfiguration(trustStorePath, trustStorePassword, tlsValidationMode == TlsValidationMode.DEFAULT ? ClientTlsConfiguration.TLS_VALIDATION_MODE_FULL : ClientTlsConfiguration.TLS_VALIDATION_MODE_NONE);
                 }
                 return AbstractLineHttpSender.createLineSender(hosts, ports, httpPath, httpClientConfiguration, tlsConfig, actualAutoFlushRows, httpToken,
@@ -1437,7 +1438,7 @@ public Sender build() {
 
                 ClientTlsConfiguration wsTlsConfig = null;
                 if (tlsEnabled) {
-                    assert (trustStorePath == null) == (trustStorePassword == null);
+                    assert trustStorePassword == null || trustStorePath != null;
                     wsTlsConfig = new ClientTlsConfiguration(
                             trustStorePath,
                             trustStorePassword,
@@ -3573,13 +3574,12 @@ private LineSenderBuilder fromConfig(CharSequence configurationString) {
             if (hosts.size() == 0) {
                 throw new LineSenderException("addr is missing");
             }
-            if (trustStorePath != null) {
-                if (trustStorePassword == null) {
-                    throw new LineSenderException("tls_roots was configured, but tls_roots_password is missing");
-                }
-            } else if (trustStorePassword != null) {
+            if (trustStorePath == null && trustStorePassword != null) {
                 throw new LineSenderException("tls_roots_password was configured, but tls_roots is missing");
             }
+            if (trustStorePath != null && tlsValidationMode == TlsValidationMode.INSECURE) {
+                throw new LineSenderException(TLS_ROOTS_INSECURE_CONFIG_ERROR);
+            }
             if (protocol == PROTOCOL_HTTP || protocol == PROTOCOL_WEBSOCKET) {
                 if (user != null) {
                     httpUsernamePassword(user, password);
@@ -3836,8 +3836,11 @@ static void validateWsConfig(ConfigView view, boolean tls) {
             if (!tls && (tlsVerify != null || tlsRoots != null || tlsRootsPassword != null)) {
                 throw new IllegalArgumentException("tls_verify/tls_roots/tls_roots_password require the wss:: schema");
             }
-            if ((tlsRoots == null) != (tlsRootsPassword == null)) {
-                throw new IllegalArgumentException("tls_roots and tls_roots_password must be provided together");
+            if (tlsRoots == null && tlsRootsPassword != null) {
+                throw new IllegalArgumentException("tls_roots_password requires tls_roots");
+            }
+            if (tlsRoots != null && "unsafe_off".equals(tlsVerify)) {
+                throw new IllegalArgumentException(TLS_ROOTS_INSECURE_CONFIG_ERROR);
             }
         }
 
@@ -3975,6 +3978,9 @@ private void validateParameters() {
             if (!tlsEnabled && tlsValidationMode != TlsValidationMode.DEFAULT) {
                 throw new LineSenderException("TLS validation disabled, but TLS was not enabled");
             }
+            if (trustStorePath != null && tlsValidationMode == TlsValidationMode.INSECURE) {
+                throw new LineSenderException("custom trust store cannot be combined with disabled TLS validation");
+            }
             if (keyId != null && bufferCapacity < MIN_BUFFER_SIZE) {
                 throw new LineSenderException("Requested buffer too small ")
                         .put("[minimalCapacity=").put(MIN_BUFFER_SIZE)
@@ -4133,26 +4139,51 @@ private void websocket() {
 
         public class AdvancedTlsSettings {
             /**
-             * Configure a custom truststore. This is only needed when using {@link #enableTls()} when your default
-             * truststore does not contain certificate chain used by a server. Most users should not need it.
+             * Configure a PEM file containing one or more custom root certificates.
+             * This is only needed when using {@link #enableTls()} and the default
+             * trust store does not contain the certificate chain used by a server.
+             * Most users should not need it.
              * <br>
-             * The path can be either a path on a local filesystem. Or you can prefix it with "classpath:" to instruct
-             * the Sender to load a trust store from a classpath.
+             * The path can be on the local filesystem, or it can use the
+             * {@code classpath:} prefix.
+             *
+             * @param pemRootsPath a path to a PEM certificate file or bundle
+             * @return an instance of LineSenderBuilder for further configuration
+             */
+            public LineSenderBuilder customTrustStore(String pemRootsPath) {
+                return setCustomTrustStore(pemRootsPath, null);
+            }
+
+            /**
+             * Configure a password-protected JKS or PKCS#12 trust store. This is
+             * only needed when using {@link #enableTls()} and the default trust
+             * store does not contain the certificate chain used by a server.
+             * Most users should not need it.
+             * <br>
+             * The path can be on the local filesystem, or it can use the
+             * {@code classpath:} prefix.
              *
              * @param trustStorePath     a path to a trust store.
-             * @param trustStorePassword a password to for the truststore
+             * @param trustStorePassword the trust store password
              * @return an instance of LineSenderBuilder for further configuration
              */
             public LineSenderBuilder customTrustStore(String trustStorePath, char[] trustStorePassword) {
+                if (trustStorePassword == null) {
+                    throw new LineSenderException("trust store password cannot be null");
+                }
+                return setCustomTrustStore(trustStorePath, trustStorePassword);
+            }
+
+            private LineSenderBuilder setCustomTrustStore(String trustStorePath, char[] trustStorePassword) {
                 if (LineSenderBuilder.this.trustStorePath != null) {
                     throw new LineSenderException("custom trust store was already configured ")
                             .put("[path=").put(LineSenderBuilder.this.trustStorePath).put("]");
                 }
                 if (Chars.isBlank(trustStorePath)) {
                     throw new LineSenderException("trust store path cannot be empty nor null");
                 }
-                if (trustStorePassword == null) {
-                    throw new LineSenderException("trust store password cannot be null");
+                if (tlsValidationMode == TlsValidationMode.INSECURE) {
+                    throw new LineSenderException("custom trust store cannot be configured when TLS validation is disabled");
                 }
 
                 LineSenderBuilder.this.trustStorePath = trustStorePath;
@@ -4165,12 +4196,16 @@ public LineSenderBuilder customTrustStore(String trustStorePath, char[] trustSto
              * This is suitable when testing self-signed certificate. It's inherently insecure and should
              * never be used in a production.
              * <br>
-             * If you cannot use trusted certificate then you should prefer {@link  #customTrustStore(String, char[])}
-             * over disabling validation.
+             * If you cannot use a certificate in the default trust store then
+             * you should prefer {@link #customTrustStore(String)} or
+             * {@link #customTrustStore(String, char[])} over disabling validation.
              *
              * @return an instance of LineSenderBuilder for further configuration
              */
             public LineSenderBuilder disableCertificateValidation() {
+                if (LineSenderBuilder.this.trustStorePath != null) {
+                    throw new LineSenderException("TLS validation cannot be disabled when a custom trust store is configured");
+                }
                 LineSenderBuilder.this.tlsValidationMode = TlsValidationMode.INSECURE;
                 return LineSenderBuilder.this;
             }

core/src/main/java/io/questdb/client/cutlass/line/tcp/DelegatingTlsChannel.java

@@ -27,46 +27,24 @@
 import io.questdb.client.Sender;
 import io.questdb.client.cutlass.line.LineChannel;
 import io.questdb.client.cutlass.line.LineSenderException;
+import io.questdb.client.network.TlsTrustStore;
 import io.questdb.client.std.MemoryTag;
 import io.questdb.client.std.Misc;
 import io.questdb.client.std.Unsafe;
 import io.questdb.client.std.Vect;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
-import javax.net.ssl.SSLContext;
 import javax.net.ssl.SSLEngine;
 import javax.net.ssl.SSLEngineResult;
 import javax.net.ssl.SSLException;
-import javax.net.ssl.SSLParameters;
-import javax.net.ssl.TrustManager;
-import javax.net.ssl.TrustManagerFactory;
-import javax.net.ssl.X509TrustManager;
-import java.io.FileInputStream;
-import java.io.FileNotFoundException;
-import java.io.InputStream;
 import java.lang.reflect.Field;
 import java.nio.Buffer;
 import java.nio.ByteBuffer;
-import java.security.KeyStore;
-import java.security.SecureRandom;
-import java.security.cert.X509Certificate;
 
 public final class DelegatingTlsChannel implements LineChannel {
     private static final long ADDRESS_FIELD_OFFSET;
     private static final int AFTER_HANDSHAKE = 1;
-    private static final TrustManager[] BLIND_TRUST_MANAGERS = new TrustManager[]{new X509TrustManager() {
-        public void checkClientTrusted(X509Certificate[] certs, String t) {
-        }
-
-        public void checkServerTrusted(X509Certificate[] certs, String t) {
-        }
-
-        public X509Certificate[] getAcceptedIssuers() {
-            return null;
-        }
-    }};
-
     private static final long CAPACITY_FIELD_OFFSET;
     private static final int CLOSED = 3;
     private static final int CLOSING = 2;
@@ -202,44 +180,14 @@ private static long allocateMemoryAndResetBuffer(ByteBuffer buffer, int capacity
     }
 
     private static SSLEngine createSslEngine(String trustStorePath, char[] trustStorePassword, Sender.TlsValidationMode validationMode, String peerHost) {
-        assert trustStorePath == null || validationMode == Sender.TlsValidationMode.DEFAULT;
         try {
-            SSLContext sslContext;
-            if (trustStorePath != null) {
-                sslContext = SSLContext.getInstance("TLS");
-                TrustManagerFactory tmf = TrustManagerFactory.getInstance(TrustManagerFactory.getDefaultAlgorithm());
-                KeyStore jks = KeyStore.getInstance("JKS");
-                try (InputStream trustStoreStream = openTruststoreStream(trustStorePath)) {
-                    jks.load(trustStoreStream, trustStorePassword);
-                }
-                tmf.init(jks);
-                TrustManager[] trustManagers = tmf.getTrustManagers();
-                sslContext.init(null, trustManagers, new SecureRandom());
-            } else if (validationMode == Sender.TlsValidationMode.INSECURE) {
-                sslContext = SSLContext.getInstance("TLS");
-                sslContext.init(null, BLIND_TRUST_MANAGERS, new SecureRandom());
-            } else {
-                sslContext = SSLContext.getDefault();
-            }
-
-            // SSLEngine needs to know hostname during TLS handshake to validate a server certificate was issued
-            // for the server we are connecting to. For details see the comment below.
-            // Hostname validation does not use port at all hence we can get away with a dummy value -1
-            SSLEngine sslEngine = sslContext.createSSLEngine(peerHost, -1);
-            if (validationMode != Sender.TlsValidationMode.INSECURE) {
-                SSLParameters sslParameters = sslEngine.getSSLParameters();
-                // The https validation algorithm? That looks confusing! After all we are not using any
-                // https here at so what does it mean?
-                // It's actually simple: It just instructs the SSLEngine to perform the same hostname validation
-                // as it does during HTTPS connections. SSLEngine does not do hostname validation by default. Without
-                // this option SSLEngine would happily accept any certificate as long as it's signed by a trusted CA.
-                // This option will make sure certificates are accepted only if they were issued for the
-                // server we are connecting to.
-                sslParameters.setEndpointIdentificationAlgorithm("https");
-                sslEngine.setSSLParameters(sslParameters);
-            }
-            sslEngine.setUseClientMode(true);
-            return sslEngine;
+            return TlsTrustStore.createSslEngine(
+                    trustStorePath,
+                    trustStorePassword,
+                    validationMode == Sender.TlsValidationMode.INSECURE,
+                    peerHost,
+                    DelegatingTlsChannel.class
+            );
         } catch (Throwable t) {
             if (t instanceof LineSenderException) {
                 throw (LineSenderException) t;
@@ -256,20 +204,6 @@ private static long expandBuffer(ByteBuffer buffer, long oldAddress) {
         return newAddress;
     }
 
-    private static InputStream openTruststoreStream(String trustStorePath) throws FileNotFoundException {
-        InputStream trustStoreStream;
-        if (trustStorePath.startsWith("classpath:")) {
-            String adjustedPath = trustStorePath.substring("classpath:".length());
-            trustStoreStream = DelegatingTlsChannel.class.getResourceAsStream(adjustedPath);
-            if (trustStoreStream == null) {
-                throw new LineSenderException("configured trust store is unavailable ")
-                        .put("[path=").put(trustStorePath).put("]");
-            }
-            return trustStoreStream;
-        }
-        return new FileInputStream(trustStorePath);
-    }
-
     private static void resetBufferToPointer(ByteBuffer buffer, long ptr, int len) {
         assert buffer.isDirect();
         Unsafe.getUnsafe().putLong(buffer, ADDRESS_FIELD_OFFSET, ptr);
@@ -437,4 +371,4 @@ private void writeToUpstreamAndClear() {
         LIMIT_FIELD_OFFSET = Unsafe.getUnsafe().objectFieldOffset(limitField);
         CAPACITY_FIELD_OFFSET = Unsafe.getUnsafe().objectFieldOffset(capacityField);
     }
-}
+}