From f53d29cd01af6d20912891fddb9cf3c693855e2a Mon Sep 17 00:00:00 2001 From: Madhavendra Rathore Date: Tue, 26 Aug 2025 00:29:18 +0530 Subject: [PATCH 01/15] Added optional EnableSQLValidationForIsValid property for isValid() to allow users to enable server-side SQL validation --- README.md | 1 + .../jdbc/api/impl/DatabricksConnection.java | 20 ++++++++++++++++++- .../jdbc/common/DatabricksJdbcConstants.java | 3 ++- .../jdbc/common/DatabricksJdbcUrlParams.java | 6 +++++- .../util/DatabricksDriverPropertyUtil.java | 3 ++- 5 files changed, 29 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index 41f930e294..db37290510 100644 --- a/README.md +++ b/README.md @@ -91,6 +91,7 @@ Optional parameters: - `OAuth2RedirectUrlPort` - Ports for redirect URL (default: 8020) - `EnableOIDCDiscovery` - Enable OIDC discovery (default: 1) - `OAuthDiscoveryURL` - OIDC discovery endpoint (default: /oidc/.well-known/oauth-authorization-server) +- `EnableSQLValidationForIsValid` - Enable SQL query based validation in `isValid()` connection checks (default: 0) ### Logging diff --git a/src/main/java/com/databricks/jdbc/api/impl/DatabricksConnection.java b/src/main/java/com/databricks/jdbc/api/impl/DatabricksConnection.java index 4c1555a20d..03e1e43e9a 100644 --- a/src/main/java/com/databricks/jdbc/api/impl/DatabricksConnection.java +++ b/src/main/java/com/databricks/jdbc/api/impl/DatabricksConnection.java @@ -10,6 +10,7 @@ import com.databricks.jdbc.api.internal.IDatabricksStatementInternal; import com.databricks.jdbc.common.DatabricksClientConfiguratorManager; import com.databricks.jdbc.common.DatabricksJdbcConstants; +import com.databricks.jdbc.common.DatabricksJdbcUrlParams; import com.databricks.jdbc.common.safe.DatabricksDriverFeatureFlagsContextFactory; import com.databricks.jdbc.common.util.DatabricksThreadContextHolder; import com.databricks.jdbc.common.util.UserAgentManager; @@ -414,7 +415,24 @@ public SQLXML createSQLXML() throws SQLException { @Override public boolean isValid(int timeout) throws SQLException { ValidationUtil.checkIfNonNegative(timeout, "timeout"); - return !isClosed(); + if (isClosed()) { + return false; + } + String enableSqlValidation = + session.getConfigValue( + DatabricksJdbcUrlParams.ENABLE_SQL_VALIDATION_FOR_ISVALID.getParamName()); + + if ("1".equals(enableSqlValidation)) { + try (Statement stmt = createStatement()) { + stmt.setQueryTimeout(timeout); + stmt.execute("SELECT 1"); + return true; + } catch (SQLException e) { + LOGGER.debug("SQL validation failed for isValid(): {}", e.getMessage()); + return false; + } + } + return true; } /** diff --git a/src/main/java/com/databricks/jdbc/common/DatabricksJdbcConstants.java b/src/main/java/com/databricks/jdbc/common/DatabricksJdbcConstants.java index bbcacb1277..e339a31311 100644 --- a/src/main/java/com/databricks/jdbc/common/DatabricksJdbcConstants.java +++ b/src/main/java/com/databricks/jdbc/common/DatabricksJdbcConstants.java @@ -90,7 +90,8 @@ public final class DatabricksJdbcConstants { ENABLE_VOLUME_OPERATIONS, ALLOWED_STAGING_INGESTION_PATHS, DatabricksJdbcUrlParams.AUTH_ACCESS_TOKEN.getParamName(), - DatabricksJdbcUrlParams.APPLICATION_NAME.getParamName()); + DatabricksJdbcUrlParams.APPLICATION_NAME.getParamName(), + DatabricksJdbcUrlParams.ENABLE_SQL_VALIDATION_FOR_ISVALID.getParamName()); public static final Map JSON_HTTP_HEADERS = Map.of( "Accept", "application/json", diff --git a/src/main/java/com/databricks/jdbc/common/DatabricksJdbcUrlParams.java b/src/main/java/com/databricks/jdbc/common/DatabricksJdbcUrlParams.java index 433447a162..e94143a479 100644 --- a/src/main/java/com/databricks/jdbc/common/DatabricksJdbcUrlParams.java +++ b/src/main/java/com/databricks/jdbc/common/DatabricksJdbcUrlParams.java @@ -153,7 +153,11 @@ public enum DatabricksJdbcUrlParams { HTTP_CONNECTION_REQUEST_TIMEOUT( "HttpConnectionRequestTimeout", "HTTP connection request timeout in seconds"), CLOUD_FETCH_SPEED_THRESHOLD( - "CloudFetchSpeedThreshold", "Minimum expected download speed in MB/s", "0.1"); + "CloudFetchSpeedThreshold", "Minimum expected download speed in MB/s", "0.1"), + ENABLE_SQL_VALIDATION_FOR_ISVALID( + "EnableSQLValidationForIsValid", + "Enable SQL query execution for connection validation in isValid() method", + "0"); private final String paramName; private final String defaultValue; diff --git a/src/main/java/com/databricks/jdbc/common/util/DatabricksDriverPropertyUtil.java b/src/main/java/com/databricks/jdbc/common/util/DatabricksDriverPropertyUtil.java index 05935c1ee5..fc6d4bf4c7 100644 --- a/src/main/java/com/databricks/jdbc/common/util/DatabricksDriverPropertyUtil.java +++ b/src/main/java/com/databricks/jdbc/common/util/DatabricksDriverPropertyUtil.java @@ -37,7 +37,8 @@ public class DatabricksDriverPropertyUtil { DatabricksJdbcUrlParams.ROWS_FETCHED_PER_BLOCK, DatabricksJdbcUrlParams.DEFAULT_STRING_COLUMN_LENGTH, DatabricksJdbcUrlParams.SOCKET_TIMEOUT, - DatabricksJdbcUrlParams.ENABLE_TOKEN_CACHE); + DatabricksJdbcUrlParams.ENABLE_TOKEN_CACHE, + DatabricksJdbcUrlParams.ENABLE_SQL_VALIDATION_FOR_ISVALID); public static List getMissingProperties(String url, Properties info) throws DatabricksParsingException { From eb683b97a06c78a228c691f55318053242aa6d74 Mon Sep 17 00:00:00 2001 From: Madhavendra Rathore Date: Tue, 26 Aug 2025 00:39:52 +0530 Subject: [PATCH 02/15] Changed property ENABLE_SQL_VALIDATION_FOR_ISVALID name to ENABLE_SQL_VALIDATION_FOR_IS_VALID --- .../java/com/databricks/jdbc/api/impl/DatabricksConnection.java | 2 +- .../com/databricks/jdbc/common/DatabricksJdbcConstants.java | 2 +- .../com/databricks/jdbc/common/DatabricksJdbcUrlParams.java | 2 +- .../jdbc/common/util/DatabricksDriverPropertyUtil.java | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/main/java/com/databricks/jdbc/api/impl/DatabricksConnection.java b/src/main/java/com/databricks/jdbc/api/impl/DatabricksConnection.java index 03e1e43e9a..a79fa33d8e 100644 --- a/src/main/java/com/databricks/jdbc/api/impl/DatabricksConnection.java +++ b/src/main/java/com/databricks/jdbc/api/impl/DatabricksConnection.java @@ -420,7 +420,7 @@ public boolean isValid(int timeout) throws SQLException { } String enableSqlValidation = session.getConfigValue( - DatabricksJdbcUrlParams.ENABLE_SQL_VALIDATION_FOR_ISVALID.getParamName()); + DatabricksJdbcUrlParams.ENABLE_SQL_VALIDATION_FOR_IS_VALID.getParamName()); if ("1".equals(enableSqlValidation)) { try (Statement stmt = createStatement()) { diff --git a/src/main/java/com/databricks/jdbc/common/DatabricksJdbcConstants.java b/src/main/java/com/databricks/jdbc/common/DatabricksJdbcConstants.java index e339a31311..93a67f98b2 100644 --- a/src/main/java/com/databricks/jdbc/common/DatabricksJdbcConstants.java +++ b/src/main/java/com/databricks/jdbc/common/DatabricksJdbcConstants.java @@ -91,7 +91,7 @@ public final class DatabricksJdbcConstants { ALLOWED_STAGING_INGESTION_PATHS, DatabricksJdbcUrlParams.AUTH_ACCESS_TOKEN.getParamName(), DatabricksJdbcUrlParams.APPLICATION_NAME.getParamName(), - DatabricksJdbcUrlParams.ENABLE_SQL_VALIDATION_FOR_ISVALID.getParamName()); + DatabricksJdbcUrlParams.ENABLE_SQL_VALIDATION_FOR_IS_VALID.getParamName()); public static final Map JSON_HTTP_HEADERS = Map.of( "Accept", "application/json", diff --git a/src/main/java/com/databricks/jdbc/common/DatabricksJdbcUrlParams.java b/src/main/java/com/databricks/jdbc/common/DatabricksJdbcUrlParams.java index e94143a479..d0b04dee37 100644 --- a/src/main/java/com/databricks/jdbc/common/DatabricksJdbcUrlParams.java +++ b/src/main/java/com/databricks/jdbc/common/DatabricksJdbcUrlParams.java @@ -154,7 +154,7 @@ public enum DatabricksJdbcUrlParams { "HttpConnectionRequestTimeout", "HTTP connection request timeout in seconds"), CLOUD_FETCH_SPEED_THRESHOLD( "CloudFetchSpeedThreshold", "Minimum expected download speed in MB/s", "0.1"), - ENABLE_SQL_VALIDATION_FOR_ISVALID( + ENABLE_SQL_VALIDATION_FOR_IS_VALID( "EnableSQLValidationForIsValid", "Enable SQL query execution for connection validation in isValid() method", "0"); diff --git a/src/main/java/com/databricks/jdbc/common/util/DatabricksDriverPropertyUtil.java b/src/main/java/com/databricks/jdbc/common/util/DatabricksDriverPropertyUtil.java index fc6d4bf4c7..8ec0766c3a 100644 --- a/src/main/java/com/databricks/jdbc/common/util/DatabricksDriverPropertyUtil.java +++ b/src/main/java/com/databricks/jdbc/common/util/DatabricksDriverPropertyUtil.java @@ -38,7 +38,7 @@ public class DatabricksDriverPropertyUtil { DatabricksJdbcUrlParams.DEFAULT_STRING_COLUMN_LENGTH, DatabricksJdbcUrlParams.SOCKET_TIMEOUT, DatabricksJdbcUrlParams.ENABLE_TOKEN_CACHE, - DatabricksJdbcUrlParams.ENABLE_SQL_VALIDATION_FOR_ISVALID); + DatabricksJdbcUrlParams.ENABLE_SQL_VALIDATION_FOR_IS_VALID); public static List getMissingProperties(String url, Properties info) throws DatabricksParsingException { From b95a0edfd1c2fc402b99c3a77c5d5ad246d34b6b Mon Sep 17 00:00:00 2001 From: Madhavendra Rathore Date: Tue, 26 Aug 2025 10:28:57 +0530 Subject: [PATCH 03/15] Updated NEXT_CHANGELOG --- NEXT_CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/NEXT_CHANGELOG.md b/NEXT_CHANGELOG.md index 035fdbf77e..b24852029c 100644 --- a/NEXT_CHANGELOG.md +++ b/NEXT_CHANGELOG.md @@ -6,6 +6,7 @@ - **Query Tags support**: Added ability to attach key-value tags to SQL queries for analytical purposes that would appear in `system.query.history` table. Example: `jdbc:databricks://host;QUERY_TAGS=team:marketing,dashboard:abc123`. - **SQL Scripting support**: Added support for [SQL Scripting](https://docs.databricks.com/aws/en/sql/language-manual/sql-ref-scripting) +- **Configurable SQL validation in isValid()**: Added `EnableSQLValidationForIsValid` connection property to control whether `isValid()` method executes an actual SQL query for server-side validation. Default behavior remains fast client-side validation only. - Added a client property `enableVolumeOperations` to enable GET/PUT/REMOVE volume operations on a stream. For backward compatibility, allowedVolumeIngestionPaths can also be used for REMOVE operation. ### Updated From d43283a65c05901c0e0f3c57fa3f623bf3ce4dd7 Mon Sep 17 00:00:00 2001 From: Madhavendra Rathore Date: Tue, 26 Aug 2025 11:21:06 +0530 Subject: [PATCH 04/15] Added a get method for EnableSQLValidation --- .../com/databricks/jdbc/api/impl/DatabricksConnection.java | 5 +---- .../jdbc/api/impl/DatabricksConnectionContext.java | 5 +++++ .../jdbc/api/internal/IDatabricksConnectionContext.java | 2 ++ 3 files changed, 8 insertions(+), 4 deletions(-) diff --git a/src/main/java/com/databricks/jdbc/api/impl/DatabricksConnection.java b/src/main/java/com/databricks/jdbc/api/impl/DatabricksConnection.java index a79fa33d8e..afa736e229 100644 --- a/src/main/java/com/databricks/jdbc/api/impl/DatabricksConnection.java +++ b/src/main/java/com/databricks/jdbc/api/impl/DatabricksConnection.java @@ -10,7 +10,6 @@ import com.databricks.jdbc.api.internal.IDatabricksStatementInternal; import com.databricks.jdbc.common.DatabricksClientConfiguratorManager; import com.databricks.jdbc.common.DatabricksJdbcConstants; -import com.databricks.jdbc.common.DatabricksJdbcUrlParams; import com.databricks.jdbc.common.safe.DatabricksDriverFeatureFlagsContextFactory; import com.databricks.jdbc.common.util.DatabricksThreadContextHolder; import com.databricks.jdbc.common.util.UserAgentManager; @@ -418,9 +417,7 @@ public boolean isValid(int timeout) throws SQLException { if (isClosed()) { return false; } - String enableSqlValidation = - session.getConfigValue( - DatabricksJdbcUrlParams.ENABLE_SQL_VALIDATION_FOR_IS_VALID.getParamName()); + String enableSqlValidation = connectionContext.getEnableSQLValidationForIsValid(); if ("1".equals(enableSqlValidation)) { try (Statement stmt = createStatement()) { diff --git a/src/main/java/com/databricks/jdbc/api/impl/DatabricksConnectionContext.java b/src/main/java/com/databricks/jdbc/api/impl/DatabricksConnectionContext.java index 9b26b4f4aa..4f600a87e0 100644 --- a/src/main/java/com/databricks/jdbc/api/impl/DatabricksConnectionContext.java +++ b/src/main/java/com/databricks/jdbc/api/impl/DatabricksConnectionContext.java @@ -225,6 +225,11 @@ public String getHttpPath() { return getParameter(DatabricksJdbcUrlParams.HTTP_PATH); } + public String getEnableSQLValidationForIsValid() { + LOGGER.debug("String getEnableSQLValidationForIsValid()"); + return getParameter(DatabricksJdbcUrlParams.ENABLE_SQL_VALIDATION_FOR_IS_VALID); + } + @Override public String getHostForOAuth() { return this.host; diff --git a/src/main/java/com/databricks/jdbc/api/internal/IDatabricksConnectionContext.java b/src/main/java/com/databricks/jdbc/api/internal/IDatabricksConnectionContext.java index 2da74837d4..7a79f4945d 100644 --- a/src/main/java/com/databricks/jdbc/api/internal/IDatabricksConnectionContext.java +++ b/src/main/java/com/databricks/jdbc/api/internal/IDatabricksConnectionContext.java @@ -89,6 +89,8 @@ public interface IDatabricksConnectionContext { String getHttpPath(); + String getEnableSQLValidationForIsValid(); + String getProxyHost(); int getProxyPort(); From e4e4141ba7d35bcce9b532438a64b515938d4bc0 Mon Sep 17 00:00:00 2001 From: Madhavendra Rathore Date: Tue, 26 Aug 2025 11:35:31 +0530 Subject: [PATCH 05/15] Updated validation query to SELECT CURRENT_TIMESTAMP for better latency --- .../java/com/databricks/jdbc/api/impl/DatabricksConnection.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/com/databricks/jdbc/api/impl/DatabricksConnection.java b/src/main/java/com/databricks/jdbc/api/impl/DatabricksConnection.java index afa736e229..4048720a39 100644 --- a/src/main/java/com/databricks/jdbc/api/impl/DatabricksConnection.java +++ b/src/main/java/com/databricks/jdbc/api/impl/DatabricksConnection.java @@ -422,7 +422,7 @@ public boolean isValid(int timeout) throws SQLException { if ("1".equals(enableSqlValidation)) { try (Statement stmt = createStatement()) { stmt.setQueryTimeout(timeout); - stmt.execute("SELECT 1"); + stmt.execute("SELECT CURRENT_TIMESTAMP"); return true; } catch (SQLException e) { LOGGER.debug("SQL validation failed for isValid(): {}", e.getMessage()); From ff3c0310f98df16000446688ccc91424f998ed1c Mon Sep 17 00:00:00 2001 From: Madhavendra Rathore Date: Tue, 26 Aug 2025 11:47:25 +0530 Subject: [PATCH 06/15] Updated validation query to SELECT VERSION() for optimised latency --- .../java/com/databricks/jdbc/api/impl/DatabricksConnection.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/com/databricks/jdbc/api/impl/DatabricksConnection.java b/src/main/java/com/databricks/jdbc/api/impl/DatabricksConnection.java index 4048720a39..7ace07689e 100644 --- a/src/main/java/com/databricks/jdbc/api/impl/DatabricksConnection.java +++ b/src/main/java/com/databricks/jdbc/api/impl/DatabricksConnection.java @@ -422,7 +422,7 @@ public boolean isValid(int timeout) throws SQLException { if ("1".equals(enableSqlValidation)) { try (Statement stmt = createStatement()) { stmt.setQueryTimeout(timeout); - stmt.execute("SELECT CURRENT_TIMESTAMP"); + stmt.execute("SELECT VERSION()"); return true; } catch (SQLException e) { LOGGER.debug("SQL validation failed for isValid(): {}", e.getMessage()); From 1641117f10f6257b5be19150e16a0044ab6cdb21 Mon Sep 17 00:00:00 2001 From: Madhavendra Rathore Date: Tue, 26 Aug 2025 12:19:15 +0530 Subject: [PATCH 07/15] Added comments and updated changelog --- NEXT_CHANGELOG.md | 2 +- .../jdbc/api/internal/IDatabricksConnectionContext.java | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/NEXT_CHANGELOG.md b/NEXT_CHANGELOG.md index b24852029c..26b197741b 100644 --- a/NEXT_CHANGELOG.md +++ b/NEXT_CHANGELOG.md @@ -6,7 +6,7 @@ - **Query Tags support**: Added ability to attach key-value tags to SQL queries for analytical purposes that would appear in `system.query.history` table. Example: `jdbc:databricks://host;QUERY_TAGS=team:marketing,dashboard:abc123`. - **SQL Scripting support**: Added support for [SQL Scripting](https://docs.databricks.com/aws/en/sql/language-manual/sql-ref-scripting) -- **Configurable SQL validation in isValid()**: Added `EnableSQLValidationForIsValid` connection property to control whether `isValid()` method executes an actual SQL query for server-side validation. Default behavior remains fast client-side validation only. +- **Configurable SQL validation in isValid()**: Added `EnableSQLValidationForIsValid` connection property to control whether `isValid()` method executes an actual SQL query for server-side validation. Default value is 0. - Added a client property `enableVolumeOperations` to enable GET/PUT/REMOVE volume operations on a stream. For backward compatibility, allowedVolumeIngestionPaths can also be used for REMOVE operation. ### Updated diff --git a/src/main/java/com/databricks/jdbc/api/internal/IDatabricksConnectionContext.java b/src/main/java/com/databricks/jdbc/api/internal/IDatabricksConnectionContext.java index 7a79f4945d..09bea379a1 100644 --- a/src/main/java/com/databricks/jdbc/api/internal/IDatabricksConnectionContext.java +++ b/src/main/java/com/databricks/jdbc/api/internal/IDatabricksConnectionContext.java @@ -89,6 +89,7 @@ public interface IDatabricksConnectionContext { String getHttpPath(); + /** Returns the value of the EnableSQLValidationForIsValid connection property. */ String getEnableSQLValidationForIsValid(); String getProxyHost(); From 5fbbc5042b442e59bb6b983d6574478f7ac91bb7 Mon Sep 17 00:00:00 2001 From: Madhavendra Rathore Date: Tue, 26 Aug 2025 14:16:22 +0530 Subject: [PATCH 08/15] Updated the get method for enable SQL validation to return a boolean --- .../jdbc/api/impl/DatabricksConnection.java | 11 +++++------ .../jdbc/api/impl/DatabricksConnectionContext.java | 5 +++-- .../api/internal/IDatabricksConnectionContext.java | 2 +- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/src/main/java/com/databricks/jdbc/api/impl/DatabricksConnection.java b/src/main/java/com/databricks/jdbc/api/impl/DatabricksConnection.java index 7ace07689e..39a7c7ea08 100644 --- a/src/main/java/com/databricks/jdbc/api/impl/DatabricksConnection.java +++ b/src/main/java/com/databricks/jdbc/api/impl/DatabricksConnection.java @@ -417,15 +417,14 @@ public boolean isValid(int timeout) throws SQLException { if (isClosed()) { return false; } - String enableSqlValidation = connectionContext.getEnableSQLValidationForIsValid(); - - if ("1".equals(enableSqlValidation)) { + if (connectionContext.getEnableSQLValidationForIsValid()) { try (Statement stmt = createStatement()) { stmt.setQueryTimeout(timeout); - stmt.execute("SELECT VERSION()"); + // This is a lightweight query to check if the connection is valid + stmt.execute("SELECT V"); return true; - } catch (SQLException e) { - LOGGER.debug("SQL validation failed for isValid(): {}", e.getMessage()); + } catch (Exception e) { + LOGGER.debug("Validation failed for isValid(): {}", e.getMessage()); return false; } } diff --git a/src/main/java/com/databricks/jdbc/api/impl/DatabricksConnectionContext.java b/src/main/java/com/databricks/jdbc/api/impl/DatabricksConnectionContext.java index 4f600a87e0..e2fe52c771 100644 --- a/src/main/java/com/databricks/jdbc/api/impl/DatabricksConnectionContext.java +++ b/src/main/java/com/databricks/jdbc/api/impl/DatabricksConnectionContext.java @@ -225,9 +225,10 @@ public String getHttpPath() { return getParameter(DatabricksJdbcUrlParams.HTTP_PATH); } - public String getEnableSQLValidationForIsValid() { + public boolean getEnableSQLValidationForIsValid() { LOGGER.debug("String getEnableSQLValidationForIsValid()"); - return getParameter(DatabricksJdbcUrlParams.ENABLE_SQL_VALIDATION_FOR_IS_VALID); + return getParameter(DatabricksJdbcUrlParams.ENABLE_SQL_VALIDATION_FOR_IS_VALID, "0") + .equals("1"); } @Override diff --git a/src/main/java/com/databricks/jdbc/api/internal/IDatabricksConnectionContext.java b/src/main/java/com/databricks/jdbc/api/internal/IDatabricksConnectionContext.java index 09bea379a1..1b22e1bdd4 100644 --- a/src/main/java/com/databricks/jdbc/api/internal/IDatabricksConnectionContext.java +++ b/src/main/java/com/databricks/jdbc/api/internal/IDatabricksConnectionContext.java @@ -90,7 +90,7 @@ public interface IDatabricksConnectionContext { String getHttpPath(); /** Returns the value of the EnableSQLValidationForIsValid connection property. */ - String getEnableSQLValidationForIsValid(); + boolean getEnableSQLValidationForIsValid(); String getProxyHost(); From 9d37fdae9f34bd7f725ee3ce38218373d2152ef5 Mon Sep 17 00:00:00 2001 From: Madhavendra Rathore Date: Tue, 26 Aug 2025 15:26:11 +0530 Subject: [PATCH 09/15] Query bug fix --- .../java/com/databricks/jdbc/api/impl/DatabricksConnection.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/com/databricks/jdbc/api/impl/DatabricksConnection.java b/src/main/java/com/databricks/jdbc/api/impl/DatabricksConnection.java index 39a7c7ea08..5775002da0 100644 --- a/src/main/java/com/databricks/jdbc/api/impl/DatabricksConnection.java +++ b/src/main/java/com/databricks/jdbc/api/impl/DatabricksConnection.java @@ -421,7 +421,7 @@ public boolean isValid(int timeout) throws SQLException { try (Statement stmt = createStatement()) { stmt.setQueryTimeout(timeout); // This is a lightweight query to check if the connection is valid - stmt.execute("SELECT V"); + stmt.execute("SELECT VERSION()"); return true; } catch (Exception e) { LOGGER.debug("Validation failed for isValid(): {}", e.getMessage()); From 471ce414a00cf5bc1268c939c4ea7a891f379c4f Mon Sep 17 00:00:00 2001 From: Madhavendra Rathore Date: Tue, 26 Aug 2025 19:08:10 +0530 Subject: [PATCH 10/15] Added unit test for isValid() and removed ENABLE_SQL_VALIDATION_FOR_IS_VALID from client info properties --- .../jdbc/common/DatabricksJdbcConstants.java | 3 +- .../api/impl/DatabricksConnectionTest.java | 34 +++++++++++++++++++ 2 files changed, 35 insertions(+), 2 deletions(-) diff --git a/src/main/java/com/databricks/jdbc/common/DatabricksJdbcConstants.java b/src/main/java/com/databricks/jdbc/common/DatabricksJdbcConstants.java index 93a67f98b2..bbcacb1277 100644 --- a/src/main/java/com/databricks/jdbc/common/DatabricksJdbcConstants.java +++ b/src/main/java/com/databricks/jdbc/common/DatabricksJdbcConstants.java @@ -90,8 +90,7 @@ public final class DatabricksJdbcConstants { ENABLE_VOLUME_OPERATIONS, ALLOWED_STAGING_INGESTION_PATHS, DatabricksJdbcUrlParams.AUTH_ACCESS_TOKEN.getParamName(), - DatabricksJdbcUrlParams.APPLICATION_NAME.getParamName(), - DatabricksJdbcUrlParams.ENABLE_SQL_VALIDATION_FOR_IS_VALID.getParamName()); + DatabricksJdbcUrlParams.APPLICATION_NAME.getParamName()); public static final Map JSON_HTTP_HEADERS = Map.of( "Accept", "application/json", diff --git a/src/test/java/com/databricks/jdbc/api/impl/DatabricksConnectionTest.java b/src/test/java/com/databricks/jdbc/api/impl/DatabricksConnectionTest.java index 25ccc392e3..afc0905117 100644 --- a/src/test/java/com/databricks/jdbc/api/impl/DatabricksConnectionTest.java +++ b/src/test/java/com/databricks/jdbc/api/impl/DatabricksConnectionTest.java @@ -483,4 +483,38 @@ public void testQueryTagsInSessionConfigs() throws SQLException { assertTrue(sessionConfigs.containsKey("query_tags")); assertEquals("team:marketing,dashboard:abc123", sessionConfigs.get("query_tags")); } + + @Test + public void testIsValidWithSQLValidationEnabled() throws SQLException { + String jdbcUrlWithValidation = CATALOG_SCHEMA_JDBC_URL + ";EnableSQLValidationForIsValid=1"; + IDatabricksConnectionContext connectionContextWithValidation = + DatabricksConnectionContext.parse(jdbcUrlWithValidation, new Properties()); + when(databricksClient.createSession( + new Warehouse(WAREHOUSE_ID), CATALOG, SCHEMA, new HashMap<>())) + .thenReturn(IMMUTABLE_SESSION_INFO); + connection = new DatabricksConnection(connectionContextWithValidation, databricksClient); + connection.open(); + DatabricksConnection spyConnection = spy(connection); + DatabricksStatement mockStatement = mock(DatabricksStatement.class); + doReturn(mockStatement).when(spyConnection).createStatement(); + doNothing().when(mockStatement).setQueryTimeout(anyInt()); + when(mockStatement.execute("SELECT VERSION()")).thenReturn(true); + + assertTrue(spyConnection.isValid(5)); + verify(spyConnection).createStatement(); + verify(mockStatement).setQueryTimeout(5); + verify(mockStatement).execute("SELECT VERSION()"); + + DatabricksStatement mockStatementFail = mock(DatabricksStatement.class); + doReturn(mockStatementFail).when(spyConnection).createStatement(); + doNothing().when(mockStatementFail).setQueryTimeout(anyInt()); + when(mockStatementFail.execute("SELECT VERSION()")) + .thenThrow(new SQLException("Connection lost")); + + assertFalse(spyConnection.isValid(5)); + verify(mockStatementFail).setQueryTimeout(5); + verify(mockStatementFail).execute("SELECT VERSION()"); + + connection.close(); + } } From 626749e4940c81a81544d6088094fb19c2bc16b1 Mon Sep 17 00:00:00 2001 From: Madhavendra Rathore Date: Sat, 30 Aug 2025 14:56:46 +0530 Subject: [PATCH 11/15] Added enableSQLValidationForIsValid in the telemetry --- .../model/telemetry/DriverConnectionParameters.java | 10 ++++++++++ .../com/databricks/jdbc/telemetry/TelemetryHelper.java | 3 ++- 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/src/main/java/com/databricks/jdbc/model/telemetry/DriverConnectionParameters.java b/src/main/java/com/databricks/jdbc/model/telemetry/DriverConnectionParameters.java index 8e974f7be9..987faa8f5f 100644 --- a/src/main/java/com/databricks/jdbc/model/telemetry/DriverConnectionParameters.java +++ b/src/main/java/com/databricks/jdbc/model/telemetry/DriverConnectionParameters.java @@ -133,6 +133,9 @@ public class DriverConnectionParameters { @JsonProperty("async_poll_interval_millis") int asyncPollIntervalMillis; + @JsonProperty("enable_sql_validation_for_is_valid") + boolean enableSqlValidationForIsValid; + public DriverConnectionParameters setHttpPath(String httpPath) { this.httpPath = httpPath; return this; @@ -348,6 +351,12 @@ public DriverConnectionParameters setAsyncPollIntervalMillis(int asyncPollInterv return this; } + public DriverConnectionParameters setEnableSqlValidationForIsValid( + boolean enableSqlValidationForIsValid) { + this.enableSqlValidationForIsValid = enableSqlValidationForIsValid; + return this; + } + @Override public String toString() { return new ToStringer(DriverConnectionParameters.class) @@ -393,6 +402,7 @@ public String toString() { .add("useSystemTrustStore", useSystemTrustStore) .add("rowsFetchedPerBlock", rowsFetchedPerBlock) .add("asyncPollIntervalMillis", asyncPollIntervalMillis) + .add("enableSqlValidationForIsValid", enableSqlValidationForIsValid) .toString(); } } diff --git a/src/main/java/com/databricks/jdbc/telemetry/TelemetryHelper.java b/src/main/java/com/databricks/jdbc/telemetry/TelemetryHelper.java index 7edef9b4cc..441a97aea3 100644 --- a/src/main/java/com/databricks/jdbc/telemetry/TelemetryHelper.java +++ b/src/main/java/com/databricks/jdbc/telemetry/TelemetryHelper.java @@ -197,7 +197,8 @@ private static DriverConnectionParameters buildDriverConnectionParameters( .setRowsFetchedPerBlock(connectionContext.getRowsFetchedPerBlock()) .setAsyncPollIntervalMillis(connectionContext.getAsyncExecPollInterval()) .setEnableTokenCache(connectionContext.isTokenCacheEnabled()) - .setHttpPath(connectionContext.getHttpPath()); + .setHttpPath(connectionContext.getHttpPath()) + .setEnableSqlValidationForIsValid(connectionContext.getEnableSQLValidationForIsValid()); if (connectionContext.useJWTAssertion()) { connectionParameters .setEnableJwtAssertion(true) From 182a50b5ed9300a6b56e6c6a44ffa4bdaa55870a Mon Sep 17 00:00:00 2001 From: Madhavendra Rathore Date: Sat, 30 Aug 2025 20:02:47 +0530 Subject: [PATCH 12/15] Commented the telemetry code for EnableSQLValidationForIsValid. To be used after it is added to the proto file --- .../telemetry/DriverConnectionParameters.java | 19 +++++++++++-------- .../jdbc/telemetry/TelemetryHelper.java | 5 +++-- 2 files changed, 14 insertions(+), 10 deletions(-) diff --git a/src/main/java/com/databricks/jdbc/model/telemetry/DriverConnectionParameters.java b/src/main/java/com/databricks/jdbc/model/telemetry/DriverConnectionParameters.java index 987faa8f5f..88d415375b 100644 --- a/src/main/java/com/databricks/jdbc/model/telemetry/DriverConnectionParameters.java +++ b/src/main/java/com/databricks/jdbc/model/telemetry/DriverConnectionParameters.java @@ -133,8 +133,9 @@ public class DriverConnectionParameters { @JsonProperty("async_poll_interval_millis") int asyncPollIntervalMillis; - @JsonProperty("enable_sql_validation_for_is_valid") - boolean enableSqlValidationForIsValid; + // TODO(PECOBLR-772) : first add the property in the proto file + // @JsonProperty("enable_sql_validation_for_is_valid") + // boolean enableSqlValidationForIsValid; public DriverConnectionParameters setHttpPath(String httpPath) { this.httpPath = httpPath; @@ -351,11 +352,12 @@ public DriverConnectionParameters setAsyncPollIntervalMillis(int asyncPollInterv return this; } - public DriverConnectionParameters setEnableSqlValidationForIsValid( - boolean enableSqlValidationForIsValid) { - this.enableSqlValidationForIsValid = enableSqlValidationForIsValid; - return this; - } + // TODO(PECOBLR-772) : first add the property in the proto file + // public DriverConnectionParameters setEnableSqlValidationForIsValid( + // boolean enableSqlValidationForIsValid) { + // this.enableSqlValidationForIsValid = enableSqlValidationForIsValid; + // return this; + // } @Override public String toString() { @@ -402,7 +404,8 @@ public String toString() { .add("useSystemTrustStore", useSystemTrustStore) .add("rowsFetchedPerBlock", rowsFetchedPerBlock) .add("asyncPollIntervalMillis", asyncPollIntervalMillis) - .add("enableSqlValidationForIsValid", enableSqlValidationForIsValid) + // TODO(PECOBLR-772) : first add the property in the proto file + // .add("enableSqlValidationForIsValid", enableSqlValidationForIsValid) .toString(); } } diff --git a/src/main/java/com/databricks/jdbc/telemetry/TelemetryHelper.java b/src/main/java/com/databricks/jdbc/telemetry/TelemetryHelper.java index 441a97aea3..2564ccf9c6 100644 --- a/src/main/java/com/databricks/jdbc/telemetry/TelemetryHelper.java +++ b/src/main/java/com/databricks/jdbc/telemetry/TelemetryHelper.java @@ -197,8 +197,9 @@ private static DriverConnectionParameters buildDriverConnectionParameters( .setRowsFetchedPerBlock(connectionContext.getRowsFetchedPerBlock()) .setAsyncPollIntervalMillis(connectionContext.getAsyncExecPollInterval()) .setEnableTokenCache(connectionContext.isTokenCacheEnabled()) - .setHttpPath(connectionContext.getHttpPath()) - .setEnableSqlValidationForIsValid(connectionContext.getEnableSQLValidationForIsValid()); + .setHttpPath(connectionContext.getHttpPath()); + // TODO(PECOBLR-772) : Add this property in the proto file + // .setEnableSqlValidationForIsValid(connectionContext.getEnableSQLValidationForIsValid()); if (connectionContext.useJWTAssertion()) { connectionParameters .setEnableJwtAssertion(true) From b5d9040117aae9ae0341bb23552457361071709a Mon Sep 17 00:00:00 2001 From: Madhavendra Rathore Date: Tue, 2 Sep 2025 17:57:28 +0530 Subject: [PATCH 13/15] Removed the comments for telemetry support for EnableSQLValidation for isValid --- .../model/telemetry/DriverConnectionParameters.java | 13 ------------- .../databricks/jdbc/telemetry/TelemetryHelper.java | 2 -- 2 files changed, 15 deletions(-) diff --git a/src/main/java/com/databricks/jdbc/model/telemetry/DriverConnectionParameters.java b/src/main/java/com/databricks/jdbc/model/telemetry/DriverConnectionParameters.java index 88d415375b..8e974f7be9 100644 --- a/src/main/java/com/databricks/jdbc/model/telemetry/DriverConnectionParameters.java +++ b/src/main/java/com/databricks/jdbc/model/telemetry/DriverConnectionParameters.java @@ -133,10 +133,6 @@ public class DriverConnectionParameters { @JsonProperty("async_poll_interval_millis") int asyncPollIntervalMillis; - // TODO(PECOBLR-772) : first add the property in the proto file - // @JsonProperty("enable_sql_validation_for_is_valid") - // boolean enableSqlValidationForIsValid; - public DriverConnectionParameters setHttpPath(String httpPath) { this.httpPath = httpPath; return this; @@ -352,13 +348,6 @@ public DriverConnectionParameters setAsyncPollIntervalMillis(int asyncPollInterv return this; } - // TODO(PECOBLR-772) : first add the property in the proto file - // public DriverConnectionParameters setEnableSqlValidationForIsValid( - // boolean enableSqlValidationForIsValid) { - // this.enableSqlValidationForIsValid = enableSqlValidationForIsValid; - // return this; - // } - @Override public String toString() { return new ToStringer(DriverConnectionParameters.class) @@ -404,8 +393,6 @@ public String toString() { .add("useSystemTrustStore", useSystemTrustStore) .add("rowsFetchedPerBlock", rowsFetchedPerBlock) .add("asyncPollIntervalMillis", asyncPollIntervalMillis) - // TODO(PECOBLR-772) : first add the property in the proto file - // .add("enableSqlValidationForIsValid", enableSqlValidationForIsValid) .toString(); } } diff --git a/src/main/java/com/databricks/jdbc/telemetry/TelemetryHelper.java b/src/main/java/com/databricks/jdbc/telemetry/TelemetryHelper.java index 2564ccf9c6..7edef9b4cc 100644 --- a/src/main/java/com/databricks/jdbc/telemetry/TelemetryHelper.java +++ b/src/main/java/com/databricks/jdbc/telemetry/TelemetryHelper.java @@ -198,8 +198,6 @@ private static DriverConnectionParameters buildDriverConnectionParameters( .setAsyncPollIntervalMillis(connectionContext.getAsyncExecPollInterval()) .setEnableTokenCache(connectionContext.isTokenCacheEnabled()) .setHttpPath(connectionContext.getHttpPath()); - // TODO(PECOBLR-772) : Add this property in the proto file - // .setEnableSqlValidationForIsValid(connectionContext.getEnableSQLValidationForIsValid()); if (connectionContext.useJWTAssertion()) { connectionParameters .setEnableJwtAssertion(true) From 5c78367d0f1421cf44f88363aa00e1a3b4a28107 Mon Sep 17 00:00:00 2001 From: Madhavendra Rathore Date: Tue, 2 Sep 2025 18:25:22 +0530 Subject: [PATCH 14/15] Updated NEXT_CHANGELOG.md --- NEXT_CHANGELOG.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/NEXT_CHANGELOG.md b/NEXT_CHANGELOG.md index 26b197741b..f9e9376421 100644 --- a/NEXT_CHANGELOG.md +++ b/NEXT_CHANGELOG.md @@ -6,8 +6,9 @@ - **Query Tags support**: Added ability to attach key-value tags to SQL queries for analytical purposes that would appear in `system.query.history` table. Example: `jdbc:databricks://host;QUERY_TAGS=team:marketing,dashboard:abc123`. - **SQL Scripting support**: Added support for [SQL Scripting](https://docs.databricks.com/aws/en/sql/language-manual/sql-ref-scripting) +- Added a client property `enableVolumeOperations` to enable GET/PUT/REMOVE volume operations on a stream. For backward compatibility, allowedVolumeIngestionPaths can also be used for REMOVE operation. +- Support for fetching schemas across all catalogs (when catalog is specified as null or a wildcard) in `DatabaseMetaData#getSchemas` API in SQL Execution mode. - **Configurable SQL validation in isValid()**: Added `EnableSQLValidationForIsValid` connection property to control whether `isValid()` method executes an actual SQL query for server-side validation. Default value is 0. -- Added a client property `enableVolumeOperations` to enable GET/PUT/REMOVE volume operations on a stream. For backward compatibility, allowedVolumeIngestionPaths can also be used for REMOVE operation. ### Updated - Databricks SDK dependency upgraded to latest version 0.60.0 From 77586311824763ee46422dc07fa507ef1dfa81b4 Mon Sep 17 00:00:00 2001 From: Madhavendra Rathore Date: Tue, 2 Sep 2025 19:10:31 +0530 Subject: [PATCH 15/15] Added a toString function in BITConverter to fix getString for boolean columns. Also fixed the toBoolean to handle null. Added tests for the BITConverter --- .../api/impl/converters/BitConverter.java | 12 +- .../api/impl/converters/BitConverterTest.java | 119 ++++++++++++++++++ 2 files changed, 130 insertions(+), 1 deletion(-) create mode 100644 src/test/java/com/databricks/jdbc/api/impl/converters/BitConverterTest.java diff --git a/src/main/java/com/databricks/jdbc/api/impl/converters/BitConverter.java b/src/main/java/com/databricks/jdbc/api/impl/converters/BitConverter.java index 5d0947ed29..eab8ffc4b6 100644 --- a/src/main/java/com/databricks/jdbc/api/impl/converters/BitConverter.java +++ b/src/main/java/com/databricks/jdbc/api/impl/converters/BitConverter.java @@ -17,7 +17,17 @@ public boolean toBoolean(Object object) throws DatabricksSQLException { return Boolean.parseBoolean((String) object); } throw new DatabricksSQLException( - "Unsupported type for conversion to BIT: " + object.getClass(), + "Unsupported type for conversion to BIT: " + (object == null ? "null" : object.getClass()), DatabricksDriverErrorCode.UNSUPPORTED_OPERATION); } + + @Override + public String toString(Object object) throws DatabricksSQLException { + if (object instanceof Boolean) { + return object.toString(); + } + // For other types, fall back to the default behavior + throw new DatabricksSQLException( + "Unsupported String conversion operation", DatabricksDriverErrorCode.UNSUPPORTED_OPERATION); + } } diff --git a/src/test/java/com/databricks/jdbc/api/impl/converters/BitConverterTest.java b/src/test/java/com/databricks/jdbc/api/impl/converters/BitConverterTest.java new file mode 100644 index 0000000000..504905a0ce --- /dev/null +++ b/src/test/java/com/databricks/jdbc/api/impl/converters/BitConverterTest.java @@ -0,0 +1,119 @@ +package com.databricks.jdbc.api.impl.converters; + +import static org.junit.jupiter.api.Assertions.*; + +import com.databricks.jdbc.exception.DatabricksSQLException; +import org.junit.jupiter.api.Test; + +public class BitConverterTest { + + private final BitConverter bitConverter = new BitConverter(); + + @Test + public void testToStringWithBooleanTrue() throws DatabricksSQLException { + assertEquals("true", bitConverter.toString(true)); + } + + @Test + public void testToStringWithBooleanFalse() throws DatabricksSQLException { + assertEquals("false", bitConverter.toString(false)); + } + + @Test + public void testToStringWithBooleanObject() throws DatabricksSQLException { + Boolean trueObject = Boolean.TRUE; + Boolean falseObject = Boolean.FALSE; + + assertEquals("true", bitConverter.toString(trueObject)); + assertEquals("false", bitConverter.toString(falseObject)); + } + + @Test + public void testToStringWithUnsupportedType() { + DatabricksSQLException exception = + assertThrows(DatabricksSQLException.class, () -> bitConverter.toString("not a boolean")); + + assertTrue(exception.getMessage().contains("Unsupported String conversion operation")); + assertEquals("UNSUPPORTED_OPERATION", exception.getSQLState()); + } + + @Test + public void testToStringWithNumber() { + DatabricksSQLException exception = + assertThrows(DatabricksSQLException.class, () -> bitConverter.toString(123)); + + assertTrue(exception.getMessage().contains("Unsupported String conversion operation")); + assertEquals("UNSUPPORTED_OPERATION", exception.getSQLState()); + } + + @Test + public void testToBooleanWithBooleanTrue() throws DatabricksSQLException { + assertTrue(bitConverter.toBoolean(true)); + } + + @Test + public void testToBooleanWithBooleanFalse() throws DatabricksSQLException { + assertFalse(bitConverter.toBoolean(false)); + } + + @Test + public void testToBooleanWithBooleanObject() throws DatabricksSQLException { + Boolean trueObject = Boolean.TRUE; + Boolean falseObject = Boolean.FALSE; + + assertTrue(bitConverter.toBoolean(trueObject)); + assertFalse(bitConverter.toBoolean(falseObject)); + } + + @Test + public void testToBooleanWithNumberZero() throws DatabricksSQLException { + assertFalse(bitConverter.toBoolean(0)); + assertFalse(bitConverter.toBoolean(0L)); + assertFalse(bitConverter.toBoolean(0.0f)); + assertFalse(bitConverter.toBoolean(0.0)); + } + + @Test + public void testToBooleanWithNumberNonZero() throws DatabricksSQLException { + assertTrue(bitConverter.toBoolean(1)); + assertTrue(bitConverter.toBoolean(-1)); + assertTrue(bitConverter.toBoolean(42L)); + assertTrue(bitConverter.toBoolean(3.14f)); + assertTrue(bitConverter.toBoolean(2.71)); + } + + @Test + public void testToBooleanWithStringTrue() throws DatabricksSQLException { + assertTrue(bitConverter.toBoolean("true")); + assertTrue(bitConverter.toBoolean("TRUE")); + assertTrue(bitConverter.toBoolean("True")); + } + + @Test + public void testToBooleanWithStringFalse() throws DatabricksSQLException { + assertFalse(bitConverter.toBoolean("false")); + assertFalse(bitConverter.toBoolean("FALSE")); + assertFalse(bitConverter.toBoolean("False")); + assertFalse(bitConverter.toBoolean("anything else")); + } + + @Test + public void testToBooleanWithUnsupportedType() { + DatabricksSQLException exception = + assertThrows(DatabricksSQLException.class, () -> bitConverter.toBoolean(new Object())); + + assertTrue(exception.getMessage().contains("Unsupported type for conversion to BIT")); + assertTrue(exception.getMessage().contains("Object")); + assertEquals("UNSUPPORTED_OPERATION", exception.getSQLState()); + } + + @Test + public void testToBooleanWithNull() { + DatabricksSQLException exception = + assertThrows(DatabricksSQLException.class, () -> bitConverter.toBoolean(null)); + + assertTrue(exception.getMessage().contains("Unsupported type for conversion to BIT")); + assertTrue(exception.getMessage().contains("null")); + assertEquals("UNSUPPORTED_OPERATION", exception.getSQLState()); + } +}