Merge branch 'main' into cb2

Author: gopalldb

.github/workflows/releaseFreeze.yml

@@ -0,0 +1,60 @@
+name: Enforce Release Freeze
+
+on:
+  pull_request:
+    branches: [main]
+
+jobs:
+  freeze-check:
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Check out repository
+        uses: actions/checkout@v4
+
+      - name: Set up jq
+        run: sudo apt-get update && sudo apt-get install -y jq
+
+      - name: Enforce branch freeze logic
+        shell: bash
+        env:
+          # GITHUB_HEAD_REF: Source branch name for PRs
+          PR_BRANCH: ${{ github.head_ref }}
+        run: |
+          CONFIG_FILE="development/.release-config.json"
+
+          if [[ ! -f "$CONFIG_FILE" ]]; then
+            echo "✅ No release config file - passing check."
+            exit 0
+          fi
+
+          FREEZE=$(jq '.freeze' "$CONFIG_FILE")
+
+          if [[ "$FREEZE" != "true" ]]; then
+            echo "✅ Freeze is off - passing check."
+            exit 0
+          fi
+
+          ALLOW_LIST=( $(jq -r '.allow_list[]' "$CONFIG_FILE") )
+
+          BRANCH="$PR_BRANCH"
+          ALLOWED=0
+
+          # Check for exact branch name in allow_list
+          for ENTRY in $(jq -r '.allow_list[]' "$CONFIG_FILE"); do
+            if [[ "$BRANCH" == "$ENTRY" ]]; then
+              ALLOWED=1
+              break
+            fi
+          done
+
+          if [[ "$ALLOWED" == "1" ]]; then
+            echo "✅ Branch '$BRANCH' is in allow_list. Passing check."
+            exit 0
+          else
+            echo "❌ Release freeze is ACTIVE!"
+            REASON=$(jq -r '.reason' "$CONFIG_FILE")
+            echo "Reason: $REASON"
+            echo "Branch '$BRANCH' is NOT permitted to merge under current freeze rules."
+            exit 1
+          fi

NEXT_CHANGELOG.md

@@ -4,9 +4,18 @@
 
 ### Added
 
+- **Query Tags support**: Added ability to attach key-value tags to SQL queries for analytical purposes that would appear in `system.query.history` table. Example: `jdbc:databricks://host;QUERY_TAGS=team:marketing,dashboard:abc123`. 
+- **SQL Scripting support**: Added support for [SQL Scripting](https://docs.databricks.com/aws/en/sql/language-manual/sql-ref-scripting)
+- Added a client property `enableVolumeOperations` to enable  GET/PUT/REMOVE volume operations on a stream. For backward compatibility, allowedVolumeIngestionPaths can also be used for REMOVE operation.
+- Support for fetching schemas across all catalogs (when catalog is specified as null or a wildcard) in `DatabaseMetaData#getSchemas` API in SQL Execution mode.
+- **Configurable SQL validation in isValid()**: Added `EnableSQLValidationForIsValid` connection property to control whether `isValid()` method executes an actual SQL query for server-side validation. Default value is 0.
+- Implement multi-row INSERT batching optimization for prepared statements to improve performance when executing large batches of INSERT operations.
+
 ### Updated
+- Databricks SDK dependency upgraded to latest version 0.60.0
 
 ### Fixed
-
+- Integrated Azure U2M flow into driver for improved stability.
+- Fixed `ResultSet.getString` for Boolean columns in Metadata result set.
 ---
 *Note: When making changes, please add your change under the appropriate section with a brief description.* 

README.md

@@ -91,6 +91,7 @@ Optional parameters:
 - `OAuth2RedirectUrlPort` - Ports for redirect URL (default: 8020)
 - `EnableOIDCDiscovery` - Enable OIDC discovery (default: 1)
 - `OAuthDiscoveryURL` - OIDC discovery endpoint (default: /oidc/.well-known/oauth-authorization-server)
+- `EnableSQLValidationForIsValid` - Enable SQL query based validation in `isValid()` connection checks (default: 0)
 
 ### Logging
 

development/.release-freeze.json

@@ -0,0 +1,5 @@
+{
+  "freeze": false,
+  "reason": "Release description",
+  "allow_list": []
+}

pom.xml

@@ -55,7 +55,7 @@
     <httpclient.version>4.5.14</httpclient.version>
     <commons-configuration.version>2.10.1</commons-configuration.version>
     <commons-io.version>2.14.0</commons-io.version>
-    <databricks-sdk.version>0.52.0</databricks-sdk.version>
+    <databricks-sdk.version>0.60.0</databricks-sdk.version>
     <maven-surefire-plugin.version>3.1.2</maven-surefire-plugin.version>
     <sql-logic-test.version>0.3</sql-logic-test.version>
     <lz4-compression.version>1.8.0</lz4-compression.version>

scripts/githooks/post-commit

@@ -12,10 +12,23 @@ if ! git diff --quiet; then
     echo "Please 'git add' and 'git commit' these to include formatting changes."
     echo "========================================================================"
     echo
-    # Optional: show the changed files
     git status --short
 else
     echo
     echo "✓ Spotless checked: No further formatting changes needed after commit."
     echo
+fi
+
+# Check if the latest commit message contains a Signed-off-by: line
+if ! git log -1 --pretty=%B | grep -q "Signed-off-by:"; then
+    echo
+    echo "========================================================================"
+    echo "❗ WARNING: Your last commit is missing a Signed-off-by line."
+    echo "Please amend your commit with 'git commit -s --amend' to add it."
+    echo "========================================================================"
+    echo
+else
+    echo
+    echo "✓ DCO check: Last commit includes a Signed-off-by line."
+    echo
 fi

src/main/java/com/databricks/jdbc/api/impl/DatabricksConnection.java

@@ -414,7 +414,21 @@ public SQLXML createSQLXML() throws SQLException {
   @Override
   public boolean isValid(int timeout) throws SQLException {
     ValidationUtil.checkIfNonNegative(timeout, "timeout");
-    return !isClosed();
+    if (isClosed()) {
+      return false;
+    }
+    if (connectionContext.getEnableSQLValidationForIsValid()) {
+      try (Statement stmt = createStatement()) {
+        stmt.setQueryTimeout(timeout);
+        // This is a lightweight query to check if the connection is valid
+        stmt.execute("SELECT VERSION()");
+        return true;
+      } catch (Exception e) {
+        LOGGER.debug("Validation failed for isValid(): {}", e.getMessage());
+        return false;
+      }
+    }
+    return true;
   }
 
   /**

src/main/java/com/databricks/jdbc/api/impl/DatabricksConnectionContext.java

@@ -215,6 +215,11 @@ public String getHostUrl() throws DatabricksParsingException {
     }
   }
 
+  @Override
+  public String getHost() {
+    return this.host;
+  }
+
   @Override
   public IDatabricksComputeResource getComputeResource() {
     return computeResource;
@@ -225,6 +230,12 @@ public String getHttpPath() {
     return getParameter(DatabricksJdbcUrlParams.HTTP_PATH);
   }
 
+  public boolean getEnableSQLValidationForIsValid() {
+    LOGGER.debug("String getEnableSQLValidationForIsValid()");
+    return getParameter(DatabricksJdbcUrlParams.ENABLE_SQL_VALIDATION_FOR_IS_VALID, "0")
+        .equals("1");
+  }
+
   @Override
   public String getHostForOAuth() {
     return this.host;
@@ -1019,4 +1030,9 @@ public int getTelemetryFlushIntervalInMilliseconds() {
     return Math.max(
         1000, Integer.parseInt(getParameter(DatabricksJdbcUrlParams.TELEMETRY_FLUSH_INTERVAL)));
   }
+
+  @Override
+  public boolean isBatchedInsertsEnabled() {
+    return getParameter(DatabricksJdbcUrlParams.ENABLE_BATCHED_INSERTS).equals("1");
+  }
 }

src/main/java/com/databricks/jdbc/api/impl/DatabricksDatabaseMetaData.java

@@ -9,7 +9,6 @@
 import com.databricks.jdbc.api.internal.IDatabricksSession;
 import com.databricks.jdbc.common.*;
 import com.databricks.jdbc.common.util.DriverUtil;
-import com.databricks.jdbc.common.util.JdbcThreadUtils;
 import com.databricks.jdbc.dbclient.impl.common.MetadataResultSetBuilder;
 import com.databricks.jdbc.dbclient.impl.common.StatementId;
 import com.databricks.jdbc.exception.DatabricksSQLException;
@@ -20,8 +19,6 @@
 import com.databricks.sdk.service.sql.StatementState;
 import java.sql.*;
 import java.util.*;
-import java.util.concurrent.ExecutorService;
-import java.util.concurrent.Executors;
 
 public class DatabricksDatabaseMetaData implements DatabaseMetaData {
 
@@ -40,9 +37,6 @@ public class DatabricksDatabaseMetaData implements DatabaseMetaData {
   public static final String SYSTEM_FUNCTIONS = "DATABASE,IFNULL,USER";
   public static final String TIME_DATE_FUNCTIONS =
       "CURDATE,CURRENT_DATE,CURRENT_TIME,CURRENT_TIMESTAMP,CURTIME,DAYNAME,DAYOFMONTH,DAYOFWEEK,DAYOFYEAR,HOUR,MINUTE,MONTH,MONTHNAME,NOW,QUARTER,SECOND,TIMESTAMPADD,TIMESTAMPDIFF,WEEK,YEAR";
-  private static final Object THREAD_POOL_LOCK = new Object();
-  private static ExecutorService schemasThreadPool = null;
-  private static final int DEFAULT_MAX_THREADS = 10;
   private final IDatabricksConnectionInternal connection;
   private final IDatabricksSession session;
   private final MetadataResultSetBuilder metadataResultSetBuilder;
@@ -1505,48 +1499,6 @@ public ResultSet getSchemas(String catalog, String schemaPattern) throws SQLExce
         catalog, schemaPattern);
     throwExceptionIfConnectionIsClosed();
 
-    if (session.getConnectionContext().getClientType() == DatabricksClientType.SEA
-        && (catalog == null || catalog.equals("*") || catalog.equals("%"))) {
-      // Fetch catalogs from the metadata client
-      List<String> catalogList = new ArrayList<>();
-      try (ResultSet catalogs = getCatalogs()) {
-        while (catalogs.next()) {
-          String c = catalogs.getString(1);
-          if (c != null && !c.isEmpty()) {
-            catalogList.add(c);
-          }
-        }
-      }
-
-      // Process catalogs in parallel, gathering schema information
-      List<List<Object>> schemaRows =
-          JdbcThreadUtils.parallelFlatMap(
-              catalogList,
-              session.getConnectionContext(),
-              DEFAULT_MAX_THREADS, // Not significant since the executor is provided as a parameter
-              90, // 90 seconds timeout
-              c -> {
-                List<List<Object>> rows = new ArrayList<>();
-                try (ResultSet catalogSchemas =
-                    session.getDatabricksMetadataClient().listSchemas(session, c, schemaPattern)) {
-                  while (catalogSchemas.next()) {
-                    List<Object> schemaRow = new ArrayList<>();
-                    schemaRow.add(catalogSchemas.getString(1)); // TABLE_SCHEM
-                    schemaRow.add(catalogSchemas.getString(2)); // TABLE_CATALOG
-                    rows.add(schemaRow);
-                  }
-                } catch (SQLException e) {
-                  LOGGER.warn("Error fetching schemas for catalog %s %s", c, e.getMessage());
-                }
-                return rows;
-              },
-              getOrCreateSchemasThreadPool());
-
-      // Convert combined data into a result set
-      return metadataResultSetBuilder.getResultSetWithGivenRowsAndColumns(
-          SCHEMA_COLUMNS, schemaRows, METADATA_STATEMENT_ID, CommandName.LIST_SCHEMAS);
-    }
-
     return session.getDatabricksMetadataClient().listSchemas(session, catalog, schemaPattern);
   }
 
@@ -1659,23 +1611,6 @@ public boolean isWrapperFor(Class<?> iface) throws SQLException {
     return iface != null && iface.isAssignableFrom(this.getClass());
   }
 
-  private static ExecutorService getOrCreateSchemasThreadPool() {
-    synchronized (THREAD_POOL_LOCK) {
-      if (schemasThreadPool == null || schemasThreadPool.isShutdown()) {
-        // Could read max threads from a configuration property
-        schemasThreadPool =
-            Executors.newFixedThreadPool(
-                DEFAULT_MAX_THREADS,
-                r -> {
-                  Thread t = new Thread(r, "jdbc-schemas-fetcher");
-                  t.setDaemon(true);
-                  return t;
-                });
-      }
-      return schemasThreadPool;
-    }
-  }
-
   private void throwExceptionIfConnectionIsClosed() throws SQLException {
     LOGGER.debug("private void throwExceptionIfConnectionIsClosed()");
     if (!connection.getSession().isOpen()) {