Merge remote-tracking branch 'databricks/main' into fetch-requests-columnar-view

Author: jayantsing-db

.github/workflows/releaseFreeze.yml

@@ -0,0 +1,60 @@
+name: Enforce Release Freeze
+
+on:
+  pull_request:
+    branches: [main]
+
+jobs:
+  freeze-check:
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Check out repository
+        uses: actions/checkout@v4
+
+      - name: Set up jq
+        run: sudo apt-get update && sudo apt-get install -y jq
+
+      - name: Enforce branch freeze logic
+        shell: bash
+        env:
+          # GITHUB_HEAD_REF: Source branch name for PRs
+          PR_BRANCH: ${{ github.head_ref }}
+        run: |
+          CONFIG_FILE="development/.release-config.json"
+
+          if [[ ! -f "$CONFIG_FILE" ]]; then
+            echo "✅ No release config file - passing check."
+            exit 0
+          fi
+
+          FREEZE=$(jq '.freeze' "$CONFIG_FILE")
+
+          if [[ "$FREEZE" != "true" ]]; then
+            echo "✅ Freeze is off - passing check."
+            exit 0
+          fi
+
+          ALLOW_LIST=( $(jq -r '.allow_list[]' "$CONFIG_FILE") )
+
+          BRANCH="$PR_BRANCH"
+          ALLOWED=0
+
+          # Check for exact branch name in allow_list
+          for ENTRY in $(jq -r '.allow_list[]' "$CONFIG_FILE"); do
+            if [[ "$BRANCH" == "$ENTRY" ]]; then
+              ALLOWED=1
+              break
+            fi
+          done
+
+          if [[ "$ALLOWED" == "1" ]]; then
+            echo "✅ Branch '$BRANCH' is in allow_list. Passing check."
+            exit 0
+          else
+            echo "❌ Release freeze is ACTIVE!"
+            REASON=$(jq -r '.reason' "$CONFIG_FILE")
+            echo "Reason: $REASON"
+            echo "Branch '$BRANCH' is NOT permitted to merge under current freeze rules."
+            exit 1
+          fi

NEXT_CHANGELOG.md

@@ -6,7 +6,8 @@
 
 - **Query Tags support**: Added ability to attach key-value tags to SQL queries for analytical purposes that would appear in `system.query.history` table. Example: `jdbc:databricks://host;QUERY_TAGS=team:marketing,dashboard:abc123`. 
 - **SQL Scripting support**: Added support for [SQL Scripting](https://docs.databricks.com/aws/en/sql/language-manual/sql-ref-scripting)
-- Added a client property `enableVolumeOperations` to enable  GET/PUT/REMOVE volume operations on a stream. For backward compatibility, allowedVolumeIngestionPaths can also be used for REMOVE operation.   
+- Added a client property `enableVolumeOperations` to enable  GET/PUT/REMOVE volume operations on a stream. For backward compatibility, allowedVolumeIngestionPaths can also be used for REMOVE operation.
+- Support for fetching schemas across all catalogs (when catalog is specified as null or a wildcard) in `DatabaseMetaData#getSchemas` API in SQL Execution mode.
 
 ### Updated
 - Databricks SDK dependency upgraded to latest version 0.60.0

development/.release-freeze.json

@@ -0,0 +1,5 @@
+{
+  "freeze": false,
+  "reason": "Release description",
+  "allow_list": []
+}

src/main/java/com/databricks/jdbc/api/impl/DatabricksDatabaseMetaData.java

@@ -9,7 +9,6 @@
 import com.databricks.jdbc.api.internal.IDatabricksSession;
 import com.databricks.jdbc.common.*;
 import com.databricks.jdbc.common.util.DriverUtil;
-import com.databricks.jdbc.common.util.JdbcThreadUtils;
 import com.databricks.jdbc.dbclient.impl.common.MetadataResultSetBuilder;
 import com.databricks.jdbc.dbclient.impl.common.StatementId;
 import com.databricks.jdbc.exception.DatabricksSQLException;
@@ -20,8 +19,6 @@
 import com.databricks.sdk.service.sql.StatementState;
 import java.sql.*;
 import java.util.*;
-import java.util.concurrent.ExecutorService;
-import java.util.concurrent.Executors;
 
 public class DatabricksDatabaseMetaData implements DatabaseMetaData {
 
@@ -40,9 +37,6 @@ public class DatabricksDatabaseMetaData implements DatabaseMetaData {
   public static final String SYSTEM_FUNCTIONS = "DATABASE,IFNULL,USER";
   public static final String TIME_DATE_FUNCTIONS =
       "CURDATE,CURRENT_DATE,CURRENT_TIME,CURRENT_TIMESTAMP,CURTIME,DAYNAME,DAYOFMONTH,DAYOFWEEK,DAYOFYEAR,HOUR,MINUTE,MONTH,MONTHNAME,NOW,QUARTER,SECOND,TIMESTAMPADD,TIMESTAMPDIFF,WEEK,YEAR";
-  private static final Object THREAD_POOL_LOCK = new Object();
-  private static ExecutorService schemasThreadPool = null;
-  private static final int DEFAULT_MAX_THREADS = 10;
   private final IDatabricksConnectionInternal connection;
   private final IDatabricksSession session;
   private final MetadataResultSetBuilder metadataResultSetBuilder;
@@ -1505,48 +1499,6 @@ public ResultSet getSchemas(String catalog, String schemaPattern) throws SQLExce
         catalog, schemaPattern);
     throwExceptionIfConnectionIsClosed();
 
-    if (session.getConnectionContext().getClientType() == DatabricksClientType.SEA
-        && (catalog == null || catalog.equals("*") || catalog.equals("%"))) {
-      // Fetch catalogs from the metadata client
-      List<String> catalogList = new ArrayList<>();
-      try (ResultSet catalogs = getCatalogs()) {
-        while (catalogs.next()) {
-          String c = catalogs.getString(1);
-          if (c != null && !c.isEmpty()) {
-            catalogList.add(c);
-          }
-        }
-      }
-
-      // Process catalogs in parallel, gathering schema information
-      List<List<Object>> schemaRows =
-          JdbcThreadUtils.parallelFlatMap(
-              catalogList,
-              session.getConnectionContext(),
-              DEFAULT_MAX_THREADS, // Not significant since the executor is provided as a parameter
-              90, // 90 seconds timeout
-              c -> {
-                List<List<Object>> rows = new ArrayList<>();
-                try (ResultSet catalogSchemas =
-                    session.getDatabricksMetadataClient().listSchemas(session, c, schemaPattern)) {
-                  while (catalogSchemas.next()) {
-                    List<Object> schemaRow = new ArrayList<>();
-                    schemaRow.add(catalogSchemas.getString(1)); // TABLE_SCHEM
-                    schemaRow.add(catalogSchemas.getString(2)); // TABLE_CATALOG
-                    rows.add(schemaRow);
-                  }
-                } catch (SQLException e) {
-                  LOGGER.warn("Error fetching schemas for catalog %s %s", c, e.getMessage());
-                }
-                return rows;
-              },
-              getOrCreateSchemasThreadPool());
-
-      // Convert combined data into a result set
-      return metadataResultSetBuilder.getResultSetWithGivenRowsAndColumns(
-          SCHEMA_COLUMNS, schemaRows, METADATA_STATEMENT_ID, CommandName.LIST_SCHEMAS);
-    }
-
     return session.getDatabricksMetadataClient().listSchemas(session, catalog, schemaPattern);
   }
 
@@ -1659,23 +1611,6 @@ public boolean isWrapperFor(Class<?> iface) throws SQLException {
     return iface != null && iface.isAssignableFrom(this.getClass());
   }
 
-  private static ExecutorService getOrCreateSchemasThreadPool() {
-    synchronized (THREAD_POOL_LOCK) {
-      if (schemasThreadPool == null || schemasThreadPool.isShutdown()) {
-        // Could read max threads from a configuration property
-        schemasThreadPool =
-            Executors.newFixedThreadPool(
-                DEFAULT_MAX_THREADS,
-                r -> {
-                  Thread t = new Thread(r, "jdbc-schemas-fetcher");
-                  t.setDaemon(true);
-                  return t;
-                });
-      }
-      return schemasThreadPool;
-    }
-  }
-
   private void throwExceptionIfConnectionIsClosed() throws SQLException {
     LOGGER.debug("private void throwExceptionIfConnectionIsClosed()");
     if (!connection.getSession().isOpen()) {

src/main/java/com/databricks/jdbc/common/MetadataResultConstants.java

@@ -13,9 +13,9 @@ public class MetadataResultConstants {
   public static final String[] DEFAULT_TABLE_TYPES = {"TABLE", "VIEW", "SYSTEM TABLE"};
   public static final ResultColumn CATALOG_COLUMN =
       new ResultColumn("TABLE_CAT", "catalogName", Types.VARCHAR);
-  private static final ResultColumn CATALOG_FULL_COLUMN =
+  public static final ResultColumn CATALOG_FULL_COLUMN =
       new ResultColumn("TABLE_CATALOG", "catalogName", Types.VARCHAR);
-  public static final ResultColumn CATALOG_COLUMN_FOR_GET_CATALOGS =
+  public static final ResultColumn CATALOG_RESULT_COLUMN =
       new ResultColumn("TABLE_CAT", "catalog", Types.VARCHAR);
   public static final ResultColumn TYPE_CATALOG_COLUMN =
       new ResultColumn("TYPE_CAT", "TYPE_CATALOG_COLUMN", Types.VARCHAR);
@@ -216,7 +216,7 @@ public class MetadataResultConstants {
           IS_AUTO_INCREMENT_COLUMN,
           IS_GENERATED_COLUMN);
 
-  public static List<ResultColumn> CATALOG_COLUMNS = List.of(CATALOG_COLUMN_FOR_GET_CATALOGS);
+  public static List<ResultColumn> CATALOG_COLUMNS = List.of(CATALOG_RESULT_COLUMN);
 
   public static List<ResultColumn> SCHEMA_COLUMNS =
       List.of(SCHEMA_COLUMN_FOR_GET_SCHEMA, CATALOG_FULL_COLUMN);
@@ -491,9 +491,7 @@ public class MetadataResultConstants {
                   MetadataResultConstants.TYPE_NAME_COLUMN,
                   MetadataResultConstants.DATA_TYPE_COLUMN,
                   MetadataResultConstants.PRECISION_COLUMN));
-          put(
-              CommandName.LIST_CATALOGS,
-              List.of(MetadataResultConstants.CATALOG_COLUMN_FOR_GET_CATALOGS));
+          put(CommandName.LIST_CATALOGS, List.of(MetadataResultConstants.CATALOG_RESULT_COLUMN));
           put(
               CommandName.LIST_TABLES,
               List.of(MetadataResultConstants.TABLE_NAME_COLUMN, TABLE_TYPE_COLUMN));

src/main/java/com/databricks/jdbc/common/util/LoggingUtil.java

@@ -19,8 +19,8 @@ public static void setupLogger(String logDir, int logFileSizeMB, int logFileCoun
       throws IOException {
     if (LOGGER instanceof JulLogger && System.getProperty(JAVA_UTIL_LOGGING_CONFIG_FILE) == null) {
       // Only configure JUL logger if it's not already configured via external properties file
-      LOGGER.info("Setting up JUL logger");
       JulLogger.initLogger(toJulLevel(level), logDir, logFileSizeMB * 1024 * 1024, logFileCount);
+      LOGGER.info("Setting up JUL logger");
     }
   }
 

src/main/java/com/databricks/jdbc/common/util/WildcardUtil.java

@@ -21,6 +21,10 @@ public static boolean isNullOrEmpty(String s) {
     return s == null || s.trim().isEmpty();
   }
 
+  public static boolean isNullOrWildcard(String s) {
+    return s == null || isWildcard(s) || s.equals("%");
+  }
+
   /**
    * This function checks if the input string is a wildcard string
    *

src/main/java/com/databricks/jdbc/dbclient/impl/common/ClientConfigurator.java

@@ -26,6 +26,7 @@
 import java.net.ServerSocket;
 import java.nio.file.Path;
 import java.nio.file.Paths;
+import java.time.Duration;
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.List;
@@ -196,6 +197,8 @@ public void setupU2MConfig() throws DatabricksParsingException {
         .setAuthType(DatabricksJdbcConstants.U2M_AUTH_TYPE)
         .setHost(host)
         .setClientId(clientId)
+        .setOAuthBrowserAuthTimeout(
+            Duration.ofHours(1)) // TODO : add a browser timeout connection config
         .setClientSecret(connectionContext.getClientSecret())
         .setOAuthRedirectUrl(redirectUrl);
 

src/main/java/com/databricks/jdbc/dbclient/impl/common/CommandConstants.java

@@ -11,14 +11,17 @@ public class CommandConstants {
   public static final String IN_CATALOG_SQL = " IN CATALOG %s";
   public static final String IN_ABSOLUTE_SCHEMA_SQL = " IN SCHEMA %s";
   public static final String IN_ABSOLUTE_TABLE_SQL = " IN TABLE %s";
-  public static final String SHOW_SCHEMA_IN_CATALOG_SQL = "SHOW SCHEMAS IN %s";
+  public static final String IN_ALL_CATALOGS_SQL = " IN ALL CATALOGS";
+  public static final String SHOW_SCHEMAS_IN_CATALOG_SQL = "SHOW SCHEMAS IN %s";
   public static final String LIKE_SQL = " LIKE '%s'";
   public static final String SCHEMA_LIKE_SQL = " SCHEMA" + LIKE_SQL;
   public static final String TABLE_LIKE_SQL = " TABLE" + LIKE_SQL;
   public static final String SHOW_TABLES_SQL = "SHOW TABLES" + IN_CATALOG_SQL;
-  public static final String SHOW_TABLES_IN_ALL_CATALOGS_SQL = "SHOW TABLES IN ALL CATALOGS";
+  public static final String SHOW_TABLES_IN_ALL_CATALOGS_SQL = "SHOW TABLES" + IN_ALL_CATALOGS_SQL;
   public static final String SHOW_COLUMNS_SQL = "SHOW COLUMNS" + IN_CATALOG_SQL;
   public static final String SHOW_FUNCTIONS_SQL = "SHOW FUNCTIONS" + IN_CATALOG_SQL;
+  public static final String SHOW_SCHEMAS_IN_ALL_CATALOGS_SQL =
+      "SHOW SCHEMAS" + IN_ALL_CATALOGS_SQL;
   public static final String SHOW_PRIMARY_KEYS_SQL =
       "SHOW KEYS" + IN_CATALOG_SQL + IN_ABSOLUTE_SCHEMA_SQL + IN_ABSOLUTE_TABLE_SQL;
   public static final String SHOW_FOREIGN_KEYS_SQL =

src/main/java/com/databricks/jdbc/dbclient/impl/common/MetadataResultSetBuilder.java

@@ -68,7 +68,9 @@ public DatabricksResultSet getCatalogsResult(DatabricksResultSet resultSet) thro
 
   public DatabricksResultSet getSchemasResult(DatabricksResultSet resultSet, String catalog)
       throws SQLException {
-    List<List<Object>> rows = getRowsForSchemas(resultSet, SCHEMA_COLUMNS, catalog);
+    List<List<Object>> rows =
+        getRowsForSchemas(
+            resultSet, SCHEMA_COLUMNS, catalog, new SchemasDatabricksResultSetAdapter());
     return buildResultSet(
         SCHEMA_COLUMNS,
         rows,
@@ -562,19 +564,39 @@ private List<List<Object>> getRowsForFunctions(
   }
 
   private List<List<Object>> getRowsForSchemas(
-      DatabricksResultSet resultSet, List<ResultColumn> columns, String catalog)
+      DatabricksResultSet resultSet,
+      List<ResultColumn> columns,
+      String catalog,
+      IDatabricksResultSetAdapter adapter)
       throws SQLException {
     List<List<Object>> rows = new ArrayList<>();
     while (resultSet.next()) {
+      // Check if this row should be included based on the adapter's filter
+      if (!adapter.includeRow(resultSet, columns)) {
+        continue;
+      }
+
       List<Object> row = new ArrayList<>();
       for (ResultColumn column : columns) {
-        if (column.getColumnName().equals("TABLE_CATALOG")) {
-          row.add(catalog);
-          continue;
+        // Map the expected column on client to column in the result set using the adapter
+        ResultColumn mappedColumn = adapter.mapColumn(column);
+
+        if (mappedColumn
+            .getResultSetColumnName()
+            .equals(CATALOG_RESULT_COLUMN.getResultSetColumnName())) {
+          try {
+            resultSet.findColumn(mappedColumn.getResultSetColumnName());
+          } catch (SQLException e) {
+            // Result set does not have a catalog column
+            // Manually add the catalog and move to next column
+            row.add(catalog);
+            continue;
+          }
         }
+
         Object object;
         try {
-          object = resultSet.getObject(column.getResultSetColumnName());
+          object = resultSet.getObject(mappedColumn.getResultSetColumnName());
           if (object == null) {
             object = NULL_STRING;
           }