Support ALL CATALOGS in GetTables for SQL Execution API

NEXT_CHANGELOG.md

@@ -3,6 +3,7 @@
 ## [Unreleased]
 
 ### Added
+- Support for fetching tables and views across all catalogs using SHOW TABLES FROM/IN ALL CATALOGS in the SQL Exec API.
 - Support for Token Exchange in OAuth flows where in third party tokens are exchanged for InHouse tokens.
 
 ### Updated

src/main/java/com/databricks/jdbc/dbclient/impl/common/CommandConstants.java

@@ -4,7 +4,6 @@ public class CommandConstants {
   public static final String METADATA_STATEMENT_ID = "metadata-statement";
   public static final String GET_TABLES_STATEMENT_ID = "gettables-metadata";
   public static final String GET_CATALOGS_STATEMENT_ID = "getcatalogs-metadata";
-  public static final String GET_TYPE_INFO_STATEMENT_ID = "typeinfo-metadata";
   public static final String GET_TABLE_TYPE_STATEMENT_ID = "gettabletype-metadata";
   public static final String GET_FUNCTIONS_STATEMENT_ID = "getfunctions-metadata";
   public static final String SHOW_CATALOGS_SQL = "SHOW CATALOGS";
@@ -17,6 +16,7 @@ public class CommandConstants {
   public static final String SCHEMA_LIKE_SQL = " SCHEMA" + LIKE_SQL;
   public static final String TABLE_LIKE_SQL = " TABLE" + LIKE_SQL;
   public static final String SHOW_TABLES_SQL = "SHOW TABLES" + IN_CATALOG_SQL;
+  public static final String SHOW_TABLES_IN_ALL_CATALOGS_SQL = "SHOW TABLES IN ALL CATALOGS";
   public static final String SHOW_COLUMNS_SQL = "SHOW COLUMNS" + IN_CATALOG_SQL;
   public static final String SHOW_FUNCTIONS_SQL = "SHOW FUNCTIONS" + IN_CATALOG_SQL;
   public static final String SHOW_PRIMARY_KEYS_SQL =

src/main/java/com/databricks/jdbc/dbclient/impl/sqlexec/CommandBuilder.java

@@ -94,8 +94,13 @@ private String fetchTablesSQL() throws SQLException {
             "Building command for fetching tables. Catalog %s, SchemaPattern %s, TablePattern %s and session context %s",
             catalogName, schemaPattern, tablePattern, sessionContext);
     LOGGER.debug(contextString);
-    throwErrorIfNull(Collections.singletonMap(CATALOG, catalogName), contextString);
-    String showTablesSQL = String.format(SHOW_TABLES_SQL, catalogName);
+    String showTablesSQL;
+    if (catalogName == null || catalogName.equals("*") || catalogName.equals("%")) {
+      // SHOW TABLES IN ALL CATALOGS
+      showTablesSQL = SHOW_TABLES_IN_ALL_CATALOGS_SQL;
+    } else {
+      showTablesSQL = String.format(SHOW_TABLES_SQL, catalogName);
+    }
     if (!WildcardUtil.isNullOrEmpty(schemaPattern)) {
       showTablesSQL += String.format(SCHEMA_LIKE_SQL, schemaPattern);
     }

src/main/java/com/databricks/jdbc/dbclient/impl/sqlexec/DatabricksMetadataSdkClient.java

@@ -2,6 +2,7 @@
 
 import static com.databricks.jdbc.common.MetadataResultConstants.DEFAULT_TABLE_TYPES;
 import static com.databricks.jdbc.common.MetadataResultConstants.PARSE_SYNTAX_ERROR_SQL_STATE;
+import static com.databricks.jdbc.dbclient.impl.common.CommandConstants.GET_TABLES_STATEMENT_ID;
 import static com.databricks.jdbc.dbclient.impl.common.CommandConstants.METADATA_STATEMENT_ID;
 import static com.databricks.jdbc.dbclient.impl.sqlexec.ResultConstants.TYPE_INFO_RESULT;
 
@@ -76,8 +77,24 @@ public DatabricksResultSet listTables(
     String SQL = commandBuilder.getSQLString(CommandName.LIST_TABLES);
     LOGGER.debug("SQL command to fetch tables: {}", SQL);
     LOGGER.debug(String.format("SQL command to fetch tables: {%s}", SQL));
-    return metadataResultSetBuilder.getTablesResult(
-        getResultSet(SQL, session), validatedTableTypes);
+    try {
+      return metadataResultSetBuilder.getTablesResult(
+          getResultSet(SQL, session), validatedTableTypes);
+    } catch (SQLException e) {
+      if (e.getSQLState().equals(PARSE_SYNTAX_ERROR_SQL_STATE)
+          && (catalog == null || catalog.equals("*") || catalog.equals("%"))) {
+        // Gracefully handles the case where an older DBSQL version doesn't support all catalogs in
+        // the SHOW TABLES command.
+        LOGGER.debug("SQL command failed with syntax error. Returning empty result set.");
+        return metadataResultSetBuilder.getResultSetWithGivenRowsAndColumns(
+            MetadataResultConstants.TABLE_COLUMNS,
+            new ArrayList<>(),
+            GET_TABLES_STATEMENT_ID,
+            com.databricks.jdbc.common.CommandName.LIST_TABLES);
+      } else {
+        throw e;
+      }
+    }
   }
 
   @Override

src/test/java/com/databricks/jdbc/dbclient/impl/sqlexec/CommandBuilderTest.java

@@ -0,0 +1,211 @@
+package com.databricks.jdbc.dbclient.impl.sqlexec;
+
+import static com.databricks.jdbc.dbclient.impl.common.CommandConstants.*;
+import static org.junit.jupiter.api.Assertions.*;
+import static org.mockito.Mockito.*;
+
+import com.databricks.jdbc.api.internal.IDatabricksSession;
+import com.databricks.jdbc.common.util.WildcardUtil;
+import com.databricks.jdbc.exception.DatabricksSQLFeatureNotSupportedException;
+import java.sql.SQLException;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.DisplayName;
+import org.junit.jupiter.api.Nested;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.extension.ExtendWith;
+import org.mockito.Mock;
+import org.mockito.junit.jupiter.MockitoExtension;
+
+@ExtendWith(MockitoExtension.class)
+class CommandBuilderTest {
+
+  @Mock private IDatabricksSession mockSession;
+
+  private static final String TEST_CATALOG = "test_catalog";
+  private static final String TEST_SCHEMA = "test_schema";
+  private static final String TEST_TABLE = "test_table";
+  private static final String TEST_SESSION_CONTEXT = "test_session_context";
+
+  @BeforeEach
+  void setUp() {
+    when(mockSession.toString()).thenReturn(TEST_SESSION_CONTEXT);
+  }
+
+  @Nested
+  @DisplayName("Tests for LIST_PRIMARY_KEYS command")
+  class ListPrimaryKeysTests {
+
+    @Test
+    @DisplayName("Should generate correct SQL for fetching primary keys")
+    void shouldGenerateCorrectSqlForPrimaryKeys() throws SQLException {
+      CommandBuilder builder =
+          new CommandBuilder(TEST_CATALOG, mockSession).setSchema(TEST_SCHEMA).setTable(TEST_TABLE);
+
+      String sql = builder.getSQLString(CommandName.LIST_PRIMARY_KEYS);
+
+      String expectedSql =
+          String.format(SHOW_PRIMARY_KEYS_SQL, TEST_CATALOG, TEST_SCHEMA, TEST_TABLE);
+      assertEquals(expectedSql, sql);
+    }
+
+    @Test
+    @DisplayName("Should throw SQLException when catalog is null for primary keys")
+    void shouldThrowExceptionWhenCatalogIsNullForPrimaryKeys() {
+      CommandBuilder builder =
+          new CommandBuilder(null, mockSession).setSchema(TEST_SCHEMA).setTable(TEST_TABLE);
+
+      assertThrows(SQLException.class, () -> builder.getSQLString(CommandName.LIST_PRIMARY_KEYS));
+    }
+
+    @Test
+    @DisplayName("Should throw SQLException when schema is null for primary keys")
+    void shouldThrowExceptionWhenSchemaIsNullForPrimaryKeys() {
+      CommandBuilder builder = new CommandBuilder(TEST_CATALOG, mockSession).setTable(TEST_TABLE);
+
+      assertThrows(SQLException.class, () -> builder.getSQLString(CommandName.LIST_PRIMARY_KEYS));
+    }
+
+    @Test
+    @DisplayName("Should throw SQLException when table is null for primary keys")
+    void shouldThrowExceptionWhenTableIsNullForPrimaryKeys() {
+      CommandBuilder builder = new CommandBuilder(TEST_CATALOG, mockSession).setSchema(TEST_SCHEMA);
+
+      assertThrows(SQLException.class, () -> builder.getSQLString(CommandName.LIST_PRIMARY_KEYS));
+    }
+  }
+
+  @Nested
+  @DisplayName("Tests for LIST_TABLES command")
+  class ListTablesTests {
+
+    @Test
+    @DisplayName("Should generate correct SQL for fetching tables with catalog")
+    void shouldGenerateCorrectSqlForTablesWithCatalog() throws SQLException {
+      CommandBuilder builder = new CommandBuilder(TEST_CATALOG, mockSession);
+
+      String sql = builder.getSQLString(CommandName.LIST_TABLES);
+
+      String expectedSql = String.format(SHOW_TABLES_SQL, TEST_CATALOG);
+      assertEquals(expectedSql, sql);
+    }
+
+    @Test
+    @DisplayName("Should generate correct SQL for fetching tables with catalog and schema pattern")
+    void shouldGenerateCorrectSqlForTablesWithCatalogAndSchemaPattern() throws SQLException {
+      String schemaPattern = "test_schema%";
+      String hiveSchemaPattern = WildcardUtil.jdbcPatternToHive(schemaPattern);
+
+      CommandBuilder builder =
+          new CommandBuilder(TEST_CATALOG, mockSession).setSchemaPattern(schemaPattern);
+
+      String sql = builder.getSQLString(CommandName.LIST_TABLES);
+
+      String expectedSql =
+          String.format(SHOW_TABLES_SQL.concat(SCHEMA_LIKE_SQL), TEST_CATALOG, hiveSchemaPattern);
+      assertEquals(expectedSql, sql);
+    }
+
+    @Test
+    @DisplayName(
+        "Should generate correct SQL for fetching tables with catalog, schema pattern, and table pattern")
+    void shouldGenerateCorrectSqlForTablesWithCatalogSchemaAndTablePattern() throws SQLException {
+      String schemaPattern = "test_schema%";
+      String tablePattern = "test_table%";
+      String hiveSchemaPattern = WildcardUtil.jdbcPatternToHive(schemaPattern);
+      String hiveTablePattern = WildcardUtil.jdbcPatternToHive(tablePattern);
+
+      CommandBuilder builder =
+          new CommandBuilder(TEST_CATALOG, mockSession)
+              .setSchemaPattern(schemaPattern)
+              .setTablePattern(tablePattern);
+
+      String sql = builder.getSQLString(CommandName.LIST_TABLES);
+
+      String expectedSql =
+          String.format(
+              SHOW_TABLES_SQL.concat(SCHEMA_LIKE_SQL).concat(LIKE_SQL),
+              TEST_CATALOG,
+              hiveSchemaPattern,
+              hiveTablePattern);
+      assertEquals(expectedSql, sql);
+    }
+
+    @Test
+    @DisplayName("Should generate correct SQL for fetching tables from all catalogs")
+    void shouldGenerateCorrectSqlForTablesFromAllCatalogs() throws SQLException {
+      CommandBuilder builder = new CommandBuilder(null, mockSession);
+
+      String sql = builder.getSQLString(CommandName.LIST_TABLES);
+
+      assertEquals(SHOW_TABLES_IN_ALL_CATALOGS_SQL, sql);
+    }
+
+    @Test
+    @DisplayName("Should generate correct SQL for fetching tables with wildcard catalog")
+    void shouldGenerateCorrectSqlForTablesWithWildcardCatalog() throws SQLException {
+      // Test with '*' wildcard
+      CommandBuilder builder1 = new CommandBuilder("*", mockSession);
+      String sql1 = builder1.getSQLString(CommandName.LIST_TABLES);
+      assertEquals(SHOW_TABLES_IN_ALL_CATALOGS_SQL, sql1);
+
+      // Test with '%' wildcard
+      CommandBuilder builder2 = new CommandBuilder("%", mockSession);
+      String sql2 = builder2.getSQLString(CommandName.LIST_TABLES);
+      assertEquals(SHOW_TABLES_IN_ALL_CATALOGS_SQL, sql2);
+    }
+  }
+
+  @Nested
+  @DisplayName("Tests for LIST_FOREIGN_KEYS command")
+  class ListForeignKeysTests {
+
+    @Test
+    @DisplayName("Should generate correct SQL for fetching foreign keys")
+    void shouldGenerateCorrectSqlForForeignKeys() throws SQLException {
+      CommandBuilder builder =
+          new CommandBuilder(TEST_CATALOG, mockSession).setSchema(TEST_SCHEMA).setTable(TEST_TABLE);
+
+      String sql = builder.getSQLString(CommandName.LIST_FOREIGN_KEYS);
+
+      String expectedSql =
+          String.format(SHOW_FOREIGN_KEYS_SQL, TEST_CATALOG, TEST_SCHEMA, TEST_TABLE);
+      assertEquals(expectedSql, sql);
+    }
+
+    @Test
+    @DisplayName("Should throw SQLException when catalog is null for foreign keys")
+    void shouldThrowExceptionWhenCatalogIsNullForForeignKeys() {
+      CommandBuilder builder =
+          new CommandBuilder(null, mockSession).setSchema(TEST_SCHEMA).setTable(TEST_TABLE);
+
+      assertThrows(SQLException.class, () -> builder.getSQLString(CommandName.LIST_FOREIGN_KEYS));
+    }
+
+    @Test
+    @DisplayName("Should throw SQLException when schema is null for foreign keys")
+    void shouldThrowExceptionWhenSchemaIsNullForForeignKeys() {
+      CommandBuilder builder = new CommandBuilder(TEST_CATALOG, mockSession).setTable(TEST_TABLE);
+
+      assertThrows(SQLException.class, () -> builder.getSQLString(CommandName.LIST_FOREIGN_KEYS));
+    }
+
+    @Test
+    @DisplayName("Should throw SQLException when table is null for foreign keys")
+    void shouldThrowExceptionWhenTableIsNullForForeignKeys() {
+      CommandBuilder builder = new CommandBuilder(TEST_CATALOG, mockSession).setSchema(TEST_SCHEMA);
+
+      assertThrows(SQLException.class, () -> builder.getSQLString(CommandName.LIST_FOREIGN_KEYS));
+    }
+  }
+
+  @Test
+  @DisplayName("Should throw exception for unsupported command")
+  void shouldThrowExceptionForUnsupportedCommand() {
+    CommandBuilder builder = new CommandBuilder(TEST_CATALOG, mockSession);
+
+    CommandName mockCommand = mock(CommandName.class);
+
+    assertThrows(
+        DatabricksSQLFeatureNotSupportedException.class, () -> builder.getSQLString(mockCommand));
+  }
+}

src/test/java/com/databricks/jdbc/dbclient/impl/sqlexec/DatabricksMetadataSdkClientTest.java

@@ -748,9 +748,6 @@ void testThrowsErrorResultInCaseOfNullCatalog() {
     assertThrows(
         DatabricksValidationException.class,
         () -> metadataClient.listColumns(session, null, TEST_SCHEMA, TEST_TABLE, TEST_COLUMN));
-    assertThrows(
-        DatabricksValidationException.class,
-        () -> metadataClient.listTables(session, null, TEST_SCHEMA, TEST_TABLE, null));
     assertThrows(
         DatabricksValidationException.class,
         () -> metadataClient.listSchemas(session, null, TEST_SCHEMA));
@@ -767,4 +764,67 @@ void testListTypeInfo() {
     DatabricksMetadataSdkClient metadataClient = new DatabricksMetadataSdkClient(mockClient);
     assertNotNull(metadataClient.listTypeInfo(session));
   }
+
+  @Test
+  void testListTablesAllCatalogs() throws SQLException {
+    when(session.getComputeResource()).thenReturn(WAREHOUSE_COMPUTE);
+    DatabricksMetadataSdkClient metadataClient = new DatabricksMetadataSdkClient(mockClient);
+    when(mockClient.executeStatement(
+            "SHOW TABLES IN ALL CATALOGS SCHEMA LIKE 'testSchema' LIKE 'testTable'",
+            WAREHOUSE_COMPUTE,
+            new HashMap<>(),
+            StatementType.METADATA,
+            session,
+            null))
+        .thenReturn(mockedResultSet);
+    when(mockedResultSet.next()).thenReturn(true, false);
+    for (ResultColumn resultColumn : TABLE_COLUMNS) {
+      when(mockedResultSet.getObject(resultColumn.getResultSetColumnName()))
+          .thenReturn(TEST_COLUMN);
+    }
+    when(mockedResultSet.getObject("tableType")).thenReturn("TABLE");
+    doReturn(10).when(mockedMetaData).getColumnCount();
+    doReturn(CATALOG_COLUMN.getResultSetColumnName()).when(mockedMetaData).getColumnName(1);
+    doReturn(SCHEMA_COLUMN.getResultSetColumnName()).when(mockedMetaData).getColumnName(2);
+    doReturn(TABLE_NAME_COLUMN.getResultSetColumnName()).when(mockedMetaData).getColumnName(3);
+    doReturn(TABLE_TYPE_COLUMN.getResultSetColumnName()).when(mockedMetaData).getColumnName(4);
+    doReturn(REMARKS_COLUMN.getResultSetColumnName()).when(mockedMetaData).getColumnName(5);
+    doReturn(TYPE_CATALOG_COLUMN.getResultSetColumnName()).when(mockedMetaData).getColumnName(6);
+    doReturn(TYPE_SCHEMA_COLUMN.getResultSetColumnName()).when(mockedMetaData).getColumnName(7);
+    doReturn(TYPE_NAME_COLUMN.getResultSetColumnName()).when(mockedMetaData).getColumnName(8);
+    doReturn(SELF_REFERENCING_COLUMN_NAME.getResultSetColumnName())
+        .when(mockedMetaData)
+        .getColumnName(9);
+    doReturn(REF_GENERATION_COLUMN.getResultSetColumnName()).when(mockedMetaData).getColumnName(10);
+    when(mockedResultSet.getMetaData()).thenReturn(mockedMetaData);
+    DatabricksResultSet actualResult =
+        metadataClient.listTables(session, null, TEST_SCHEMA, TEST_TABLE, null);
+    assertEquals(actualResult.getStatementStatus().getState(), StatementState.SUCCEEDED);
+    assertEquals(actualResult.getStatementId(), GET_TABLES_STATEMENT_ID);
+    assertEquals(((DatabricksResultSetMetaData) actualResult.getMetaData()).getTotalRows(), 1);
+  }
+
+  @Test
+  void testGetTablesAllCatalogs_throwsParseSyntaxError() throws Exception {
+    DatabricksSQLException exception =
+        new DatabricksSQLException("syntax error at or near \"IN\"", PARSE_SYNTAX_ERROR_SQL_STATE);
+    when(session.getComputeResource()).thenReturn(WAREHOUSE_COMPUTE);
+    DatabricksMetadataSdkClient metadataClient = new DatabricksMetadataSdkClient(mockClient);
+    when(mockClient.executeStatement(
+            "SHOW TABLES IN ALL CATALOGS SCHEMA LIKE 'testSchema' LIKE 'testTable'",
+            WAREHOUSE_COMPUTE,
+            new HashMap<>(),
+            StatementType.METADATA,
+            session,
+            null))
+        .thenThrow(exception);
+    try (DatabricksResultSet actualResult =
+        metadataClient.listTables(session, null, TEST_SCHEMA, TEST_TABLE, null)) {
+      assertEquals(StatementState.SUCCEEDED, actualResult.getStatementStatus().getState());
+      assertEquals(GET_TABLES_STATEMENT_ID, actualResult.getStatementId());
+      assertEquals(10, actualResult.getMetaData().getColumnCount());
+      // Parse syntax error is handled gracefully to return empty result set
+      assertEquals(0, ((DatabricksResultSetMetaData) actualResult.getMetaData()).getTotalRows());
+    }
+  }
 }