Skip to content

Commit 016dc1f

Browse files
jayantsing-dbjayantsing-db
authored
Support ALL CATALOGS in GetTables for SQL Execution API (#815)
DBR now supports retrieving tables and views from all catalogs using the SQL command SHOW TABLES FROM/IN ALL CATALOGS .... This pull request updates the SQL Exec API metadata client to leverage this command when the catalog parameter is set to null in the DatabaseMetaData#getTables method.
1 parent 149d867 commit 016dc1f

6 files changed

Lines changed: 302 additions & 8 deletions

File tree

‎NEXT_CHANGELOG.md‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,7 @@
33
## [Unreleased]
44

55
### Added
6+
- Support for fetching tables and views across all catalogs using SHOW TABLES FROM/IN ALL CATALOGS in the SQL Exec API.
67
- Support for Token Exchange in OAuth flows where in third party tokens are exchanged for InHouse tokens.
78

89
### Updated

‎src/main/java/com/databricks/jdbc/dbclient/impl/common/CommandConstants.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,6 @@ public class CommandConstants {
44
public static final String METADATA_STATEMENT_ID = "metadata-statement";
55
public static final String GET_TABLES_STATEMENT_ID = "gettables-metadata";
66
public static final String GET_CATALOGS_STATEMENT_ID = "getcatalogs-metadata";
7-
public static final String GET_TYPE_INFO_STATEMENT_ID = "typeinfo-metadata";
87
public static final String GET_TABLE_TYPE_STATEMENT_ID = "gettabletype-metadata";
98
public static final String GET_FUNCTIONS_STATEMENT_ID = "getfunctions-metadata";
109
public static final String SHOW_CATALOGS_SQL = "SHOW CATALOGS";
@@ -17,6 +16,7 @@ public class CommandConstants {
1716
public static final String SCHEMA_LIKE_SQL = " SCHEMA" + LIKE_SQL;
1817
public static final String TABLE_LIKE_SQL = " TABLE" + LIKE_SQL;
1918
public static final String SHOW_TABLES_SQL = "SHOW TABLES" + IN_CATALOG_SQL;
19+
public static final String SHOW_TABLES_IN_ALL_CATALOGS_SQL = "SHOW TABLES IN ALL CATALOGS";
2020
public static final String SHOW_COLUMNS_SQL = "SHOW COLUMNS" + IN_CATALOG_SQL;
2121
public static final String SHOW_FUNCTIONS_SQL = "SHOW FUNCTIONS" + IN_CATALOG_SQL;
2222
public static final String SHOW_PRIMARY_KEYS_SQL =

‎src/main/java/com/databricks/jdbc/dbclient/impl/sqlexec/CommandBuilder.java‎

Lines changed: 7 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -94,8 +94,13 @@ private String fetchTablesSQL() throws SQLException {
9494
"Building command for fetching tables. Catalog %s, SchemaPattern %s, TablePattern %s and session context %s",
9595
catalogName, schemaPattern, tablePattern, sessionContext);
9696
LOGGER.debug(contextString);
97-
throwErrorIfNull(Collections.singletonMap(CATALOG, catalogName), contextString);
98-
String showTablesSQL = String.format(SHOW_TABLES_SQL, catalogName);
97+
String showTablesSQL;
98+
if (catalogName == null || catalogName.equals("*") || catalogName.equals("%")) {
99+
// SHOW TABLES IN ALL CATALOGS
100+
showTablesSQL = SHOW_TABLES_IN_ALL_CATALOGS_SQL;
101+
} else {
102+
showTablesSQL = String.format(SHOW_TABLES_SQL, catalogName);
103+
}
99104
if (!WildcardUtil.isNullOrEmpty(schemaPattern)) {
100105
showTablesSQL += String.format(SCHEMA_LIKE_SQL, schemaPattern);
101106
}

‎src/main/java/com/databricks/jdbc/dbclient/impl/sqlexec/DatabricksMetadataSdkClient.java‎

Lines changed: 19 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,7 @@
22

33
import static com.databricks.jdbc.common.MetadataResultConstants.DEFAULT_TABLE_TYPES;
44
import static com.databricks.jdbc.common.MetadataResultConstants.PARSE_SYNTAX_ERROR_SQL_STATE;
5+
import static com.databricks.jdbc.dbclient.impl.common.CommandConstants.GET_TABLES_STATEMENT_ID;
56
import static com.databricks.jdbc.dbclient.impl.common.CommandConstants.METADATA_STATEMENT_ID;
67
import static com.databricks.jdbc.dbclient.impl.sqlexec.ResultConstants.TYPE_INFO_RESULT;
78

@@ -76,8 +77,24 @@ public DatabricksResultSet listTables(
7677
String SQL = commandBuilder.getSQLString(CommandName.LIST_TABLES);
7778
LOGGER.debug("SQL command to fetch tables: {}", SQL);
7879
LOGGER.debug(String.format("SQL command to fetch tables: {%s}", SQL));
79-
return metadataResultSetBuilder.getTablesResult(
80-
getResultSet(SQL, session), validatedTableTypes);
80+
try {
81+
return metadataResultSetBuilder.getTablesResult(
82+
getResultSet(SQL, session), validatedTableTypes);
83+
} catch (SQLException e) {
84+
if (e.getSQLState().equals(PARSE_SYNTAX_ERROR_SQL_STATE)
85+
&& (catalog == null || catalog.equals("*") || catalog.equals("%"))) {
86+
// Gracefully handles the case where an older DBSQL version doesn't support all catalogs in
87+
// the SHOW TABLES command.
88+
LOGGER.debug("SQL command failed with syntax error. Returning empty result set.");
89+
return metadataResultSetBuilder.getResultSetWithGivenRowsAndColumns(
90+
MetadataResultConstants.TABLE_COLUMNS,
91+
new ArrayList<>(),
92+
GET_TABLES_STATEMENT_ID,
93+
com.databricks.jdbc.common.CommandName.LIST_TABLES);
94+
} else {
95+
throw e;
96+
}
97+
}
8198
}
8299

83100
@Override

‎src/test/java/com/databricks/jdbc/dbclient/impl/sqlexec/CommandBuilderTest.java‎

Lines changed: 211 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,211 @@
1+
package com.databricks.jdbc.dbclient.impl.sqlexec;
2+
3+
import static com.databricks.jdbc.dbclient.impl.common.CommandConstants.*;
4+
import static org.junit.jupiter.api.Assertions.*;
5+
import static org.mockito.Mockito.*;
6+
7+
import com.databricks.jdbc.api.internal.IDatabricksSession;
8+
import com.databricks.jdbc.common.util.WildcardUtil;
9+
import com.databricks.jdbc.exception.DatabricksSQLFeatureNotSupportedException;
10+
import java.sql.SQLException;
11+
import org.junit.jupiter.api.BeforeEach;
12+
import org.junit.jupiter.api.DisplayName;
13+
import org.junit.jupiter.api.Nested;
14+
import org.junit.jupiter.api.Test;
15+
import org.junit.jupiter.api.extension.ExtendWith;
16+
import org.mockito.Mock;
17+
import org.mockito.junit.jupiter.MockitoExtension;
18+
19+
@ExtendWith(MockitoExtension.class)
20+
class CommandBuilderTest {
21+
22+
@Mock private IDatabricksSession mockSession;
23+
24+
private static final String TEST_CATALOG = "test_catalog";
25+
private static final String TEST_SCHEMA = "test_schema";
26+
private static final String TEST_TABLE = "test_table";
27+
private static final String TEST_SESSION_CONTEXT = "test_session_context";
28+
29+
@BeforeEach
30+
void setUp() {
31+
when(mockSession.toString()).thenReturn(TEST_SESSION_CONTEXT);
32+
}
33+
34+
@Nested
35+
@DisplayName("Tests for LIST_PRIMARY_KEYS command")
36+
class ListPrimaryKeysTests {
37+
38+
@Test
39+
@DisplayName("Should generate correct SQL for fetching primary keys")
40+
void shouldGenerateCorrectSqlForPrimaryKeys() throws SQLException {
41+
CommandBuilder builder =
42+
new CommandBuilder(TEST_CATALOG, mockSession).setSchema(TEST_SCHEMA).setTable(TEST_TABLE);
43+
44+
String sql = builder.getSQLString(CommandName.LIST_PRIMARY_KEYS);
45+
46+
String expectedSql =
47+
String.format(SHOW_PRIMARY_KEYS_SQL, TEST_CATALOG, TEST_SCHEMA, TEST_TABLE);
48+
assertEquals(expectedSql, sql);
49+
}
50+
51+
@Test
52+
@DisplayName("Should throw SQLException when catalog is null for primary keys")
53+
void shouldThrowExceptionWhenCatalogIsNullForPrimaryKeys() {
54+
CommandBuilder builder =
55+
new CommandBuilder(null, mockSession).setSchema(TEST_SCHEMA).setTable(TEST_TABLE);
56+
57+
assertThrows(SQLException.class, () -> builder.getSQLString(CommandName.LIST_PRIMARY_KEYS));
58+
}
59+
60+
@Test
61+
@DisplayName("Should throw SQLException when schema is null for primary keys")
62+
void shouldThrowExceptionWhenSchemaIsNullForPrimaryKeys() {
63+
CommandBuilder builder = new CommandBuilder(TEST_CATALOG, mockSession).setTable(TEST_TABLE);
64+
65+
assertThrows(SQLException.class, () -> builder.getSQLString(CommandName.LIST_PRIMARY_KEYS));
66+
}
67+
68+
@Test
69+
@DisplayName("Should throw SQLException when table is null for primary keys")
70+
void shouldThrowExceptionWhenTableIsNullForPrimaryKeys() {
71+
CommandBuilder builder = new CommandBuilder(TEST_CATALOG, mockSession).setSchema(TEST_SCHEMA);
72+
73+
assertThrows(SQLException.class, () -> builder.getSQLString(CommandName.LIST_PRIMARY_KEYS));
74+
}
75+
}
76+
77+
@Nested
78+
@DisplayName("Tests for LIST_TABLES command")
79+
class ListTablesTests {
80+
81+
@Test
82+
@DisplayName("Should generate correct SQL for fetching tables with catalog")
83+
void shouldGenerateCorrectSqlForTablesWithCatalog() throws SQLException {
84+
CommandBuilder builder = new CommandBuilder(TEST_CATALOG, mockSession);
85+
86+
String sql = builder.getSQLString(CommandName.LIST_TABLES);
87+
88+
String expectedSql = String.format(SHOW_TABLES_SQL, TEST_CATALOG);
89+
assertEquals(expectedSql, sql);
90+
}
91+
92+
@Test
93+
@DisplayName("Should generate correct SQL for fetching tables with catalog and schema pattern")
94+
void shouldGenerateCorrectSqlForTablesWithCatalogAndSchemaPattern() throws SQLException {
95+
String schemaPattern = "test_schema%";
96+
String hiveSchemaPattern = WildcardUtil.jdbcPatternToHive(schemaPattern);
97+
98+
CommandBuilder builder =
99+
new CommandBuilder(TEST_CATALOG, mockSession).setSchemaPattern(schemaPattern);
100+
101+
String sql = builder.getSQLString(CommandName.LIST_TABLES);
102+
103+
String expectedSql =
104+
String.format(SHOW_TABLES_SQL.concat(SCHEMA_LIKE_SQL), TEST_CATALOG, hiveSchemaPattern);
105+
assertEquals(expectedSql, sql);
106+
}
107+
108+
@Test
109+
@DisplayName(
110+
"Should generate correct SQL for fetching tables with catalog, schema pattern, and table pattern")
111+
void shouldGenerateCorrectSqlForTablesWithCatalogSchemaAndTablePattern() throws SQLException {
112+
String schemaPattern = "test_schema%";
113+
String tablePattern = "test_table%";
114+
String hiveSchemaPattern = WildcardUtil.jdbcPatternToHive(schemaPattern);
115+
String hiveTablePattern = WildcardUtil.jdbcPatternToHive(tablePattern);
116+
117+
CommandBuilder builder =
118+
new CommandBuilder(TEST_CATALOG, mockSession)
119+
.setSchemaPattern(schemaPattern)
120+
.setTablePattern(tablePattern);
121+
122+
String sql = builder.getSQLString(CommandName.LIST_TABLES);
123+
124+
String expectedSql =
125+
String.format(
126+
SHOW_TABLES_SQL.concat(SCHEMA_LIKE_SQL).concat(LIKE_SQL),
127+
TEST_CATALOG,
128+
hiveSchemaPattern,
129+
hiveTablePattern);
130+
assertEquals(expectedSql, sql);
131+
}
132+
133+
@Test
134+
@DisplayName("Should generate correct SQL for fetching tables from all catalogs")
135+
void shouldGenerateCorrectSqlForTablesFromAllCatalogs() throws SQLException {
136+
CommandBuilder builder = new CommandBuilder(null, mockSession);
137+
138+
String sql = builder.getSQLString(CommandName.LIST_TABLES);
139+
140+
assertEquals(SHOW_TABLES_IN_ALL_CATALOGS_SQL, sql);
141+
}
142+
143+
@Test
144+
@DisplayName("Should generate correct SQL for fetching tables with wildcard catalog")
145+
void shouldGenerateCorrectSqlForTablesWithWildcardCatalog() throws SQLException {
146+
// Test with '*' wildcard
147+
CommandBuilder builder1 = new CommandBuilder("*", mockSession);
148+
String sql1 = builder1.getSQLString(CommandName.LIST_TABLES);
149+
assertEquals(SHOW_TABLES_IN_ALL_CATALOGS_SQL, sql1);
150+
151+
// Test with '%' wildcard
152+
CommandBuilder builder2 = new CommandBuilder("%", mockSession);
153+
String sql2 = builder2.getSQLString(CommandName.LIST_TABLES);
154+
assertEquals(SHOW_TABLES_IN_ALL_CATALOGS_SQL, sql2);
155+
}
156+
}
157+
158+
@Nested
159+
@DisplayName("Tests for LIST_FOREIGN_KEYS command")
160+
class ListForeignKeysTests {
161+
162+
@Test
163+
@DisplayName("Should generate correct SQL for fetching foreign keys")
164+
void shouldGenerateCorrectSqlForForeignKeys() throws SQLException {
165+
CommandBuilder builder =
166+
new CommandBuilder(TEST_CATALOG, mockSession).setSchema(TEST_SCHEMA).setTable(TEST_TABLE);
167+
168+
String sql = builder.getSQLString(CommandName.LIST_FOREIGN_KEYS);
169+
170+
String expectedSql =
171+
String.format(SHOW_FOREIGN_KEYS_SQL, TEST_CATALOG, TEST_SCHEMA, TEST_TABLE);
172+
assertEquals(expectedSql, sql);
173+
}
174+
175+
@Test
176+
@DisplayName("Should throw SQLException when catalog is null for foreign keys")
177+
void shouldThrowExceptionWhenCatalogIsNullForForeignKeys() {
178+
CommandBuilder builder =
179+
new CommandBuilder(null, mockSession).setSchema(TEST_SCHEMA).setTable(TEST_TABLE);
180+
181+
assertThrows(SQLException.class, () -> builder.getSQLString(CommandName.LIST_FOREIGN_KEYS));
182+
}
183+
184+
@Test
185+
@DisplayName("Should throw SQLException when schema is null for foreign keys")
186+
void shouldThrowExceptionWhenSchemaIsNullForForeignKeys() {
187+
CommandBuilder builder = new CommandBuilder(TEST_CATALOG, mockSession).setTable(TEST_TABLE);
188+
189+
assertThrows(SQLException.class, () -> builder.getSQLString(CommandName.LIST_FOREIGN_KEYS));
190+
}
191+
192+
@Test
193+
@DisplayName("Should throw SQLException when table is null for foreign keys")
194+
void shouldThrowExceptionWhenTableIsNullForForeignKeys() {
195+
CommandBuilder builder = new CommandBuilder(TEST_CATALOG, mockSession).setSchema(TEST_SCHEMA);
196+
197+
assertThrows(SQLException.class, () -> builder.getSQLString(CommandName.LIST_FOREIGN_KEYS));
198+
}
199+
}
200+
201+
@Test
202+
@DisplayName("Should throw exception for unsupported command")
203+
void shouldThrowExceptionForUnsupportedCommand() {
204+
CommandBuilder builder = new CommandBuilder(TEST_CATALOG, mockSession);
205+
206+
CommandName mockCommand = mock(CommandName.class);
207+
208+
assertThrows(
209+
DatabricksSQLFeatureNotSupportedException.class, () -> builder.getSQLString(mockCommand));
210+
}
211+
}

‎src/test/java/com/databricks/jdbc/dbclient/impl/sqlexec/DatabricksMetadataSdkClientTest.java‎

Lines changed: 63 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -748,9 +748,6 @@ void testThrowsErrorResultInCaseOfNullCatalog() {
748748
assertThrows(
749749
DatabricksValidationException.class,
750750
() -> metadataClient.listColumns(session, null, TEST_SCHEMA, TEST_TABLE, TEST_COLUMN));
751-
assertThrows(
752-
DatabricksValidationException.class,
753-
() -> metadataClient.listTables(session, null, TEST_SCHEMA, TEST_TABLE, null));
754751
assertThrows(
755752
DatabricksValidationException.class,
756753
() -> metadataClient.listSchemas(session, null, TEST_SCHEMA));
@@ -767,4 +764,67 @@ void testListTypeInfo() {
767764
DatabricksMetadataSdkClient metadataClient = new DatabricksMetadataSdkClient(mockClient);
768765
assertNotNull(metadataClient.listTypeInfo(session));
769766
}
767+
768+
@Test
769+
void testListTablesAllCatalogs() throws SQLException {
770+
when(session.getComputeResource()).thenReturn(WAREHOUSE_COMPUTE);
771+
DatabricksMetadataSdkClient metadataClient = new DatabricksMetadataSdkClient(mockClient);
772+
when(mockClient.executeStatement(
773+
"SHOW TABLES IN ALL CATALOGS SCHEMA LIKE 'testSchema' LIKE 'testTable'",
774+
WAREHOUSE_COMPUTE,
775+
new HashMap<>(),
776+
StatementType.METADATA,
777+
session,
778+
null))
779+
.thenReturn(mockedResultSet);
780+
when(mockedResultSet.next()).thenReturn(true, false);
781+
for (ResultColumn resultColumn : TABLE_COLUMNS) {
782+
when(mockedResultSet.getObject(resultColumn.getResultSetColumnName()))
783+
.thenReturn(TEST_COLUMN);
784+
}
785+
when(mockedResultSet.getObject("tableType")).thenReturn("TABLE");
786+
doReturn(10).when(mockedMetaData).getColumnCount();
787+
doReturn(CATALOG_COLUMN.getResultSetColumnName()).when(mockedMetaData).getColumnName(1);
788+
doReturn(SCHEMA_COLUMN.getResultSetColumnName()).when(mockedMetaData).getColumnName(2);
789+
doReturn(TABLE_NAME_COLUMN.getResultSetColumnName()).when(mockedMetaData).getColumnName(3);
790+
doReturn(TABLE_TYPE_COLUMN.getResultSetColumnName()).when(mockedMetaData).getColumnName(4);
791+
doReturn(REMARKS_COLUMN.getResultSetColumnName()).when(mockedMetaData).getColumnName(5);
792+
doReturn(TYPE_CATALOG_COLUMN.getResultSetColumnName()).when(mockedMetaData).getColumnName(6);
793+
doReturn(TYPE_SCHEMA_COLUMN.getResultSetColumnName()).when(mockedMetaData).getColumnName(7);
794+
doReturn(TYPE_NAME_COLUMN.getResultSetColumnName()).when(mockedMetaData).getColumnName(8);
795+
doReturn(SELF_REFERENCING_COLUMN_NAME.getResultSetColumnName())
796+
.when(mockedMetaData)
797+
.getColumnName(9);
798+
doReturn(REF_GENERATION_COLUMN.getResultSetColumnName()).when(mockedMetaData).getColumnName(10);
799+
when(mockedResultSet.getMetaData()).thenReturn(mockedMetaData);
800+
DatabricksResultSet actualResult =
801+
metadataClient.listTables(session, null, TEST_SCHEMA, TEST_TABLE, null);
802+
assertEquals(actualResult.getStatementStatus().getState(), StatementState.SUCCEEDED);
803+
assertEquals(actualResult.getStatementId(), GET_TABLES_STATEMENT_ID);
804+
assertEquals(((DatabricksResultSetMetaData) actualResult.getMetaData()).getTotalRows(), 1);
805+
}
806+
807+
@Test
808+
void testGetTablesAllCatalogs_throwsParseSyntaxError() throws Exception {
809+
DatabricksSQLException exception =
810+
new DatabricksSQLException("syntax error at or near \"IN\"", PARSE_SYNTAX_ERROR_SQL_STATE);
811+
when(session.getComputeResource()).thenReturn(WAREHOUSE_COMPUTE);
812+
DatabricksMetadataSdkClient metadataClient = new DatabricksMetadataSdkClient(mockClient);
813+
when(mockClient.executeStatement(
814+
"SHOW TABLES IN ALL CATALOGS SCHEMA LIKE 'testSchema' LIKE 'testTable'",
815+
WAREHOUSE_COMPUTE,
816+
new HashMap<>(),
817+
StatementType.METADATA,
818+
session,
819+
null))
820+
.thenThrow(exception);
821+
try (DatabricksResultSet actualResult =
822+
metadataClient.listTables(session, null, TEST_SCHEMA, TEST_TABLE, null)) {
823+
assertEquals(StatementState.SUCCEEDED, actualResult.getStatementStatus().getState());
824+
assertEquals(GET_TABLES_STATEMENT_ID, actualResult.getStatementId());
825+
assertEquals(10, actualResult.getMetaData().getColumnCount());
826+
// Parse syntax error is handled gracefully to return empty result set
827+
assertEquals(0, ((DatabricksResultSetMetaData) actualResult.getMetaData()).getTotalRows());
828+
}
829+
}
770830
}

0 commit comments

Comments
 (0)