XML Nav Tree Improvements

Author: cgivre

build-frontend.sh

@@ -0,0 +1,53 @@
+#!/bin/bash
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+# Build the SQL Lab frontend and update the distribution.
+# Usage: ./build-frontend.sh
+
+set -e
+
+SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
+WEBAPP_DIR="$SCRIPT_DIR/exec/java-exec/src/main/resources/webapp"
+DIST_BASE="$SCRIPT_DIR/distribution/target"
+
+echo "=== Building SQL Lab frontend ==="
+cd "$WEBAPP_DIR"
+npm run build
+
+echo ""
+echo "=== Building java-exec module ==="
+cd "$SCRIPT_DIR"
+mvn package -pl exec/java-exec -DskipTests -Dcheckstyle.skip=true -q
+
+# Find the distribution directory and copy the jar
+DIST_DIR=$(find "$DIST_BASE" -name "jars" -type d 2>/dev/null | head -1)
+if [ -n "$DIST_DIR" ]; then
+  echo ""
+  echo "=== Copying jar to distribution ==="
+  cp "$SCRIPT_DIR/exec/java-exec/target/drill-java-exec-"*"-SNAPSHOT.jar" "$DIST_DIR/"
+  echo "Updated: $DIST_DIR/"
+else
+  echo ""
+  echo "WARNING: Distribution directory not found under $DIST_BASE"
+  echo "Run a full 'mvn package' first to create the distribution, or copy the jar manually:"
+  echo "  cp exec/java-exec/target/drill-java-exec-*-SNAPSHOT.jar <drill-distribution>/jars/"
+fi
+
+echo ""
+echo "=== Done! Restart Drill and hard-refresh your browser (Cmd+Shift+R) ==="

contrib/format-xml/src/main/java/org/apache/drill/exec/store/xml/XMLReader.java

@@ -161,8 +161,8 @@ public void close() {
 
   /**
    * This function processes the XML elements.  This function stops reading when the
-   * limit (if any) which came from the query has been reached or the Iterator runs out of
-   * elements.
+   * limit (if any) which came from the query has been reached, a complete row has been
+   * read, or the Iterator runs out of elements.
    * @return True if there are more elements to parse, false if not
    */
   private boolean processElements() {
@@ -197,6 +197,13 @@ private boolean processElements() {
 
         // Process the event
         processEvent(currentEvent, lastEvent, reader.peek());
+
+        // After completing a row, return to let next() check batch capacity.
+        // This prevents batch overflow errors that occur when rows accumulate
+        // beyond what the batch can hold without the isFull() check running.
+        if (currentState == xmlState.ROW_ENDED) {
+          return true;
+        }
       } catch (XMLStreamException e) {
         throw UserException
           .dataReadError(e)

exec/java-exec/src/main/java/org/apache/drill/exec/server/rest/MetadataResources.java

@@ -550,8 +550,8 @@ public TablePreviewResponse previewTable(
     int safeLimit = Math.min(Math.max(1, limit), 1000);
 
     String sql = String.format(
-        "SELECT * FROM `%s`.`%s` LIMIT %d",
-        escapeBackticks(schema), escapeBackticks(table), safeLimit);
+        "SELECT * FROM %s.`%s` LIMIT %d",
+        formatSchemaPath(schema), escapeBackticks(table), safeLimit);
 
     try {
       QueryResult result = executeQuery(sql);
@@ -577,9 +577,9 @@ public ColumnsResponse getFileColumns(
     }
 
     // Build the fully qualified path
-    // Handle paths that may contain special characters
-    String fullPath = String.format("`%s`.`%s`",
-        escapeBackticks(schema), escapeBackticks(filePath));
+    // Plugin name stays unquoted, workspace parts are individually backtick-quoted,
+    // and the file path is backtick-quoted. e.g. dfs.`test`.`file.xml`
+    String fullPath = formatSchemaPath(schema) + ".`" + escapeBackticks(filePath) + "`";
 
     String sql = String.format("SELECT * FROM %s LIMIT 1", fullPath);
 
@@ -588,11 +588,15 @@ public ColumnsResponse getFileColumns(
     try {
       QueryResult result = executeQuery(sql);
 
-      // Get column names and infer types from the first row of results
-      for (String columnName : result.columns) {
-        // Try to infer type from first row value
+      // Use result.metadata for column types when available (preferred),
+      // fall back to value-based inference.
+      List<String> columnNames = new ArrayList<>(result.columns);
+      for (int i = 0; i < columnNames.size(); i++) {
+        String columnName = columnNames.get(i);
         String dataType = "ANY";
-        if (!result.rows.isEmpty()) {
+        if (result.metadata != null && i < result.metadata.size()) {
+          dataType = result.metadata.get(i);
+        } else if (!result.rows.isEmpty()) {
           String value = result.rows.get(0).get(columnName);
           dataType = inferDataType(value);
         }
@@ -717,4 +721,22 @@ private String escapeBackticks(String value) {
     }
     return value.replace("`", "``");
   }
+
+  /**
+   * Format a compound schema name for SQL queries.
+   * Plugin name stays unquoted; workspace parts are individually backtick-quoted.
+   * e.g. "dfs.test" → "dfs.`test`", "dfs" → "dfs"
+   */
+  private String formatSchemaPath(String schema) {
+    if (schema == null || !schema.contains(".")) {
+      return schema;
+    }
+    String[] parts = schema.split("\\.", 2);
+    String[] workspaceParts = parts[1].split("\\.");
+    StringBuilder sb = new StringBuilder(parts[0]);
+    for (String wp : workspaceParts) {
+      sb.append(".`").append(escapeBackticks(wp)).append("`");
+    }
+    return sb.toString();
+  }
 }

exec/java-exec/src/main/resources/webapp/src/api/metadata.ts

@@ -16,11 +16,24 @@
  * limitations under the License.
  */
 import apiClient from './client';
-import type { SchemaInfo, TableInfo, ColumnInfo, PluginInfo, NestedFieldInfo } from '../types';
+import type { SchemaInfo, TableInfo, ColumnInfo, PluginInfo, NestedFieldInfo, SubTableInfo } from '../types';
 import { executeQuery } from './queries';
 
 const METADATA_BASE = '/api/v1/metadata';
 
+/**
+ * Format a compound schema name for use in SQL queries.
+ * Plugin name stays unquoted; workspace parts are backtick-quoted.
+ * e.g. "dfs.test" → "dfs.`test`", "dfs" → "dfs"
+ */
+function formatSchema(schema: string): string {
+  const parts = schema.split('.');
+  if (parts.length <= 1) {
+    return schema;
+  }
+  return parts[0] + '.' + parts.slice(1).map((p) => `\`${p}\``).join('.');
+}
+
 export interface PluginsResponse {
   plugins: PluginInfo[];
 }
@@ -147,14 +160,24 @@ export async function getFiles(schema: string, subPath?: string): Promise<FileIn
 }
 
 /**
- * Fetch columns from a file by executing SELECT * LIMIT 1
+ * Fetch columns from a file by executing SELECT * LIMIT 1 via the query API.
+ * Uses the same code path as the SQL editor so format plugins work consistently.
  */
 export async function getFileColumns(schema: string, filePath: string): Promise<ColumnInfo[]> {
-  const response = await apiClient.get<ColumnsResponse>(
-    `${METADATA_BASE}/schemas/${encodeURIComponent(schema)}/files/columns`,
-    { params: { path: filePath } }
-  );
-  return response.data.columns;
+  const query = `SELECT * FROM ${formatSchema(schema)}.\`${filePath}\` LIMIT 1`;
+  const result = await executeQuery({ query, queryType: 'SQL', autoLimitRowCount: 1 });
+
+  if (!result.columns || result.columns.length === 0) {
+    return [];
+  }
+
+  return result.columns.map((colName, idx) => ({
+    name: colName,
+    type: result.metadata?.[idx] || 'ANY',
+    nullable: true,
+    schema,
+    table: filePath,
+  }));
 }
 
 /**
@@ -192,7 +215,7 @@ export async function getNestedColumns(
 
   const query =
     `SELECT getMapSchema(${columnExpr}) AS \`schema\`` +
-    ` FROM \`${schema}\`.\`${tableOrFile}\` LIMIT 1`;
+    ` FROM ${formatSchema(schema)}.\`${tableOrFile}\` LIMIT 1`;
 
   const result = await executeQuery({
     query,
@@ -221,3 +244,105 @@ export async function getNestedColumns(
 
   return [];
 }
+
+/**
+ * Fetch sub-tables (sheets, datasets, tables) within a multi-table file.
+ *
+ * @param schema     the schema name (e.g. "dfs.tmp")
+ * @param filePath   the file path (e.g. "data.xlsx")
+ * @param formatType "excel" | "hdf5" | "msaccess"
+ */
+export async function getSubTables(
+  schema: string,
+  filePath: string,
+  formatType: string,
+): Promise<SubTableInfo[]> {
+  let query: string;
+
+  switch (formatType) {
+    case 'excel':
+      query = `SELECT _sheets FROM ${formatSchema(schema)}.\`${filePath}\` LIMIT 1`;
+      break;
+    case 'hdf5':
+      query = `SELECT path, data_type FROM ${formatSchema(schema)}.\`${filePath}\``;
+      break;
+    case 'msaccess':
+      query = `SELECT \`table\` FROM ${formatSchema(schema)}.\`${filePath}\``;
+      break;
+    default:
+      return [];
+  }
+
+  const result = await executeQuery({ query, queryType: 'SQL', autoLimitRowCount: 1000 });
+
+  if (!result.rows || result.rows.length === 0) {
+    return [];
+  }
+
+  if (formatType === 'excel') {
+    const sheetsVal = result.rows[0]['_sheets'];
+    if (Array.isArray(sheetsVal)) {
+      return sheetsVal.map((s) => ({ name: String(s) }));
+    }
+    if (typeof sheetsVal === 'string') {
+      // Could be JSON array string or comma-separated
+      try {
+        const parsed = JSON.parse(sheetsVal);
+        if (Array.isArray(parsed)) {
+          return parsed.map((s: unknown) => ({ name: String(s) }));
+        }
+      } catch {
+        // Treat as comma-separated
+        return sheetsVal.split(',').map((s) => ({ name: s.trim() })).filter((s) => s.name.length > 0);
+      }
+    }
+    return [];
+  }
+
+  if (formatType === 'hdf5') {
+    return result.rows
+      .filter((row) => String(row['data_type']).toUpperCase() === 'DATASET')
+      .map((row) => ({ name: String(row['path']), dataType: String(row['data_type']) }));
+  }
+
+  if (formatType === 'msaccess') {
+    return result.rows.map((row) => ({ name: String(row['table']) }));
+  }
+
+  return [];
+}
+
+/**
+ * Fetch columns for a sub-table within a multi-table file using table function syntax.
+ *
+ * @param schema       the schema name (e.g. "dfs.tmp")
+ * @param filePath     the file path (e.g. "data.xlsx")
+ * @param formatType   "excel" | "hdf5" | "msaccess"
+ * @param paramName    the table-function parameter name (e.g. "sheetName")
+ * @param subTableName the specific sub-table name (e.g. "Sheet1")
+ */
+export async function getSubTableColumns(
+  schema: string,
+  filePath: string,
+  formatType: string,
+  paramName: string,
+  subTableName: string,
+): Promise<ColumnInfo[]> {
+  const query =
+    `SELECT * FROM table( ${formatSchema(schema)}.\`${filePath}\`` +
+    ` (type => '${formatType}', ${paramName} => '${subTableName}')) LIMIT 1`;
+
+  const result = await executeQuery({ query, queryType: 'SQL', autoLimitRowCount: 1 });
+
+  if (!result.columns || result.columns.length === 0) {
+    return [];
+  }
+
+  return result.columns.map((colName, idx) => ({
+    name: colName,
+    type: result.metadata?.[idx] || 'ANY',
+    nullable: true,
+    schema,
+    table: `${filePath}/${subTableName}`,
+  }));
+}