|
| 1 | +package com.databricks.jdbc.api.impl; |
| 2 | + |
| 3 | +import static com.databricks.jdbc.common.EnvironmentVariables.DEFAULT_RESULT_ROW_LIMIT; |
| 4 | +import static com.databricks.jdbc.common.util.DatabricksThriftUtil.extractRowsFromColumnar; |
| 5 | + |
| 6 | +import com.databricks.jdbc.api.internal.IDatabricksSession; |
| 7 | +import com.databricks.jdbc.api.internal.IDatabricksStatementInternal; |
| 8 | +import com.databricks.jdbc.exception.DatabricksSQLException; |
| 9 | +import com.databricks.jdbc.log.JdbcLogger; |
| 10 | +import com.databricks.jdbc.log.JdbcLoggerFactory; |
| 11 | +import com.databricks.jdbc.model.client.thrift.generated.TFetchResultsResp; |
| 12 | +import com.databricks.jdbc.model.telemetry.enums.DatabricksDriverErrorCode; |
| 13 | +import java.util.List; |
| 14 | + |
| 15 | +public class LazyThriftResult implements IExecutionResult { |
| 16 | + private static final JdbcLogger LOGGER = JdbcLoggerFactory.getLogger(LazyThriftResult.class); |
| 17 | + |
| 18 | + private TFetchResultsResp currentResponse; |
| 19 | + private List<List<Object>> currentBatch; |
| 20 | + private int currentBatchIndex; |
| 21 | + private long globalRowIndex; |
| 22 | + private final IDatabricksSession session; |
| 23 | + private final IDatabricksStatementInternal statement; |
| 24 | + private final int maxRows; |
| 25 | + private boolean hasReachedEnd; |
| 26 | + private boolean isClosed; |
| 27 | + private long totalRowsFetched; |
| 28 | + |
| 29 | + /** |
| 30 | + * Creates a new LazyThriftResult that lazily fetches data on demand. |
| 31 | + * |
| 32 | + * @param initialResponse the initial response from the server |
| 33 | + * @param statement the statement that generated this result |
| 34 | + * @param session the session to use for fetching additional data |
| 35 | + * @throws DatabricksSQLException if the initial response cannot be processed |
| 36 | + */ |
| 37 | + public LazyThriftResult( |
| 38 | + TFetchResultsResp initialResponse, |
| 39 | + IDatabricksStatementInternal statement, |
| 40 | + IDatabricksSession session) |
| 41 | + throws DatabricksSQLException { |
| 42 | + this.currentResponse = initialResponse; |
| 43 | + this.statement = statement; |
| 44 | + this.session = session; |
| 45 | + this.maxRows = statement != null ? statement.getMaxRows() : DEFAULT_RESULT_ROW_LIMIT; |
| 46 | + this.globalRowIndex = -1; |
| 47 | + this.currentBatchIndex = -1; |
| 48 | + this.hasReachedEnd = false; |
| 49 | + this.isClosed = false; |
| 50 | + this.totalRowsFetched = 0; |
| 51 | + |
| 52 | + // Load initial batch |
| 53 | + loadCurrentBatch(); |
| 54 | + LOGGER.debug( |
| 55 | + "LazyThriftResult initialized with {} rows in first batch, hasMoreRows: {}", |
| 56 | + currentBatch.size(), |
| 57 | + currentResponse.hasMoreRows); |
| 58 | + } |
| 59 | + |
| 60 | + /** |
| 61 | + * Gets the value at the specified column index for the current row. |
| 62 | + * |
| 63 | + * @param columnIndex the zero-based column index |
| 64 | + * @return the value at the specified column |
| 65 | + * @throws DatabricksSQLException if the result is closed, cursor is invalid, or column index is |
| 66 | + * out of bounds |
| 67 | + */ |
| 68 | + @Override |
| 69 | + public Object getObject(int columnIndex) throws DatabricksSQLException { |
| 70 | + if (isClosed) { |
| 71 | + throw new DatabricksSQLException( |
| 72 | + "Result is already closed", DatabricksDriverErrorCode.STATEMENT_CLOSED); |
| 73 | + } |
| 74 | + if (globalRowIndex == -1) { |
| 75 | + throw new DatabricksSQLException( |
| 76 | + "Cursor is before first row", DatabricksDriverErrorCode.INVALID_STATE); |
| 77 | + } |
| 78 | + if (currentBatchIndex < 0 || currentBatchIndex >= currentBatch.size()) { |
| 79 | + throw new DatabricksSQLException( |
| 80 | + "Invalid cursor position", DatabricksDriverErrorCode.INVALID_STATE); |
| 81 | + } |
| 82 | + List<Object> currentRowData = currentBatch.get(currentBatchIndex); |
| 83 | + if (columnIndex < 0 || columnIndex >= currentRowData.size()) { |
| 84 | + throw new DatabricksSQLException( |
| 85 | + "Column index out of bounds " + columnIndex, DatabricksDriverErrorCode.INVALID_STATE); |
| 86 | + } |
| 87 | + return currentRowData.get(columnIndex); |
| 88 | + } |
| 89 | + |
| 90 | + /** |
| 91 | + * Gets the current row index (0-based). Returns -1 if before the first row. |
| 92 | + * |
| 93 | + * @return the current row index |
| 94 | + */ |
| 95 | + @Override |
| 96 | + public long getCurrentRow() { |
| 97 | + return globalRowIndex; |
| 98 | + } |
| 99 | + |
| 100 | + /** |
| 101 | + * Moves the cursor to the next row. Fetches additional data from server if needed. |
| 102 | + * |
| 103 | + * @return true if there is a next row, false if at the end |
| 104 | + * @throws DatabricksSQLException if an error occurs while fetching data |
| 105 | + */ |
| 106 | + @Override |
| 107 | + public boolean next() throws DatabricksSQLException { |
| 108 | + if (isClosed || hasReachedEnd) { |
| 109 | + return false; |
| 110 | + } |
| 111 | + |
| 112 | + if (!hasNext()) { |
| 113 | + // Ideally the client code should first call, hasNext() and then next() |
| 114 | + // However, the client code like in DatabricksResultSet#next directly calls next |
| 115 | + // So, this is a safeguard to ensure we don't move past the end |
| 116 | + return false; |
| 117 | + } |
| 118 | + |
| 119 | + // Check if we've reached the maxRows limit |
| 120 | + boolean hasRowLimit = maxRows > 0; |
| 121 | + if (hasRowLimit && globalRowIndex + 1 >= maxRows) { |
| 122 | + hasReachedEnd = true; |
| 123 | + return false; |
| 124 | + } |
| 125 | + |
| 126 | + // Move to next row in current batch |
| 127 | + currentBatchIndex++; |
| 128 | + globalRowIndex++; |
| 129 | + |
| 130 | + // Check if we need to fetch the next batch |
| 131 | + if (currentBatchIndex >= currentBatch.size()) { |
| 132 | + // Keep fetching until we get a non-empty batch or no more rows |
| 133 | + while (currentResponse.hasMoreRows) { |
| 134 | + fetchNextBatch(); |
| 135 | + |
| 136 | + // If we got a non-empty batch, we can proceed |
| 137 | + if (!currentBatch.isEmpty()) { |
| 138 | + currentBatchIndex = 0; // Reset to first row of new batch |
| 139 | + break; |
| 140 | + } |
| 141 | + |
| 142 | + // If batch is still empty but hasMoreRows is false after fetch, we'll exit the loop |
| 143 | + } |
| 144 | + |
| 145 | + // If we exited the loop and still have an empty batch, we've reached the end |
| 146 | + if (currentBatch.isEmpty()) { |
| 147 | + hasReachedEnd = true; |
| 148 | + globalRowIndex--; // Revert the increment since we didn't actually move to a new row |
| 149 | + return false; |
| 150 | + } |
| 151 | + } |
| 152 | + |
| 153 | + return true; |
| 154 | + } |
| 155 | + |
| 156 | + /** |
| 157 | + * Checks if there are more rows available without advancing the cursor. |
| 158 | + * |
| 159 | + * @return true if there are more rows, false otherwise |
| 160 | + */ |
| 161 | + @Override |
| 162 | + public boolean hasNext() { |
| 163 | + if (isClosed || hasReachedEnd) { |
| 164 | + return false; |
| 165 | + } |
| 166 | + |
| 167 | + // Check maxRows limit |
| 168 | + boolean hasRowLimit = maxRows > 0; |
| 169 | + if (hasRowLimit && globalRowIndex + 1 >= maxRows) { |
| 170 | + return false; |
| 171 | + } |
| 172 | + |
| 173 | + // Check if there are more rows in current batch |
| 174 | + if (currentBatchIndex + 1 < currentBatch.size()) { |
| 175 | + return true; |
| 176 | + } |
| 177 | + |
| 178 | + // Check if there are more batches to fetch |
| 179 | + return currentResponse.hasMoreRows; |
| 180 | + } |
| 181 | + |
| 182 | + /** Closes this result and releases associated resources. */ |
| 183 | + @Override |
| 184 | + public void close() { |
| 185 | + this.isClosed = true; |
| 186 | + this.currentBatch = null; |
| 187 | + this.currentResponse = null; |
| 188 | + LOGGER.debug("LazyThriftResult closed after fetching {} total rows", totalRowsFetched); |
| 189 | + } |
| 190 | + |
| 191 | + /** |
| 192 | + * Gets the number of rows in the current batch. |
| 193 | + * |
| 194 | + * @return the number of rows in the current batch |
| 195 | + */ |
| 196 | + @Override |
| 197 | + public long getRowCount() { |
| 198 | + // Return the number of rows in the current batch |
| 199 | + return currentBatch != null ? currentBatch.size() : 0; |
| 200 | + } |
| 201 | + |
| 202 | + /** |
| 203 | + * Gets the chunk count. Always returns 0 for thrift columnar results. |
| 204 | + * |
| 205 | + * @return 0 (thrift results don't use chunks like Arrow) |
| 206 | + */ |
| 207 | + @Override |
| 208 | + public long getChunkCount() { |
| 209 | + // For thrift columnar results, we don't have chunks in the same sense as Arrow |
| 210 | + return 0; |
| 211 | + } |
| 212 | + |
| 213 | + /** |
| 214 | + * Loads the current response data into memory as a batch of rows. |
| 215 | + * |
| 216 | + * @throws DatabricksSQLException if the response data cannot be processed |
| 217 | + */ |
| 218 | + private void loadCurrentBatch() throws DatabricksSQLException { |
| 219 | + currentBatch = extractRowsFromColumnar(currentResponse.getResults()); |
| 220 | + currentBatchIndex = -1; // Reset batch index |
| 221 | + totalRowsFetched += currentBatch.size(); |
| 222 | + LOGGER.debug( |
| 223 | + "Loaded batch with {} rows, total fetched: {}", currentBatch.size(), totalRowsFetched); |
| 224 | + } |
| 225 | + |
| 226 | + /** |
| 227 | + * Fetches the next batch of data from the server and loads it into memory. |
| 228 | + * |
| 229 | + * @throws DatabricksSQLException if the fetch operation fails |
| 230 | + */ |
| 231 | + private void fetchNextBatch() throws DatabricksSQLException { |
| 232 | + try { |
| 233 | + LOGGER.debug("Fetching next batch, current total rows fetched: {}", totalRowsFetched); |
| 234 | + currentResponse = session.getDatabricksClient().getMoreResults(statement); |
| 235 | + loadCurrentBatch(); |
| 236 | + |
| 237 | + LOGGER.debug( |
| 238 | + "Fetched batch with {} rows, hasMoreRows: {}", |
| 239 | + currentBatch.size(), |
| 240 | + currentResponse.hasMoreRows); |
| 241 | + } catch (DatabricksSQLException e) { |
| 242 | + LOGGER.error("Failed to fetch next batch: {}", e.getMessage()); |
| 243 | + hasReachedEnd = true; |
| 244 | + throw e; // Propagate exception to fail fast |
| 245 | + } |
| 246 | + } |
| 247 | + |
| 248 | + /** |
| 249 | + * Gets the total number of rows fetched from the server so far. This is different from |
| 250 | + * getRowCount() which returns current batch size. |
| 251 | + * |
| 252 | + * @return the total number of rows fetched from the server |
| 253 | + */ |
| 254 | + public long getTotalRowsFetched() { |
| 255 | + return totalRowsFetched; |
| 256 | + } |
| 257 | + |
| 258 | + /** |
| 259 | + * Checks if all data has been fetched from the server. |
| 260 | + * |
| 261 | + * @return true if all data has been fetched (either reached end or maxRows limit) |
| 262 | + */ |
| 263 | + public boolean isCompletelyFetched() { |
| 264 | + return hasReachedEnd || !currentResponse.hasMoreRows; |
| 265 | + } |
| 266 | +} |
0 commit comments