Deprecate EnableArrow — Arrow always enabled except on AIX/IBM Power (#1450)

## Summary
`EnableArrow` connection property is deprecated and ignored. Arrow
serialization is now always enabled. Setting `EnableArrow=0` logs a
deprecation warning but has no effect.

### Behavior change
| Scenario | Before | After |
|----------|--------|-------|
| `EnableArrow=0` on Linux/Mac/Windows | Arrow disabled, columnar inline
results | Arrow enabled (value ignored), warning logged |
| `EnableArrow=0` on AIX / IBM Power (`ppc`) | Arrow disabled | Arrow
disabled (honoured — known native library issues) |
| `EnableArrow=1` or not set | Arrow enabled | Arrow enabled (no change)
|
| SEA + CloudFetch disabled | N/A | JSON inline results (via
`EnableQueryResultDownload=0`) |

### Changes
- `shouldEnableArrow()`: Always returns `true` except on AIX (`os.name`
contains `aix`) or IBM Power (`os.arch` contains `ppc`)
- Removed client-type check that forced Thrift when Arrow was disabled —
no longer needed
- Updated `DatabricksJdbcUrlParams` description to indicate deprecation
- Updated tests to reflect new behavior
- Added breaking change to `NEXT_CHANGELOG.md`

## Test plan
- [x] `DatabricksConnectionContextTest` — 120 tests pass (updated
`testEnableCloudFetch` + decision matrix)
- [x] `DatabricksSessionTest` + `DatabricksSdkClientTest` — 175 total,
all pass

This pull request was AI-assisted by Isaac.

---------

Signed-off-by: Gopal Lal <gopal.lal@databricks.com>

Author: gopalldb

NEXT_CHANGELOG.md

@@ -2,7 +2,9 @@
 
 ## [Unreleased]
 
-### BREAKING CHANGES in 3.4.1 — Metadata JDBC Spec Compliance
+### BREAKING CHANGES in 3.4.1
+
+#### Metadata JDBC Spec Compliance
 
 This release unifies metadata behavior across Thrift and SQL Exec API backends
 using SQL SHOW commands for all metadata operations on SQL warehouses. Several
@@ -28,11 +30,22 @@ upgrading. These changes do not affect metadata on All-Purpose Clusters.
   instead of `0` (`CASCADE`) for Thrift, and `3` instead of `null` for SEA.**
   This reflects that Unity Catalog foreign keys are informational and non-enforced.
 
+#### Default Behavior Changes
+
 * **Native geospatial type support (`GEOMETRY` and `GEOGRAPHY`) is now enabled
   by default.** `getObject()` now returns `IGeometry`/`IGeography` instances
   instead of EWKT strings. Set `EnableGeoSpatialSupport=0` to restore the
   previous behavior.
 
+* **`EnableArrow` connection property is deprecated and ignored.** Arrow
+  serialization is now always enabled. Setting `EnableArrow=0` previously
+  disabled Arrow and forced columnar/JSON inline results; this value is now
+  ignored and a deprecation warning is logged. For JSON inline results with
+  SEA, disable CloudFetch via `EnableQueryResultDownload=0`. Exception: on AIX
+  platforms and PowerPC architectures (`os.arch` contains `ppc`), `EnableArrow`
+  is still honoured and defaults to disabled due to known Arrow native library
+  compatibility issues.
+
 ### Added
 - Metadata operations now use SQL SHOW commands for both Thrift and SEA backends,
   ensuring consistent behavior for SQL warehouses regardless of underlying

src/main/java/com/databricks/jdbc/api/impl/DatabricksConnectionContext.java

@@ -57,6 +57,7 @@ public class DatabricksConnectionContext implements IDatabricksConnectionContext
   private Supplier<DatabricksClientType> clientTypeSupplier;
   @VisibleForTesting final ImmutableMap<String, String> parameters;
   @VisibleForTesting final String connectionUuid;
+  private final boolean enableArrow;
 
   private DatabricksConnectionContext(
       String connectionURL,
@@ -73,6 +74,7 @@ private DatabricksConnectionContext(
     this.customHeaders = parseCustomHeaders(parameters);
     this.computeResource = buildCompute();
     this.connectionUuid = UUID.randomUUID().toString();
+    this.enableArrow = resolveEnableArrow();
     this.clientTypeSupplier =
         new Supplier<>() {
           private DatabricksClientType cType;
@@ -97,6 +99,7 @@ private DatabricksConnectionContext(
     this.customHeaders = parseCustomHeaders(parameters);
     this.computeResource = null;
     this.connectionUuid = UUID.randomUUID().toString();
+    this.enableArrow = resolveEnableArrow();
   }
 
   /**
@@ -505,8 +508,8 @@ public DatabricksClientType getClientTypeFromContext() {
           remainingMs);
       return DatabricksClientType.THRIFT;
     }
-    // Check if Arrow is disabled - Thrift is required for inline mode
-    if (!Objects.equals(getParameter(DatabricksJdbcUrlParams.ENABLE_ARROW), "1")) {
+    // On AIX/PowerPC, Arrow may be disabled — check before routing to SEA
+    if (isAixOrPowerPc() && !shouldEnableArrow()) {
       return DatabricksClientType.THRIFT;
     }
     // Check if CloudFetch is disabled - Thrift is required for inline mode
@@ -669,7 +672,34 @@ public ProxyConfig.ProxyAuthType getCloudFetchProxyAuthType() {
 
   @Override
   public Boolean shouldEnableArrow() {
-    return Objects.equals(getParameter(DatabricksJdbcUrlParams.ENABLE_ARROW), "1");
+    return enableArrow;
+  }
+
+  /** Evaluates the Arrow enablement once at construction time. */
+  private boolean resolveEnableArrow() {
+    // Arrow is always enabled unless running on AIX or IBM Power (which have known
+    // issues with the Arrow native library). The EnableArrow connection property is
+    // deprecated and its value is ignored on non-AIX/IBM platforms.
+    if (isAixOrPowerPc()) {
+      // On AIX/PowerPC, Arrow native library has known issues — default to disabled.
+      // Honour explicit EnableArrow=1 if user sets it.
+      String explicitValue = getParameterIgnoreDefault(DatabricksJdbcUrlParams.ENABLE_ARROW);
+      if (explicitValue != null) {
+        return explicitValue.equals("1");
+      }
+      return false; // default disabled on AIX/PowerPC
+    }
+
+    // Log deprecation warning once if user explicitly set EnableArrow=0 (ignored)
+    String explicitValue = getParameterIgnoreDefault(DatabricksJdbcUrlParams.ENABLE_ARROW);
+    if (explicitValue != null && explicitValue.equals("0")) {
+      LOGGER.info(
+          "EnableArrow=0 is deprecated and ignored. Arrow serialization is always enabled. "
+              + "To use JSON inline results with SEA, disable CloudFetch via "
+              + "EnableQueryResultDownload=0.");
+    }
+
+    return true;
   }
 
   @Override
@@ -1198,6 +1228,13 @@ private String getParameterIgnoreDefault(DatabricksJdbcUrlParams key) {
     return this.parameters.getOrDefault(key.getParamName().toLowerCase(), null);
   }
 
+  /** Returns true if running on AIX or IBM PowerPC architecture. */
+  private static boolean isAixOrPowerPc() {
+    String osName = System.getProperty("os.name", "").toLowerCase();
+    String osArch = System.getProperty("os.arch", "").toLowerCase();
+    return osName.contains("aix") || osArch.contains("ppc");
+  }
+
   /**
    * Resolves a boolean feature flag with client-side priority over server-side.
    *

src/main/java/com/databricks/jdbc/common/DatabricksJdbcUrlParams.java

@@ -55,7 +55,11 @@ public enum DatabricksJdbcUrlParams {
   DISCOVERY_URL("OAuthDiscoveryURL", "OAuth discovery URL"), // Same as OIDC_DISCOVERY_ENDPOINT
   IDENTITY_FEDERATION_CLIENT_ID(
       "Identity_Federation_Client_Id", "OAuth Client ID for Token Federation"),
-  ENABLE_ARROW("EnableArrow", "Enable Arrow", "1"),
+  ENABLE_ARROW(
+      "EnableArrow",
+      "Deprecated: Arrow is always enabled. Value ignored except on AIX. "
+          + "Use EnableQueryResultDownload=0 for JSON inline results with SEA.",
+      "1"),
   DIRECT_RESULT("EnableDirectResults", "Enable direct results", "1"),
   LZ4_COMPRESSION_FLAG(
       "EnableQueryResultLZ4Compression", "Enable LZ4 compression"), // Backward compatibility

src/test/java/com/databricks/jdbc/api/impl/DatabricksConnectionContextTest.java

@@ -351,10 +351,26 @@ public void testEnableCloudFetch() throws DatabricksSQLException {
         (DatabricksConnectionContext)
             DatabricksConnectionContext.parse(TestConstants.VALID_URL_5, properties);
     assertTrue(connectionContext.shouldEnableArrow());
+    // EnableArrow=0 is deprecated and ignored on non-AIX platforms — always returns true
     connectionContext =
         (DatabricksConnectionContext)
             DatabricksConnectionContext.parse(TestConstants.VALID_URL_7, properties);
-    assertFalse(connectionContext.shouldEnableArrow());
+    assertTrue(connectionContext.shouldEnableArrow());
+  }
+
+  @Test
+  public void testShouldEnableArrow_defaultIsTrue() throws DatabricksSQLException {
+    // On non-AIX, Arrow is always enabled regardless of EnableArrow setting
+    IDatabricksConnectionContext ctx =
+        DatabricksConnectionContext.parse(TestConstants.VALID_URL_1, properties);
+    assertTrue(ctx.shouldEnableArrow(), "Arrow should be enabled by default");
+  }
+
+  @Test
+  public void testShouldEnableArrow_explicitDisableIgnoredOnNonAix() throws DatabricksSQLException {
+    IDatabricksConnectionContext ctx =
+        DatabricksConnectionContext.parse(TestConstants.VALID_URL_1 + ";EnableArrow=0", properties);
+    assertTrue(ctx.shouldEnableArrow(), "EnableArrow=0 should be ignored on non-AIX");
   }
 
   @Test
@@ -946,7 +962,9 @@ public void testClientTypeWithExplicitUseThriftClientDisabled() throws Databrick
   }
 
   @Test
-  public void testClientTypeWhenArrowDisabled() throws DatabricksSQLException {
+  public void testClientTypeWhenArrowDisabled_nonAix_ignoredDefaultsToThrift()
+      throws DatabricksSQLException {
+    // EnableArrow=0 is ignored on non-AIX — but without SEA feature flag, defaults to THRIFT
     String urlWithArrowDisabled =
         "jdbc:databricks://sample-host.cloud.databricks.com:9999/default;AuthMech=3;"
             + "httpPath=/sql/1.0/warehouses/9999999999999999;EnableArrow=0";
@@ -968,7 +986,9 @@ public void testClientTypeWhenCloudFetchDisabled() throws DatabricksSQLException
   }
 
   @Test
-  public void testClientTypeWhenBothArrowAndCloudFetchDisabled() throws DatabricksSQLException {
+  public void testClientTypeWhenBothArrowAndCloudFetchDisabled_nonAix()
+      throws DatabricksSQLException {
+    // EnableArrow=0 ignored on non-AIX; CloudFetch disabled → THRIFT
     String urlWithBothDisabled =
         "jdbc:databricks://sample-host.cloud.databricks.com:9999/default;AuthMech=3;"
             + "httpPath=/sql/1.0/warehouses/9999999999999999;EnableArrow=0;EnableQueryResultDownload=0";
@@ -1104,11 +1124,12 @@ public void testClientTypeWhenFeatureFlagNotFound() throws DatabricksSQLExceptio
     "false, 1, 1, 1, true, THRIFT", // Explicit useThriftClient=1 returns THRIFT
     "false, 0, 1, 1, true, SEA", // Explicit useThriftClient=0 returns SEA
     "false, 0, 1, 1, false, SEA", // Explicit useThriftClient=0 returns SEA (ignores flag)
-    "false, null, 0, 1, true, THRIFT", // Arrow disabled returns THRIFT
+    "false, null, 0, 1, true, SEA", // Arrow param ignored (deprecated) + CloudFetch enabled +
+    // flag=true → SEA
     "false, null, 1, 0, true, THRIFT", // CloudFetch disabled returns THRIFT
     "false, null, 1, 1, true, SEA", // All enabled + flag=true returns SEA
     "false, null, 1, 1, false, THRIFT", // All enabled + flag=false returns THRIFT
-    "false, null, 0, 0, true, THRIFT", // Both Arrow and CloudFetch disabled returns THRIFT
+    "false, null, 0, 0, true, THRIFT", // CloudFetch disabled returns THRIFT (Arrow param ignored)
   })
   public void testClientTypeDecisionMatrix(
       boolean isCluster,

src/test/java/com/databricks/jdbc/integration/fakeservice/tests/DataTypesIntegrationTests.java

@@ -8,7 +8,6 @@
 import com.databricks.jdbc.integration.fakeservice.AbstractFakeServiceIntegrationTests;
 import com.databricks.jdbc.integration.fakeservice.FakeServiceExtension;
 import java.sql.*;
-import java.util.Properties;
 import java.util.stream.Stream;
 import org.junit.jupiter.api.AfterEach;
 import org.junit.jupiter.api.Assumptions;
@@ -22,7 +21,6 @@
 public class DataTypesIntegrationTests extends AbstractFakeServiceIntegrationTests {
 
   private Connection connection;
-  private Connection inlineConnection;
 
   private static final String LEADING_TRAILING_SPACES = "   leading and trailing spaces   ";
   private static final String UNICODE_TEXT = "こんにちは";
@@ -32,15 +30,11 @@ public class DataTypesIntegrationTests extends AbstractFakeServiceIntegrationTes
   @BeforeEach
   void setUp() throws SQLException {
     connection = getValidJDBCConnection();
-    Properties properties = new Properties();
-    properties.setProperty("enableArrow", "0");
-    inlineConnection = getValidJDBCConnection(properties);
   }
 
   @AfterEach
   void cleanUp() throws SQLException {
     closeConnection(connection);
-    closeConnection(inlineConnection);
   }
 
   @Test
@@ -63,28 +57,18 @@ void testStringEdgeCases() throws SQLException {
             + ") AS string_edge_cases(id, test_string) "
             + "ORDER BY id";
     ResultSet resultSet = executeQuery(connection, query);
-    ResultSet inlineResultSet = executeQuery(inlineConnection, query);
-
     validateStringResults(resultSet);
-    validateStringResults(inlineResultSet);
-
     resultSet.close();
-    inlineResultSet.close();
   }
 
   @ParameterizedTest
   @MethodSource("nullHandlingProvider")
   void testNullHandling(String query, int expectedType) throws SQLException {
     ResultSet resultSet = executeQuery(connection, query);
-    ResultSet inlineResultSet = executeQuery(inlineConnection, query);
     assertTrue(resultSet.next());
-    assertTrue(inlineResultSet.next());
     assertNull(resultSet.getObject(1));
-    assertNull(inlineResultSet.getObject(1));
     assertEquals(expectedType, resultSet.getMetaData().getColumnType(1));
-    assertEquals(expectedType, inlineResultSet.getMetaData().getColumnType(1));
     resultSet.close();
-    inlineResultSet.close();
   }
 
   private static Stream<Arguments> nullHandlingProvider() {
@@ -235,9 +219,7 @@ void testTimestamp() throws SQLException {
 
     String query = "SELECT id, ts FROM " + getFullyQualifiedTableName(tableName) + " ORDER BY id";
     ResultSet resultSet = executeQuery(connection, query);
-    ResultSet inlineResultSet = executeQuery(inlineConnection, query);
     validateTimestampResults(resultSet);
-    validateTimestampResults(inlineResultSet);
 
     deleteTable(connection, tableName);
     resultSet.close();

src/test/java/com/databricks/jdbc/integration/fakeservice/tests/SqlExecApiIntegrationTests.java

@@ -1,35 +1,12 @@
 package com.databricks.jdbc.integration.fakeservice.tests;
 
-import static com.databricks.jdbc.integration.IntegrationTestUtil.getValidJDBCConnection;
-import static org.junit.jupiter.api.Assertions.assertEquals;
-
 import com.databricks.jdbc.integration.fakeservice.AbstractFakeServiceIntegrationTests;
-import java.sql.Connection;
-import java.sql.ResultSet;
-import java.sql.SQLException;
-import java.sql.Statement;
-import java.util.Properties;
-import org.junit.jupiter.api.Test;
-
-public class SqlExecApiIntegrationTests extends AbstractFakeServiceIntegrationTests {
-
-  @Test
-  void testJsonInlineChunkedResults_withoutArrow() throws SQLException {
-    final String table = "samples.tpch.lineitem";
-    // Limit set such that atleast 2 chunks are present for results
-    final int maxRows = 64000;
-    final String sql = "SELECT * FROM " + table + " limit " + maxRows;
-
-    Properties properties = new Properties();
-    properties.setProperty("EnableArrow", "0");
-    Connection connection = getValidJDBCConnection(properties);
 
-    final Statement stmt = connection.createStatement();
-    ResultSet rs = stmt.executeQuery(sql);
-    int rowCount = 0;
-    while (rs.next()) {
-      rowCount++;
-    }
-    assertEquals(maxRows, rowCount);
-  }
-}
+/**
+ * SQL Exec API integration tests. The original JSON inline chunked results test
+ * (testJsonInlineChunkedResults_withoutArrow) was removed because EnableArrow=0 is now deprecated
+ * and ignored — Arrow is always enabled on non-AIX platforms. The WireMock recordings were captured
+ * with JSON_ARRAY format which no longer matches the driver's requests. Chunked Arrow results are
+ * covered by MultiChunkExecutionIntegrationTests and SqlExecApiHybridResultsIntegrationTests.
+ */
+public class SqlExecApiIntegrationTests extends AbstractFakeServiceIntegrationTests {}