Merge remote-tracking branch 'origin/main' into hosttype-metadata

Author: tanmay-db

.github/actions/setup-jfrog-pypi/action.yml

@@ -0,0 +1,21 @@
+name: Setup JFrog PyPI proxy
+description: Authenticate to JFrog via OIDC and configure uv to use the db-pypi proxy
+
+runs:
+  using: composite
+  steps:
+    - name: Setup JFrog CLI with OIDC
+      id: jfrog
+      uses: jfrog/setup-jfrog-cli@279b1f629f43dd5bc658d8361ac4802a7ef8d2d5 # v4.9.1
+      env:
+        JF_URL: https://databricks.jfrog.io
+      with:
+        oidc-provider-name: github-actions
+
+    - name: Configure uv for JFrog
+      shell: bash
+      run: |
+        # Route uv package resolution through the JFrog PyPI proxy. Hardened
+        # runners block direct access to pypi.org, so all index queries go
+        # through this authenticated mirror instead.
+        echo "UV_INDEX_URL=https://${{ steps.jfrog.outputs.oidc-user }}:${{ steps.jfrog.outputs.oidc-token }}@databricks.jfrog.io/artifactory/api/pypi/db-pypi/simple" >> "$GITHUB_ENV"

.github/workflows/push.yml

@@ -6,13 +6,17 @@ on:
   merge_group:
     types: [checks_requested]
 
+permissions:
+  id-token: write
+  contents: read
+
 jobs:
   tests-ubuntu:
     uses: ./.github/workflows/test.yml
     strategy:
       fail-fast: false
       matrix:
-        pyVersion: [ '3.8', '3.9', '3.10', '3.11', '3.12' ]
+        pyVersion: [ '3.10', '3.11', '3.12' ]
     with:
       os: ubuntu-latest
       pyVersion: ${{ matrix.pyVersion }}
@@ -22,7 +26,7 @@ jobs:
       strategy:
         fail-fast: false
         matrix:
-          pyVersion: [ '3.9', '3.10', '3.11', '3.12' ]
+          pyVersion: [ '3.10', '3.11', '3.12' ]
       with:
         os: windows-latest
         pyVersion: ${{ matrix.pyVersion }}
@@ -39,9 +43,20 @@ jobs:
         with:
           version: "0.6.5"
 
+      - name: Setup JFrog PyPI proxy
+        uses: ./.github/actions/setup-jfrog-pypi
+
+      - name: Re-lock for JFrog
+        run: uv lock
+
       - name: Format all files
         run: make dev fmt
 
+      - name: Restore lockfiles and fix proxy URLs
+        run: |
+          git checkout -- '*.lock'
+          make fix-lockfile
+
       - name: Fail on differences
         run: git diff --exit-code
 
@@ -57,5 +72,11 @@ jobs:
         with:
           version: "0.6.5"
 
+      - name: Setup JFrog PyPI proxy
+        uses: ./.github/actions/setup-jfrog-pypi
+
+      - name: Re-lock for JFrog
+        run: uv lock
+
       - name: Check MANIFEST.in
         run: make dev && uv run check-manifest .

.github/workflows/test.yml

@@ -15,6 +15,11 @@ jobs:
     strategy:
         fail-fast: false
     runs-on: ${{ inputs.os }}
+
+    permissions:
+      id-token: write
+      contents: read
+
     steps:
       - name: Checkout
         uses: actions/checkout@ee0669bd1cc54295c223e0bb666b733df41de1c5 # v2.7.0
@@ -28,6 +33,15 @@ jobs:
           version: "0.6.5"
           python-version: ${{ inputs.pyVersion }}
 
+      - name: Setup JFrog PyPI proxy
+        uses: ./.github/actions/setup-jfrog-pypi
+
+      # Re-lock so uv.lock matches the JFrog-configured UV_INDEX_URL.
+      # Keeps the same versions; only registry URLs change. Ephemeral (not committed).
+      - name: Re-lock for JFrog
+        shell: bash
+        run: uv lock
+
       - name: Run tests
         run: make dev test
 

Makefile

@@ -30,5 +30,13 @@ benchmark:
 coverage: test
 	open htmlcov/index.html
 
+fix-lockfile:
+	@# Replace JFrog proxy URLs with public equivalents in lockfiles.
+	@# Prevents proxy URLs from being accidentally committed.
+	find . -type f -name '*.lock' -not -path './.github/*' \
+	  -exec sed -i 's|databricks\.jfrog\.io/artifactory/api/pypi/db-pypi/simple|pypi.org/simple|g' {} +
+	find . -type f -name '*.lock' -not -path './.github/*' \
+	  -exec sed -i 's|databricks\.jfrog\.io/artifactory/api/pypi/db-pypi/packages|files.pythonhosted.org|g' {} +
+
 clean:
 	rm -fr dist *.egg-info .pytest_cache build htmlcov .venv

NEXT_CHANGELOG.md

@@ -3,6 +3,7 @@
 ## Release v0.103.0
 
 ### New Features and Improvements
+* Add support for unified hosts. A single configuration profile can now be used for both account-level and workspace-level operations when the host supports it and both `account_id` and `workspace_id` are available. The `experimental_is_unified_host` flag has been removed; unified host detection is now automatic.
 * Accept `DATABRICKS_OIDC_TOKEN_FILEPATH` environment variable for consistency with other Databricks SDKs (Go, CLI, Terraform). The previous `DATABRICKS_OIDC_TOKEN_FILE` is still supported as an alias.
 
 ### Security
@@ -11,6 +12,9 @@
 
 ### Documentation
 
+### Breaking Changes
+* Drop support for Python 3.8 and 3.9. The minimum supported Python version is now 3.10, in line with the oldest supported Databricks Runtime LTS (DBR 13.3).
+
 ### Internal Changes
 * Replace the async-disabling mechanism on token refresh failure with a 1-minute retry backoff. Previously, a single failed async refresh would disable proactive token renewal until the token expired. Now, the SDK waits a short cooldown period and retries, improving resilience to transient errors.
 * Extract `_resolve_profile` to simplify config file loading and improve `__settings__` error messages.
@@ -25,4 +29,4 @@
 * Add `cascade` field for `databricks.sdk.service.pipelines.DeletePipelineRequest`.
 * Add `default_branch` field for `databricks.sdk.service.postgres.ProjectSpec`.
 * Add `default_branch` field for `databricks.sdk.service.postgres.ProjectStatus`.
-* Add `ingress` and `ingress_dry_run` fields for `databricks.sdk.service.settings.AccountNetworkPolicy`.
+* Add `ingress` and `ingress_dry_run` fields for `databricks.sdk.service.settings.AccountNetworkPolicy`.

README.md

@@ -92,6 +92,7 @@ The conventional name for the variable that holds the workspace-level client of
 ### In this section
 
 - [Default authentication flow](#default-authentication-flow)
+- [Unified host support](#unified-host-support)
 - [Databricks native authentication](#databricks-native-authentication)
 - [Azure native authentication](#azure-native-authentication)
 - [Overriding .databrickscfg](#overriding-databrickscfg)
@@ -107,10 +108,19 @@ in the following order, until it succeeds:
 
 1. [Databricks native authentication](#databricks-native-authentication)
 2. [Azure native authentication](#azure-native-authentication)
+3. [GCP native authentication](#google-cloud-platform-native-authentication)
 4. If the SDK is unsuccessful at this point, it returns an authentication error and stops running.
 
-You can instruct the Databricks SDK for Python to use a specific authentication method by setting the `auth_type` argument
-as described in the following sections.
+Each authentication method requires specific configuration attributes (e.g., `token` for PAT auth, `azure_client_id` for Azure service principal auth). The SDK automatically detects the cloud provider and skips authentication methods whose required configuration attributes are not present. This means that Azure-specific methods like `azure-cli` are automatically skipped when connecting to an AWS or GCP workspace, and vice versa for GCP-specific methods.
+
+To force a specific authentication method instead of relying on auto-detection, set the `auth_type` argument:
+
+```python
+from databricks.sdk import WorkspaceClient
+# Force Azure CLI authentication — skip all other methods
+w = WorkspaceClient(host='https://mycompany.databricks.com', auth_type='azure-cli', cloud='AZURE')
+```
+This is useful when your environment has credentials for multiple authentication methods and you want to ensure a specific one is used or when auto detection is not accurate.
 
 For each authentication method, the SDK searches for compatible authentication credentials in the following locations,
 in the following order. Once the SDK finds a compatible set of credentials that it can use, it stops searching:
@@ -125,6 +135,38 @@ in the following order. Once the SDK finds a compatible set of credentials that
 
 Depending on the Databricks authentication method, the SDK uses the following information. Presented are the `WorkspaceClient` and `AccountClient` arguments (which have corresponding `.databrickscfg` file fields), their descriptions, and any corresponding environment variables.
 
+### Unified host support
+
+Certain Databricks host types support both account-level and workspace-level API operations from a single endpoint. When using such a unified host, a single configuration profile can be used to create both `WorkspaceClient` and `AccountClient` instances without changing the `host`.
+
+For this to work, the following conditions must be met:
+
+1. The host must support unified operations.
+2. Both `account_id` and `workspace_id` must be available — either set explicitly in the configuration or auto-discovered.
+
+When both values are present, the SDK uses `workspace_id` to route workspace-level requests and `account_id` to route account-level requests, all through the same host.
+
+```ini
+# .databrickscfg
+[unified]
+host         = https://mycompany.databricks.com
+account_id   = 00000000-0000-0000-0000-000000000000
+workspace_id = 1234567890
+```
+
+```python
+from databricks.sdk import WorkspaceClient, AccountClient
+
+# Both clients share the same host and profile
+w = WorkspaceClient(profile='unified')
+a = AccountClient(profile='unified')
+
+# A WorkspaceClient for a different workspace under the same host and account
+w = WorkspaceClient(profile='unified', workspace_id='2345678901')
+```
+
+If the host supports it, `account_id` and `workspace_id` may be auto-discovered, reducing the required explicit configuration.
+
 ### Databricks native authentication
 
 By default, the Databricks SDK for Python initially tries [Databricks token authentication](https://docs.databricks.com/dev-tools/api/latest/authentication.html) (`auth_type='pat'` argument). If the SDK is unsuccessful, it then tries Workload Identity Federation (WIF). See [Supported WIF](https://docs.databricks.com/aws/en/dev-tools/auth/oauth-federation-provider) for the supported JWT token providers.
@@ -133,10 +175,15 @@ By default, the Databricks SDK for Python initially tries [Databricks token auth
 - For Databricks OIDC authentication, you must provide the `host`, `client_id` and `token_audience` _(optional)_ either directly, through the corresponding environment variables, or in your `.databrickscfg` configuration file.
 - For Azure DevOps OIDC authentication, the `token_audience` is irrelevant as the audience is always set to `api://AzureADTokenExchange`. Also, the `System.AccessToken` pipeline variable required for OIDC request must be exposed as the `SYSTEM_ACCESSTOKEN` environment variable, following [Pipeline variables](https://learn.microsoft.com/en-us/azure/devops/pipelines/build/variables?view=azure-devops&tabs=yaml#systemaccesstoken)
 
+During initialization, the SDK automatically resolves missing configuration fields (`account_id`, `workspace_id`, `cloud`, and `discovery_url`). Any explicitly provided values take precedence and are never overwritten. If the auto discovery fails, the SDK falls back to the explicit configuration. It is recommended to always set explicit configuration.
+
 | Argument         | Description                                                                                                                                                                                                                                                               | Environment variable    |
 |------------------|---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|-------------------------|
-| `host`           | _(String)_ The Databricks host URL for either the Databricks workspace endpoint or the Databricks accounts endpoint.                                                                                                                                                      | `DATABRICKS_HOST`       |     
-| `account_id`     | _(String)_ The Databricks account ID for the Databricks accounts endpoint. Only has effect when `Host` is either `https://accounts.cloud.databricks.com/` _(AWS)_, `https://accounts.azuredatabricks.net/` _(Azure)_, or `https://accounts.gcp.databricks.com/` _(GCP)_.  | `DATABRICKS_ACCOUNT_ID` |
+| `host`           | _(String)_ The Databricks host URL for either the Databricks workspace endpoint or the Databricks accounts endpoint.                                                                                                                                                      | `DATABRICKS_HOST`       |
+| `account_id`     | _(String)_ The Databricks account ID for the Databricks accounts endpoint. Auto-discovered if not provided.  | `DATABRICKS_ACCOUNT_ID` |
+| `workspace_id`   | _(String)_ The Databricks workspace ID for the Databricks workspace endpoint. Auto-discovered if not provided. | `DATABRICKS_WORKSPACE_ID` |
+| `cloud`          | _(String)_ The cloud provider for the Databricks workspace (`AWS`, `AZURE`, or `GCP`). Auto-discovered if not provided. When set, `is_aws`, `is_azure`, and `is_gcp` use this value directly instead of inferring from hostname. | `DATABRICKS_CLOUD` |
+| `discovery_url`  | _(String)_ The OpenID Connect discovery URL. Auto-discovered if not provided. When set, OIDC endpoints are fetched directly from this URL instead of using the default host-based well-known endpoint logic. | `DATABRICKS_DISCOVERY_URL` |
 | `token`          | _(String)_ The Databricks personal access token (PAT) _(AWS, Azure, and GCP)_ or Azure Active Directory (Azure AD) token _(Azure)_.                                                                                                                                       | `DATABRICKS_TOKEN`      |
 | `client_id`      | _(String)_ The Databricks Service Principal Application ID.                                                                                                                                                                                                               | `DATABRICKS_CLIENT_ID`  |
 | `token_audience` | _(String)_ When using Workload Identity Federation, the audience to specify when fetching an ID token from the ID token supplier.                                                                                                                                         | `TOKEN_AUDIENCE`        |
@@ -528,7 +575,7 @@ useragent.with_partner("partner-xyz")
 
 `with_product()` can be used to define the name and version of the product that is built with the Databricks SDK for Python. The product name has the same restrictions as the partner name above, and the product version must be a valid [SemVer](https://semver.org/). Subsequent calls to `with_product()` replace the original product with the new user-specified one.
 
-```go
+```python
 from databricks.sdk import useragent
 useragent.with_product("databricks-example-product", "1.2.0")
 ```

databricks/sdk/config.py

@@ -5,7 +5,6 @@
 import os
 import pathlib
 import re
-import sys
 import urllib.parse
 from typing import Dict, Iterable, List, Optional
 
@@ -92,14 +91,11 @@ class Config:
     account_id: str = ConfigAttribute(env="DATABRICKS_ACCOUNT_ID")
     workspace_id: str = ConfigAttribute(env="DATABRICKS_WORKSPACE_ID")
 
-    # Experimental flag to indicate if the host is a unified host (supports both workspace and account APIs)
-    experimental_is_unified_host: bool = ConfigAttribute(env="DATABRICKS_EXPERIMENTAL_IS_UNIFIED_HOST")
-
-    # [Experimental] Cloud provider. When set, is_aws/is_azure/is_gcp use this value directly
+    # Cloud provider. When set, is_aws/is_azure/is_gcp use this value directly
     # instead of inferring from hostname. Populated automatically from /.well-known/databricks-config.
     cloud: Cloud = ConfigAttribute(env="DATABRICKS_CLOUD", transform=_parse_cloud)
 
-    # [Experimental] OpenID Connect discovery URL. When set, OIDC endpoints are fetched directly
+    # OpenID Connect discovery URL. When set, OIDC endpoints are fetched directly
     # from this URL instead of the default host-type-based well-known endpoint logic.
     discovery_url: str = ConfigAttribute(env="DATABRICKS_DISCOVERY_URL")
 
@@ -615,15 +611,9 @@ def attributes(cls) -> Iterable[ConfigAttribute]:
         """Returns a list of Databricks SDK configuration metadata"""
         if hasattr(cls, "_attributes"):
             return cls._attributes
-        if sys.version_info[1] >= 10:
-            import inspect
+        import inspect
 
-            anno = inspect.get_annotations(cls)
-        else:
-            # Python 3.7 compatibility: getting type hints require extra hop, as described in
-            # "Accessing The Annotations Dict Of An Object In Python 3.9 And Older" section of
-            # https://docs.python.org/3/howto/annotations.html
-            anno = cls.__dict__["__annotations__"]
+        anno = inspect.get_annotations(cls)
         attrs = []
         for name, v in cls.__dict__.items():
             if type(v) != ConfigAttribute:

databricks/sdk/credentials_provider.py

@@ -160,8 +160,21 @@ def runtime_native_auth(cfg: "Config") -> Optional[CredentialsProvider]:
     # runtime and no config variables are set.
     from databricks.sdk.runtime import (init_runtime_legacy_auth,
                                         init_runtime_native_auth,
+                                        init_runtime_native_unified,
                                         init_runtime_repl_auth)
 
+    # Try the unified provider first (returns host, account_id, workspace_id, inner).
+    if init_runtime_native_unified is not None:
+        host, account_id, workspace_id, inner = init_runtime_native_unified()
+        if host is not None:
+            cfg.host = host
+            cfg.account_id = account_id
+            cfg.workspace_id = workspace_id
+            logger.debug("[init_runtime_native_unified] runtime native auth configured")
+            return inner
+        logger.debug("[init_runtime_native_unified] no host detected")
+
+    # Fall back to legacy providers (return host, inner).
     for init in [
         init_runtime_native_auth,
         init_runtime_repl_auth,

databricks/sdk/mixins/files.py

@@ -9,7 +9,6 @@
 import platform
 import re
 import shutil
-import sys
 import xml.etree.ElementTree as ET
 from abc import ABC, abstractmethod
 from collections import deque
@@ -430,10 +429,7 @@ def delete(self, *, recursive=False):
                     _LocalPath(leaf.path).delete()
             self._path.rmdir()
         else:
-            kw = {}
-            if sys.version_info[:2] > (3, 7):
-                kw["missing_ok"] = True
-            self._path.unlink(**kw)
+            self._path.unlink(missing_ok=True)
 
     def __repr__(self) -> str:
         return f"<_LocalPath {self._path}>"

databricks/sdk/runtime/__init__.py

@@ -23,6 +23,13 @@
 ]
 
 # DO NOT MOVE THE TRY-CATCH BLOCK BELOW AND DO NOT ADD THINGS BEFORE IT! WILL MAKE TEST FAIL.
+try:
+    from dbruntime.sdk_credential_provider import init_runtime_native_unified
+
+    logger.debug("runtime SDK credential provider (unified) available")
+except ImportError:
+    init_runtime_native_unified = None
+
 try:
     # We don't want to expose additional entity to user namespace, so
     # a workaround here for exposing required information in notebook environment
@@ -34,6 +41,7 @@
     init_runtime_native_auth = None
 
 globals()["init_runtime_native_auth"] = init_runtime_native_auth
+globals()["init_runtime_native_unified"] = init_runtime_native_unified
 
 
 def init_runtime_repl_auth():