Merge branch 'main' into ssh-connect-budget-policy

Author: rclarey

.github/workflows/check.yml

@@ -40,17 +40,14 @@ jobs:
         with:
           version: "0.8.9"
 
-      - name: Run Go lint checks (does not include formatting checks)
+      - name: Run Go and Python lint checks (does not include formatting checks)
+        # `task lint` runs golangci-lint and `ruff check` (lint-python). ruff is
+        # provided by uvx, installed above; formatting is checked by `task fmt` below.
         run: go tool -modfile=tools/task/go.mod task lint
 
-      - name: Run ruff (Python linter and formatter)
-        uses: astral-sh/ruff-action@0ce1b0bf8b818ef400413f810f8a11cdbda0034b # v4.0.0
-        with:
-          version: "0.9.1"
-          args: "format --check"
-
-      - name: "task fmt: Python and Go formatting"
-        # Python formatting is already checked above, but this also checks Go and YAML formatting
+      - name: "task fmt: Python, Go and YAML formatting"
+        # `task fmt` runs ruff format (Python) plus Go and YAML formatters; the diff
+        # check fails the build if anything was left unformatted.
         run: |
           go tool -modfile=tools/task/go.mod task fmt
           git diff --exit-code

NEXT_CHANGELOG.md

@@ -7,8 +7,17 @@
 ### CLI
 * `ssh connect` now supports specifying a serverless usage policy with `--usage-policy-id` ([#5781](https://github.com/databricks/cli/pull/5781)).
 
+ * `ssh connect` now accepts a `--base-environment` flag to run a serverless session on a custom base environment. It takes an `env.yaml` path, a `workspace-base-environments/...` resource ID, or a base environment display name, and is rejected together with `--environment-version` or `--cluster` ([#5706](https://github.com/databricks/cli/pull/5706)).
+ * `databricks aitools install` is now plugin-first: it installs the Databricks plugin through each agent's own CLI (Claude Code, Codex, GitHub Copilot) instead of copying raw skill files. Agents without a plugin (OpenCode, Antigravity) still get skill files, and Cursor prints the `/add-plugin databricks` step. Use `--skills-only` to force raw skill files for every agent, or `--path <dir>` to write skills to a directory ([#5738](https://github.com/databricks/cli/pull/5738)).
+ * `databricks labs list` now only shows projects that can be installed ([#5560](https://github.com/databricks/cli/pull/5560)).
+
 ### Bundles
 
+ * direct: Fixed persistent drift on `model_serving_endpoints` caused by the `traffic_config` field ([#5708](https://github.com/databricks/cli/pull/5708)).
+ * direct: Fix spurious update when `apply_policy_default_values: true` is set on job task, for-each-task, or job cluster new_cluster ([#5731](https://github.com/databricks/cli/pull/5731)). Also fix spurious updates for for-each-task clusters due to missing backend defaults for `data_security_mode`, `node_type_id`, `driver_node_type_id`, `driver_instance_pool_id`, `enable_elastic_disk`, and `enable_local_disk_encryption`.
+ * direct: Cluster resize now falls back to regular update if resize fails due to `INVALID_STATE` ([#5716](https://github.com/databricks/cli/pull/5716)).
+ * Fixed `bundle deployment migrate` failing on `model_serving_endpoints`/`database_instances` with permissions (regression since v1.5.0) ([#5775](https://github.com/databricks/cli/pull/5775)).
+
 ### Dependency updates
 
 ### API Changes

Taskfile.yml

@@ -4,6 +4,9 @@ vars:
   # Absolute path so tasks with `dir:` (lint-go-tools, lint-go-codegen) can use it.
   GO_TOOL: go tool -modfile={{.ROOT_DIR}}/tools/go.mod
   EXE_EXT: '{{if eq OS "windows"}}.exe{{end}}'
+  # Pinned ruff (linter + formatter). Keep in sync with the version in
+  # .github/workflows/check.yml and python/Taskfile.yml.
+  RUFF: uvx ruff@0.15.17
   TEST_PACKAGES: ./acceptance/internal ./libs/... ./internal/... ./cmd/... ./bundle/... ./experimental/ssh/... .
   ACCEPTANCE_TEST_FILTER: ""
   # Single brace-expansion glob covering every //go:embed target in the repo,
@@ -75,14 +78,29 @@ tasks:
   # speed; `./task all` uses the full variants.
 
   lint:
-    desc: Lint all Go files (root + tools + codegen modules)
+    desc: Lint all Go files (root + tools + codegen modules) and Python scripts
     cmds:
       - task: lint-go
+      - task: lint-python
 
   lint-q:
-    desc: Lint changed Go files in root module (diff vs main, with --fix)
+    desc: Lint changed Go files in root module (diff vs main, with --fix) and Python scripts
     cmds:
       - task: lint-q-go-root
+      - task: lint-python
+
+  lint-python:
+    desc: Lint first-party Python scripts with ruff
+    sources:
+      - "**/*.py"
+      - ruff.toml
+    cmds:
+      # Lints the whole repo; fixtures and generated outputs are skipped via
+      # `lint.exclude` / `extend-exclude` in ruff.toml. --no-cache: some excluded
+      # acceptance fixtures carry their own pyproject.toml/ruff.toml (config
+      # roots), so without it ruff writes a .ruff_cache into those checked-in
+      # dirs, polluting the goldens and breaking the acceptance tests.
+      - "{{.RUFF}} check --no-cache ."
 
   # `golangci-lint run` typechecks, so it stops at go.mod boundaries. We have
   # one task per Go module; `lint-go` composes them to cover the whole repo.
@@ -166,14 +184,19 @@ tasks:
     deps: ['fmt-python', 'fmt-q-go', 'fmt-yaml']
 
   fmt-python:
-    desc: Format Python files
+    desc: Lint-fix and format Python files
     sources:
       - "**/*.py"
+      - ruff.toml
     cmds:
-      # Pinned to match the version used by the `ruff format --check` step in
+      # Auto-fix the lint issues that ruff can fix (whole repo, minus lint.exclude);
+      # `|| true` so unfixable findings don't abort fmt (lint-python reports them).
+      # --no-cache: see lint-python (avoids polluting acceptance fixture goldens).
+      - "{{.RUFF}} check --no-cache --fix . || true"
+      # Pinned (via the RUFF var) to match the `ruff format --check` step in
       # .github/workflows/check.yml — newer ruff versions reformat files that
       # CI considers already formatted.
-      - "uvx ruff@0.9.1 format -n"
+      - "{{.RUFF}} format -n"
 
   # `golangci-lint fmt` walks the filesystem and doesn't typecheck, so it
   # formats files across all nested modules (tools/, bundle/internal/tf/codegen/)
@@ -694,11 +717,11 @@ tasks:
           -- -parallel 4 -timeout=12h
 
   integration-short:
-    desc: Run short integration tests
+    desc: Run integration tests except the slow ones (skips CloudSlow via -short)
     deps: [install-pythons]
     cmds:
       - |
-        DATABRICKS_TEST_SKIPLOCAL=1 VERBOSE_TEST=1 \
+        VERBOSE_TEST=1 \
         go run -modfile=tools/go.mod ./tools/testrunner/main.go \
           {{.GO_TOOL}} gotestsum \
           --format github-actions \
@@ -707,6 +730,20 @@ tasks:
           --packages "./acceptance ./integration/..." \
           -- -parallel 4 -timeout=12h -short
 
+  integration-short-skiplocal:
+    desc: Run integration tests for PRs (skips CloudSlow and tests with testserver coverage)
+    deps: [install-pythons]
+    cmds:
+      - |
+        DATABRICKS_TEST_SKIPLOCAL=1 VERBOSE_TEST=1 \
+        go run -modfile=tools/go.mod ./tools/testrunner/main.go \
+          {{.GO_TOOL}} gotestsum \
+          --format github-actions \
+          --rerun-fails \
+          --jsonfile output.json \
+          --packages "./acceptance ./integration/..." \
+          -- -parallel 4 -timeout=2h -short
+
   dbr-integration:
     desc: Run DBR acceptance tests on Databricks Runtime
     deps: [install-pythons]

acceptance/acceptance_test.go

@@ -45,6 +45,7 @@ var (
 	LogRequests     bool
 	LogConfig       bool
 	UseVersion      string
+	CLIPath         string
 	WorkspaceTmpDir bool
 	OnlyOutTestToml bool
 	Subset          bool
@@ -75,6 +76,7 @@ func init() {
 	flag.BoolVar(&LogRequests, "logrequests", false, "Log request and responses from testserver")
 	flag.BoolVar(&LogConfig, "logconfig", false, "Log merged for each test case")
 	flag.StringVar(&UseVersion, "useversion", "", "Download previously released version of CLI and use it to run the tests")
+	flag.StringVar(&CLIPath, "clipath", "", "Use the CLI binary at this path instead of building from source (e.g. a CLI built from main for regression comparison)")
 
 	// DABs in the workspace runs on the workspace file system. This flags does the same for acceptance tests
 	// to simulate an identical environment.
@@ -191,17 +193,27 @@ func hasRunFilter() bool {
 // requirePrerequisites verifies external tool prerequisites before doing any
 // work, so a stale toolchain fails fast with an actionable message instead of
 // producing confusing diffs deep into the run.
-func requirePrerequisites(t *testing.T) {
-	// Scripts use jq 1.7 features (the pick/1 builtin and the `.foo.[]` iteration syntax).
-	internal.RequireJQ(t, "1.7")
-	// uv builds the databricks-bundles wheel and provides the test interpreter
-	// via `uv python find`, which landed in the 0.3 line.
-	internal.RequireUV(t, "0.4")
-	// ruff 0.9.1 is pinned across the repo (python/pyproject.toml, Taskfile.yml);
-	// the check-formatting test's golden output assumes its formatter behavior.
-	internal.RequireRuff(t, "0.9.1")
-	// Acceptance scripts import the stdlib tomllib module, added in Python 3.11.
-	internal.EnsurePython(t, "3.11")
+//
+// It reports whether all checks passed; a failure surfaces as
+// TestAccept/prerequisites rather than a bare TestAccept.
+//
+// On DBR the serverless image only provides what the test archive ships, so
+// every tool required here must also be bundled in internal/testarchive. When
+// adding a new RequireX prerequisite, add a matching downloader there too, or
+// DBR runs will fail this check before any test runs.
+func requirePrerequisites(t *testing.T) bool {
+	return t.Run("prerequisites", func(t *testing.T) {
+		// Scripts use jq 1.7 features (the pick/1 builtin and the `.foo.[]` iteration syntax).
+		internal.RequireJQ(t, "1.7")
+		// uv builds the databricks-bundles wheel and provides the test interpreter
+		// via `uv python find`, which landed in the 0.3 line.
+		internal.RequireUV(t, "0.4")
+		// ruff 0.9.1 is pinned across the repo (python/pyproject.toml, Taskfile.yml);
+		// the check-formatting test's golden output assumes its formatter behavior.
+		internal.RequireRuff(t, "0.9.1")
+		// Acceptance scripts import the stdlib tomllib module, added in Python 3.11.
+		internal.RequirePython(t, "3.11")
+	})
 }
 
 func testAccept(t *testing.T, inprocessMode bool, singleTest string) int {
@@ -245,7 +257,14 @@ func testAccept(t *testing.T, inprocessMode bool, singleTest string) int {
 		os.Unsetenv(v) //nolint:usetesting // t.Setenv cannot unset
 	}
 
-	requirePrerequisites(t)
+	if !requirePrerequisites(t) {
+		// Don't run the suite against a stale toolchain; the failed subtest
+		// has already marked the parent test as failed.
+		return 0
+	}
+	// Run after the version check passed; must use the top-level t so the PATH
+	// change survives for the rest of the run.
+	internal.ConfigurePython(t, "3.11")
 
 	buildDir := getBuildDir(t, cwd, runtime.GOOS, runtime.GOARCH)
 
@@ -285,7 +304,11 @@ func testAccept(t *testing.T, inprocessMode bool, singleTest string) int {
 		t.Setenv("CMD_SERVER_URL", cmdServer.URL)
 		execPath = filepath.Join(cwd, "bin", "callserver.py")
 	} else {
-		if UseVersion != "" {
+		if CLIPath != "" {
+			// Use a prebuilt binary (e.g. a CLI built from main) instead of building
+			// from the current source, so the test infra and tests stay on this branch.
+			execPath = CLIPath
+		} else if UseVersion != "" {
 			version := UseVersion
 			if version == "latest" {
 				version = resolveLatestVersion(t, buildDir)

acceptance/bin/add_repl.py

@@ -5,8 +5,8 @@
 If entry already exists, it'll add suffix in _<number> format.
 """
 
-import os
 import argparse
+import os
 from pathlib import Path
 
 ACC_REPLS = Path(os.environ["TEST_TMP_DIR"]) / "ACC_REPLS"

acceptance/bin/assert_exists.py

@@ -1,5 +1,6 @@
 #!/usr/bin/env python3
-import os, sys
+import os
+import sys
 
 errors = 0
 

acceptance/bin/assert_not_exists.py

@@ -1,5 +1,6 @@
 #!/usr/bin/env python3
-import os, sys
+import os
+import sys
 
 errors = 0
 

acceptance/bin/benchmark.py

@@ -1,11 +1,11 @@
 #!/usr/bin/env python3
 import argparse
-import subprocess
-import time
+import json
+import os
 import statistics
+import subprocess
 import sys
-import os
-import json
+import time
 
 try:
     import resource
@@ -19,7 +19,7 @@ def run_benchmark(command, warmup, runs):
 
     for i in range(runs):
         # double fork to reset max statistics like ru_maxrss
-        cp = subprocess.run([sys.executable, sys.argv[0], "--once"] + command, stdout=subprocess.PIPE)
+        cp = subprocess.run([sys.executable, sys.argv[0], "--once", *command], stdout=subprocess.PIPE)
         if cp.returncode != 0:
             sys.exit(cp.returncode)
 
@@ -54,7 +54,7 @@ def run_benchmark(command, warmup, runs):
 
 
 def run_once(command):
-    if len(command) == 1 and " " in command[0] or ">" in command[0]:
+    if (len(command) == 1 and " " in command[0]) or ">" in command[0]:
         shell = True
         command = command[0]
     else:

acceptance/bin/browser.py

@@ -6,8 +6,8 @@
 Usage: browser.py <url>
 """
 
-import urllib.request
 import sys
+import urllib.request
 
 if len(sys.argv) < 2:
     sys.stderr.write("Usage: browser.py <url>\n")

acceptance/bin/callserver.py

@@ -1,7 +1,7 @@
 #!/usr/bin/env python3
-import sys
-import os
 import json
+import os
+import sys
 import urllib.request
 from urllib.parse import urlencode