Propagate auth env to experimental.python subprocess (#5074)

## Why

When `bundle deploy` / `bundle validate` runs with a profile set (via
`--profile`, `DATABRICKS_CONFIG_PROFILE`, or `workspace.profile` in
`databricks.yml`) and the bundle uses `experimental.python`, the Python
subprocess does not inherit `DATABRICKS_CONFIG_PROFILE`. The Databricks
SDK inside Python then re-invokes the CLI via `databricks auth token
--host <host>` without a profile, and fails with a multi-profile
ambiguity error when `~/.databrickscfg` has several profiles sharing the
same host.

This is the remaining scenario on #4649. The Terraform path was fixed in
#4624; the `experimental.python` path was missed because the Python
mutator spawned its subprocess via `process.Background` without any auth
env setup.

## Changes

**Before:** `bundle/config/mutator/python/python_mutator.go` called
`process.Background` with no environment propagation. The Python
subprocess only saw whatever env vars the user happened to export;
`DATABRICKS_CONFIG_PROFILE` from `--profile` or `databricks.yml` was
silently dropped.

**Now:** The mutator calls `b.AuthEnv(ctx)` (same call the Terraform
path uses) and passes the resulting map through `process.WithEnvs(...)`.
The Python SDK and any subprocesses it spawns now see the same auth
configuration as the CLI itself.

## Test plan

- [x] New unit test `TestPythonMutator_propagatesAuthEnv` asserts that
`DATABRICKS_CONFIG_PROFILE` is present in the Python subprocess env when
`workspace.profile` is set.
- [x] Existing python mutator tests pass (`go test
./bundle/config/mutator/python/`).
- [x] Full bundle test suite passes (`go test ./bundle/...`).
- [x] `make checks` and `make lint` clean.

Author: simonfaltum

NEXT_CHANGELOG.md

@@ -1,13 +1,14 @@
 # NEXT CHANGELOG
 
-## Release v0.300.0
+## Release v0.299.2
 
 ### CLI
 
 * `databricks auth describe` now reports where U2M (`databricks-cli`) tokens are stored: `plaintext` (`~/.databricks/token-cache.json`) or `secure` (OS keyring), and the source of the choice (env var, config setting, or default).
 
 ### Bundles
 
+* Propagate authentication environment (including `DATABRICKS_CONFIG_PROFILE`) to the `experimental.python` subprocess so bundle validate/deploy no longer fails with a multi-profile host ambiguity error when several profiles in `~/.databrickscfg` share the same host.
 * Fixed `--force-pull` on `bundle summary` and `bundle open` so the flag bypasses the local state cache and reads state from the workspace.
 
 ### Dependency updates

acceptance/bundle/python/propagates-auth-env/.databrickscfg

@@ -0,0 +1,7 @@
+[my-profile]
+host = $DATABRICKS_HOST
+token = $DATABRICKS_TOKEN
+
+[other-profile]
+host = $DATABRICKS_HOST
+token = other-token

acceptance/bundle/python/propagates-auth-env/databricks.yml

@@ -0,0 +1,16 @@
+bundle:
+  name: my_project
+
+sync: {paths: []} # don't need to copy files
+
+python:
+  mutators:
+    - "mutators:capture_profile_env"
+
+workspace:
+  profile: my-profile
+
+resources:
+  jobs:
+    my_job:
+      name: "Job"

acceptance/bundle/python/propagates-auth-env/mutators.py

@@ -0,0 +1,13 @@
+from databricks.bundles.jobs import Job
+from databricks.bundles.core import job_mutator, Bundle
+import os
+
+
+@job_mutator
+def capture_profile_env(bundle: Bundle, job: Job) -> Job:
+    # The CLI must propagate DATABRICKS_CONFIG_PROFILE to the python subprocess
+    # so the Databricks SDK can disambiguate when multiple profiles share a host.
+    value = os.getenv("DATABRICKS_CONFIG_PROFILE", "<unset>")
+    with open("captured_env.txt", "w") as f:
+        f.write(value)
+    return job

acceptance/bundle/python/propagates-auth-env/out.test.toml

@@ -0,0 +1,8 @@
+
+>>> uv run [UV_ARGS] -q [CLI] bundle summary -o json
+{
+  "profile": "my-profile"
+}
+
+>>> cat captured_env.txt
+my-profile

acceptance/bundle/python/propagates-auth-env/output.txt

@@ -0,0 +1,18 @@
+
+# Two workspace profiles share the same host so picking one is meaningful.
+envsubst < .databrickscfg > out && mv out .databrickscfg
+export DATABRICKS_CONFIG_FILE=.databrickscfg
+unset DATABRICKS_HOST
+unset DATABRICKS_TOKEN
+unset DATABRICKS_CONFIG_PROFILE
+
+trace uv run $UV_ARGS -q $CLI bundle summary -o json | jq '{profile: .workspace.profile}'
+
+# The python mutator captures DATABRICKS_CONFIG_PROFILE from its subprocess env.
+# Without the fix, the CLI does not propagate the bundle's resolved profile,
+# so the SDK inside python re-invokes the CLI without a profile and fails on
+# multi-profile ambiguity.
+trace cat captured_env.txt
+echo ""
+
+rm -fr .databricks __pycache__ captured_env.txt

acceptance/bundle/python/propagates-auth-env/script

@@ -104,6 +104,7 @@ type runPythonMutatorOpts struct {
 	bundleRootPath string
 	pythonPath     string
 	loadLocations  bool
+	authEnv        map[string]string
 }
 
 // getOpts adapts deprecated PyDABs and upcoming Python configuration
@@ -217,6 +218,15 @@ func (m *pythonMutator) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagno
 		return diag.Errorf("Running Python code is not allowed when DATABRICKS_BUNDLE_RESTRICTED_CODE_EXECUTION is set")
 	}
 
+	// Propagate auth env so the Databricks SDK in the Python subprocess uses the
+	// same credentials as the CLI. In particular this carries DATABRICKS_CONFIG_PROFILE,
+	// which lets the CLI disambiguate profiles sharing the same host when the SDK
+	// re-invokes `databricks auth token --host <host>`.
+	authEnv, err := b.AuthEnv(ctx)
+	if err != nil {
+		return diag.FromErr(err)
+	}
+
 	// mutateDiags is used because Mutate returns 'error' instead of 'diag.Diagnostics'
 	var mutateDiags diag.Diagnostics
 	var result applyPythonOutputResult
@@ -238,6 +248,7 @@ func (m *pythonMutator) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagno
 			bundleRootPath: b.BundleRootPath,
 			pythonPath:     pythonPath,
 			loadLocations:  opts.loadLocations,
+			authEnv:        authEnv,
 		})
 		mutateDiags = diags
 		if diags.HasError() {
@@ -364,6 +375,7 @@ func (m *pythonMutator) runPythonMutator(ctx context.Context, root dyn.Value, op
 		process.WithDir(opts.bundleRootPath),
 		process.WithStderrWriter(stderrWriter),
 		process.WithStdoutWriter(stdoutWriter),
+		process.WithEnvs(opts.authEnv),
 	)
 	if processErr != nil {
 		logger.Debugf(ctx, "python mutator process failed: %s", processErr)