Skip auth-steering env vars gap-filling a selected profile

The final loader in ProfileAuthLoaders was the SDK's config.ConfigAttributes,
which re-reads every still-empty attribute from the environment after the
profile is loaded. That let host/routing/auth-steering env vars (auth_type,
discovery_url, audience, cloud, workspace_id, account_id) shadow an explicitly
selected profile, e.g. DATABRICKS_AUTH_TYPE=basic forcing basic auth over a
profile's PAT (#5096).

Replace it with resolveAuthGapFromEnv, a filtered env loader that skips
nonAuthEnvSkipAttrs but still gap-fills real auth attrs, so a complete
conflicting env auth method surfaces as an error rather than being silently
dropped. Add an end-to-end ProfileAuthLoaders test covering the steering-env
repro.

Author: radakam

libs/databrickscfg/loader.go

@@ -16,22 +16,31 @@ var ResolveProfileFromHost = profileFromHostLoader{}
 
 // ResolveNonAuthFromEnv reads non-auth, non-host config from environment
 // variables. See ProfileAuthLoaders for how it fits into the chain.
-var ResolveNonAuthFromEnv = nonAuthEnvLoader{}
+var ResolveNonAuthFromEnv = envLoader{name: "environment (excluding auth)", skipAuth: true}
+
+// resolveAuthGapFromEnv gap-fills auth attributes the profile left empty from
+// environment variables. See ProfileAuthLoaders for how it fits into the chain.
+var resolveAuthGapFromEnv = envLoader{name: "environment (auth gap-fill)", skipAuth: false}
 
 // ProfileAuthLoaders is the loader chain for an explicitly selected profile
 // (--profile or a bundle's workspace.profile). Unlike the SDK's default
 // env-first chain, the profile wins over auth env vars (#5096):
 //
 //  1. ResolveNonAuthFromEnv: non-auth env attrs (e.g. cluster_id), env-wins.
 //  2. ConfigFile: the selected profile (host, routing, auth).
-//  3. ConfigAttributes: gap-fills only fields the profile left empty.
+//  3. resolveAuthGapFromEnv: gap-fills auth attrs the profile left empty from
+//     env, so a complete conflicting env auth method still surfaces as an error
+//     rather than being silently dropped. It skips nonAuthEnvSkipAttrs (like the
+//     env-first pass) so host/routing/auth-steering attrs come from the profile
+//     only; using the SDK's ConfigAttributes here would let those env vars shadow
+//     the profile (e.g. DATABRICKS_AUTH_TYPE=basic over a PAT profile).
 //
 // A profile from DATABRICKS_CONFIG_PROFILE keeps env-first precedence; only an
 // out-of-band profile name triggers this chain.
 var ProfileAuthLoaders = []config.Loader{
 	ResolveNonAuthFromEnv,
 	config.ConfigFile,
-	config.ConfigAttributes,
+	resolveAuthGapFromEnv,
 }
 
 var errNoMatchingProfiles = errors.New("no matching config profiles found")
@@ -96,16 +105,28 @@ var nonAuthEnvSkipAttrs = map[string]bool{
 	"cloud":         true,
 }
 
-type nonAuthEnvLoader struct{}
+// envLoader reads config attributes from environment variables. It always skips
+// nonAuthEnvSkipAttrs so host/routing/auth-steering attrs come from the selected
+// profile only (#5096). When skipAuth is true it additionally skips the SDK's
+// auth attributes, for the env-first pass that runs before the profile.
+type envLoader struct {
+	name     string
+	skipAuth bool
+}
 
-func (nonAuthEnvLoader) Name() string {
-	return "environment (excluding auth)"
+func (l envLoader) Name() string {
+	return l.name
 }
 
-func (nonAuthEnvLoader) Configure(cfg *config.Config) error {
+func (l envLoader) Configure(cfg *config.Config) error {
 	for _, attr := range config.ConfigAttributes {
-		// Host and auth come from the profile (config file), not env.
-		if nonAuthEnvSkipAttrs[attr.Name] || attr.HasAuthAttribute() {
+		// Host/routing/auth-steering come from the profile (config file), not env.
+		if nonAuthEnvSkipAttrs[attr.Name] {
+			continue
+		}
+		// The env-first pass leaves auth for the profile; the gap-fill pass runs
+		// after the profile and fills only auth attrs the profile didn't set.
+		if l.skipAuth && attr.HasAuthAttribute() {
 			continue
 		}
 		// Don't overwrite an already-set value (SDK loader semantics).

libs/databrickscfg/loader_test.go

@@ -65,6 +65,36 @@ func TestResolveNonAuthFromEnvSkipsHostAndAuth(t *testing.T) {
 	assert.Equal(t, "env-cluster", cfg.ClusterID)
 }
 
+func TestProfileAuthLoadersProfileWinsOverSteeringEnv(t *testing.T) {
+	// A PAT profile plus auth-steering/routing env vars. The profile owns
+	// host/routing/auth-steering, so these env vars must not shadow it (#5096).
+	// In particular DATABRICKS_AUTH_TYPE=basic must not force basic auth over the
+	// profile's PAT.
+	t.Setenv("DATABRICKS_AUTH_TYPE", "basic")
+	t.Setenv("DATABRICKS_DISCOVERY_URL", "https://discovery.env.test")
+	t.Setenv("DATABRICKS_CLOUD", "azure")
+	t.Setenv("DATABRICKS_WORKSPACE_ID", "env-workspace")
+	t.Setenv("DATABRICKS_ACCOUNT_ID", "env-account")
+
+	cfg := config.Config{
+		Loaders:    ProfileAuthLoaders,
+		ConfigFile: "profile/testdata/databrickscfg",
+		Profile:    "DEFAULT",
+	}
+
+	err := cfg.EnsureResolved()
+	require.NoError(t, err)
+
+	assert.Equal(t, "https://default.test", cfg.Host)
+	assert.Equal(t, "default", cfg.Token)
+	// Steering/routing env vars did not shadow the profile.
+	assert.Empty(t, cfg.AuthType)
+	assert.Empty(t, cfg.DiscoveryURL)
+	assert.Empty(t, cfg.Cloud)
+	assert.Empty(t, cfg.WorkspaceID)
+	assert.Empty(t, cfg.AccountID)
+}
+
 func TestProfileAuthLoadersConflictingEnvAuthMethodErrors(t *testing.T) {
 	// Profile has a PAT; env gap-fills a different complete method (OAuth M2M).
 	// The config then carries two auth methods, which the SDK rejects rather