Fall back to file cache on read when keyring is unreachable

Mirrors the login-path fallback for the read path. When the resolver
returns (mode=Secure, source=Default) and the keyring probes as
definitively unreachable, ResolveCache now returns the file cache
instead of the keyring. This means users upgrading on systems without
a usable keyring (e.g. Linux containers without a D-Bus session bus)
keep reading from their pre-upgrade token-cache.json without any
manual configuration.

Two behaviours intentionally differ from the login-path fallback:

1. The probe is read-only (ProbeKeyringRead does a Get on a non-existent
   account name, treating keyring.ErrNotFound as "reachable"). The
   login-path probe still uses the write+delete cycle because login
   needs to validate write capability before committing tokens.

2. The read-path fallback does NOT persist auth_storage = plaintext to
   [__settings__]. A read failure could be transient (e.g. a momentarily
   broken D-Bus connection) and we don't have strong enough evidence to
   pin the user out of the keyring permanently. Pinning stays the
   responsibility of the login command, which has the stronger
   write-probe signal and an explicit user action.

Timeouts in the read probe stay on the keyring (same logic as the login
path): a locked keyring being unlocked is the most common timeout case,
and a misdiagnosed hang fails the actual Lookup, which is preferable to
silently routing reads to a different backend.

Explicit-secure callers (env var or config) get the keyring cache
regardless of probe result, so an explicit "I want secure" request
surfaces the unreachability instead of being silently downgraded.

Co-authored-by: Isaac

Author: simonfaltum

NEXT_CHANGELOG.md

@@ -4,7 +4,7 @@
 
 ### Notable Changes
 
-* Breaking change: OAuth tokens for interactive logins (`auth_type = databricks-cli`) are now stored in the OS-native secure store by default (Keychain on macOS, Credential Manager on Windows, Secret Service on Linux) instead of `~/.databricks/token-cache.json`. After upgrading, run `databricks auth login` once per profile to re-authenticate; cached tokens from older versions are not migrated. To keep the previous file-backed storage, set `DATABRICKS_AUTH_STORAGE=plaintext` or add `auth_storage = plaintext` under `[__settings__]` in `~/.databrickscfg` (the env var takes precedence over the config setting), then re-run `databricks auth login`.
+* Breaking change: OAuth tokens for interactive logins (`auth_type = databricks-cli`) are now stored in the OS-native secure store by default (Keychain on macOS, Credential Manager on Windows, Secret Service on Linux) instead of `~/.databricks/token-cache.json`. After upgrading, run `databricks auth login` once per profile to re-authenticate; cached tokens from older versions are not migrated. To keep the previous file-backed storage, set `DATABRICKS_AUTH_STORAGE=plaintext` or add `auth_storage = plaintext` under `[__settings__]` in `~/.databrickscfg` (the env var takes precedence over the config setting), then re-run `databricks auth login`. On systems where the OS keyring is not reachable (e.g. Linux containers without a D-Bus session bus), the CLI transparently falls back to the file cache when reading tokens so legacy `token-cache.json` entries remain accessible without manual configuration.
 
 ### CLI
 

libs/auth/storage/cache.go

@@ -16,17 +16,19 @@ import (
 // so unit tests can inject stubs without hitting the real OS keyring or
 // filesystem. Production code uses defaultCacheFactories().
 type cacheFactories struct {
-	newFile      func(context.Context) (cache.TokenCache, error)
-	newKeyring   func() cache.TokenCache
-	probeKeyring func() error
+	newFile          func(context.Context) (cache.TokenCache, error)
+	newKeyring       func() cache.TokenCache
+	probeKeyring     func() error
+	probeKeyringRead func() error
 }
 
 // defaultCacheFactories returns the production factory set.
 func defaultCacheFactories() cacheFactories {
 	return cacheFactories{
-		newFile:      func(ctx context.Context) (cache.TokenCache, error) { return NewFileTokenCache(ctx) },
-		newKeyring:   NewKeyringCache,
-		probeKeyring: ProbeKeyring,
+		newFile:          func(ctx context.Context) (cache.TokenCache, error) { return NewFileTokenCache(ctx) },
+		newKeyring:       NewKeyringCache,
+		probeKeyring:     ProbeKeyring,
+		probeKeyringRead: ProbeKeyringRead,
 	}
 }
 
@@ -37,11 +39,18 @@ func defaultCacheFactories() cacheFactories {
 // override is usually the command-level flag value. Pass "" when the command
 // has no flag; precedence then falls through to env -> config -> default.
 //
+// When the resolver returns (mode=Secure, source=Default) and the OS
+// keyring is definitively unreachable (a non-timeout probe error), reads
+// fall back to the plaintext file cache so post-upgrade users with legacy
+// token-cache.json entries are not stranded. Unlike the login path, this
+// fallback does not persist auth_storage = plaintext to [__settings__];
+// pinning happens only on successful login.
+//
 // Every CLI code path that calls u2m.NewPersistentAuth must route the result
 // through u2m.WithTokenCache, otherwise the SDK defaults to the file cache
 // and splits the user's tokens across two backends.
 func ResolveCache(ctx context.Context, override StorageMode) (cache.TokenCache, StorageMode, error) {
-	inner, mode, err := resolveCacheWith(ctx, override, defaultCacheFactories())
+	inner, mode, err := resolveCacheForReadWith(ctx, override, defaultCacheFactories())
 	if err != nil {
 		return nil, "", err
 	}
@@ -89,8 +98,11 @@ func WrapForOAuthArgument(tokenCache cache.TokenCache, mode StorageMode, arg u2m
 	return NewDualWritingTokenCache(tokenCache, arg)
 }
 
-// resolveCacheWith is the pure form of ResolveCache. It takes the factory
-// set as a parameter so tests can inject stubs.
+// resolveCacheWith is the pure form of ResolveCache without the read-path
+// fallback. Takes the factory set as a parameter so tests can inject stubs.
+// Used directly by ResolveCacheForLogin (which has its own fallback rules)
+// and indirectly by ResolveCache (which adds the read-path fallback in
+// resolveCacheForReadWith).
 func resolveCacheWith(ctx context.Context, override StorageMode, f cacheFactories) (cache.TokenCache, StorageMode, error) {
 	mode, err := ResolveStorageMode(ctx, override)
 	if err != nil {
@@ -110,6 +122,61 @@ func resolveCacheWith(ctx context.Context, override StorageMode, f cacheFactorie
 	}
 }
 
+// resolveCacheForReadWith is the pure form of ResolveCache. It applies the
+// read-path fallback: when mode is secure-from-default and the keyring
+// probes as definitively unavailable, return the file cache instead.
+// Timeouts keep the keyring (could be transient).
+func resolveCacheForReadWith(ctx context.Context, override StorageMode, f cacheFactories) (cache.TokenCache, StorageMode, error) {
+	mode, source, err := ResolveStorageModeWithSource(ctx, override)
+	if err != nil {
+		return nil, "", err
+	}
+	return applyReadFallback(ctx, mode, source.Explicit(), f)
+}
+
+// applyReadFallback realizes the read-path fallback. Mirrors
+// applyLoginFallback but:
+//
+//   - Uses a read-only probe (ProbeKeyringRead) so calls do not write to
+//     the keyring on every CLI invocation.
+//   - Does not persist auth_storage = plaintext. Pinning happens only on
+//     successful login, where the write-probe gives us stronger evidence
+//     that the keyring is truly unavailable on this machine.
+//
+// Explicit secure is honored: callers who asked for secure get the keyring
+// cache even if the probe fails, so the actual Lookup error surfaces the
+// unreachability instead of silently using a different backend.
+func applyReadFallback(ctx context.Context, mode StorageMode, explicit bool, f cacheFactories) (cache.TokenCache, StorageMode, error) {
+	switch mode {
+	case StorageModePlaintext:
+		c, err := f.newFile(ctx)
+		if err != nil {
+			return nil, "", fmt.Errorf("open file token cache: %w", err)
+		}
+		return c, mode, nil
+	case StorageModeSecure:
+		if explicit {
+			return f.newKeyring(), mode, nil
+		}
+		if probeErr := f.probeKeyringRead(); probeErr != nil {
+			var timeoutErr *TimeoutError
+			if errors.As(probeErr, &timeoutErr) {
+				log.Debugf(ctx, "keyring read probe timed out (%v); staying on keyring", probeErr)
+				return f.newKeyring(), mode, nil
+			}
+			log.Debugf(ctx, "secure storage unavailable on read path (%v), using file cache", probeErr)
+			fileCache, fileErr := f.newFile(ctx)
+			if fileErr != nil {
+				return nil, "", fmt.Errorf("open file token cache: %w", fileErr)
+			}
+			return fileCache, StorageModePlaintext, nil
+		}
+		return f.newKeyring(), mode, nil
+	default:
+		return nil, "", fmt.Errorf("unsupported storage mode %q", string(mode))
+	}
+}
+
 // resolveCacheForLoginWith is the pure form of ResolveCacheForLogin. It takes
 // the factory set as a parameter so tests can inject stubs.
 func resolveCacheForLoginWith(ctx context.Context, override StorageMode, f cacheFactories) (cache.TokenCache, StorageMode, error) {

libs/auth/storage/cache_test.go

@@ -28,9 +28,10 @@ func (stubCache) Lookup(string) (*oauth2.Token, error) { return nil, cache.ErrNo
 func fakeFactories(t *testing.T) cacheFactories {
 	t.Helper()
 	return cacheFactories{
-		newFile:      func(context.Context) (cache.TokenCache, error) { return stubCache{source: "file"}, nil },
-		newKeyring:   func() cache.TokenCache { return stubCache{source: "keyring"} },
-		probeKeyring: func() error { return nil },
+		newFile:          func(context.Context) (cache.TokenCache, error) { return stubCache{source: "file"}, nil },
+		newKeyring:       func() cache.TokenCache { return stubCache{source: "keyring"} },
+		probeKeyring:     func() error { return nil },
+		probeKeyringRead: func() error { return nil },
 	}
 }
 
@@ -111,9 +112,10 @@ func TestResolveCache_FileFactoryErrorPropagates(t *testing.T) {
 	ctx := t.Context()
 	boom := errors.New("disk full")
 	factories := cacheFactories{
-		newFile:      func(context.Context) (cache.TokenCache, error) { return nil, boom },
-		newKeyring:   func() cache.TokenCache { return stubCache{source: "keyring"} },
-		probeKeyring: func() error { return nil },
+		newFile:          func(context.Context) (cache.TokenCache, error) { return nil, boom },
+		newKeyring:       func() cache.TokenCache { return stubCache{source: "keyring"} },
+		probeKeyring:     func() error { return nil },
+		probeKeyringRead: func() error { return nil },
 	}
 
 	_, _, err := resolveCacheWith(ctx, StorageModePlaintext, factories)
@@ -122,6 +124,113 @@ func TestResolveCache_FileFactoryErrorPropagates(t *testing.T) {
 	assert.ErrorIs(t, err, boom)
 }
 
+// applyReadFallback mirrors applyLoginFallback's logic on the read path:
+// keyring is probed read-only, definitive failures fall through to the file
+// cache so legacy plaintext tokens stay reachable, timeouts stay on the
+// keyring, and explicit-secure is honored even when the probe fails.
+
+func TestApplyReadFallback_PlaintextSkipsProbe(t *testing.T) {
+	hermetic(t)
+	ctx := t.Context()
+	probed := false
+	f := fakeFactories(t)
+	f.probeKeyringRead = func() error {
+		probed = true
+		return nil
+	}
+
+	got, mode, err := applyReadFallback(ctx, StorageModePlaintext, false, f)
+
+	require.NoError(t, err)
+	assert.Equal(t, StorageModePlaintext, mode)
+	assert.Equal(t, "file", got.(stubCache).source)
+	assert.False(t, probed, "probe must not run when mode is already plaintext")
+}
+
+func TestApplyReadFallback_ExplicitSecureSkipsProbe(t *testing.T) {
+	hermetic(t)
+	ctx := t.Context()
+	probed := false
+	f := fakeFactories(t)
+	f.probeKeyringRead = func() error {
+		probed = true
+		return errors.New("unreachable")
+	}
+
+	got, mode, err := applyReadFallback(ctx, StorageModeSecure, true, f)
+
+	require.NoError(t, err)
+	assert.Equal(t, StorageModeSecure, mode)
+	assert.Equal(t, "keyring", got.(stubCache).source)
+	assert.False(t, probed, "probe must not run when user is explicit about secure mode")
+}
+
+func TestApplyReadFallback_DefaultSecure_ProbeOK_UsesKeyring(t *testing.T) {
+	hermetic(t)
+	ctx := t.Context()
+
+	got, mode, err := applyReadFallback(ctx, StorageModeSecure, false, fakeFactories(t))
+
+	require.NoError(t, err)
+	assert.Equal(t, StorageModeSecure, mode)
+	assert.Equal(t, "keyring", got.(stubCache).source)
+}
+
+func TestApplyReadFallback_DefaultSecure_ProbeFail_FallsBack(t *testing.T) {
+	hermetic(t)
+	ctx := t.Context()
+	configPath := env.Get(ctx, "DATABRICKS_CONFIG_FILE")
+
+	f := fakeFactories(t)
+	f.probeKeyringRead = func() error { return errors.New("no keyring") }
+
+	got, mode, err := applyReadFallback(ctx, StorageModeSecure, false, f)
+
+	require.NoError(t, err)
+	assert.Equal(t, StorageModePlaintext, mode)
+	assert.Equal(t, "file", got.(stubCache).source)
+
+	// Read-path fallback must NOT pin: pinning is reserved for login,
+	// where the write-probe gives stronger evidence of unavailability.
+	persisted, gerr := databrickscfg.GetConfiguredAuthStorage(ctx, configPath)
+	require.NoError(t, gerr)
+	assert.Equal(t, "", persisted, "read-path fallback must not persist auth_storage")
+}
+
+// A timeout could mean a locked keyring that will work once the user unlocks
+// it. Stay on the keyring so the actual Lookup surfaces the real outcome
+// rather than silently routing reads to the file cache.
+func TestApplyReadFallback_DefaultSecure_ProbeTimeout_StaysOnKeyring(t *testing.T) {
+	cases := []struct {
+		name     string
+		probeErr error
+	}{
+		{"bare TimeoutError", &TimeoutError{Op: "get"}},
+		{"wrapped TimeoutError", fmt.Errorf("get: %w", &TimeoutError{Op: "get"})},
+	}
+
+	for _, tc := range cases {
+		t.Run(tc.name, func(t *testing.T) {
+			hermetic(t)
+			ctx := t.Context()
+			configPath := env.Get(ctx, "DATABRICKS_CONFIG_FILE")
+
+			f := fakeFactories(t)
+			f.probeKeyringRead = func() error { return tc.probeErr }
+
+			got, mode, err := applyReadFallback(ctx, StorageModeSecure, false, f)
+
+			require.NoError(t, err)
+			assert.Equal(t, StorageModeSecure, mode)
+			assert.Equal(t, "keyring", got.(stubCache).source)
+
+			persisted, gerr := databrickscfg.GetConfiguredAuthStorage(ctx, configPath)
+			require.NoError(t, gerr)
+			assert.Equal(t, "", persisted, "probe timeout must not persist anything")
+		})
+	}
+}
+
 func TestResolveCacheForLogin_PlaintextSkipsProbe(t *testing.T) {
 	hermetic(t)
 	ctx := t.Context()

libs/auth/storage/keyring.go

@@ -92,6 +92,9 @@ func NewKeyringCache() cache.TokenCache {
 // cycle within the standard timeout. *TimeoutError means the keyring
 // was unresponsive (locked or hung, indistinguishable here); other
 // errors are definitive failures.
+//
+// Used by the login path, where we want to validate both read and write
+// capability before committing to the keyring backend.
 func ProbeKeyring() error {
 	return probeWithBackend(zalandoBackend{}, defaultKeyringTimeout)
 }
@@ -113,6 +116,35 @@ func probeWithBackend(backend keyringBackend, timeout time.Duration) error {
 	return nil
 }
 
+// ProbeKeyringRead returns nil if the OS keyring accepted a Get for a
+// non-existent account within the standard timeout (i.e. the backend is
+// reachable and responded with keyring.ErrNotFound). *TimeoutError means
+// the keyring was unresponsive; other errors are definitive failures.
+//
+// Used by the read path so probing does not write to the keyring. A
+// successful probe is indistinguishable from the user not having an
+// entry for that probe account; we treat both as "reachable".
+func ProbeKeyringRead() error {
+	return probeReadWithBackend(zalandoBackend{}, defaultKeyringTimeout)
+}
+
+func probeReadWithBackend(backend keyringBackend, timeout time.Duration) error {
+	c := &keyringCache{
+		backend:        backend,
+		timeout:        timeout,
+		keyringSvcName: keyringServiceName,
+	}
+	account := keyringProbeAccountPrefix + uuid.NewString()
+	err := c.withTimeout("get", func() error {
+		_, gerr := c.backend.Get(c.keyringSvcName, account)
+		return gerr
+	})
+	if errors.Is(err, keyring.ErrNotFound) {
+		return nil
+	}
+	return err
+}
+
 // Store stores t under key. Nil t deletes the entry; deleting a missing
 // entry is not an error.
 func (k *keyringCache) Store(key string, t *oauth2.Token) error {

libs/auth/storage/keyring_test.go

@@ -296,3 +296,59 @@ func TestProbeKeyring(t *testing.T) {
 		})
 	}
 }
+
+func TestProbeKeyringRead(t *testing.T) {
+	boom := errors.New("backend boom")
+	cases := []struct {
+		name        string
+		getErr      error
+		getBlock    bool
+		timeout     time.Duration
+		wantErr     error
+		wantTimeout bool
+	}{
+		{
+			// keyring.ErrNotFound is the success signal: the backend
+			// responded that no entry exists for our probe account,
+			// which means it is reachable.
+			name:    "ErrNotFound counts as reachable",
+			getErr:  keyring.ErrNotFound,
+			timeout: 100 * time.Millisecond,
+		},
+		{
+			name:    "other backend error propagates",
+			getErr:  boom,
+			timeout: 100 * time.Millisecond,
+			wantErr: boom,
+		},
+		{
+			name:        "get times out",
+			getBlock:    true,
+			timeout:     50 * time.Millisecond,
+			wantTimeout: true,
+		},
+	}
+
+	for _, tc := range cases {
+		t.Run(tc.name, func(t *testing.T) {
+			backend := newFakeBackend()
+			backend.getErr = tc.getErr
+			backend.getBlock = tc.getBlock
+
+			err := probeReadWithBackend(backend, tc.timeout)
+
+			switch {
+			case tc.wantErr != nil:
+				require.Error(t, err)
+				assert.ErrorIs(t, err, tc.wantErr)
+			case tc.wantTimeout:
+				require.Error(t, err)
+				var timeoutErr *TimeoutError
+				assert.ErrorAs(t, err, &timeoutErr)
+			default:
+				require.NoError(t, err)
+				assert.Empty(t, backend.items, "read probe must not write to the keyring")
+			}
+		})
+	}
+}