Introduce a CLI-owned Store interface for token storage (#5383)

## What is this about?

The CLI's token storage is currently shaped by the SDK's TokenCache
interface. That interface is internal to the SDK's user-to-machine (U2M)
login flow and only carries a bare OAuth token keyed by a string, so
there is no room to store anything alongside a token, and the CLI's
storage is coupled to an SDK type that exists specifically for U2M
login.

This PR introduces a CLI-owned `Store` interface (`Put`, `Lookup`,
`Delete`) over an explicit `Entry` envelope that wraps the token. A
small adapter, `ToU2MTokenCache`, presents a `Store` to the SDK at the
one place that still requires the SDK interface: passing a cache to
`u2m.PersistentAuth`. The file and keyring backends now implement
`Store`, and the OAuth helpers adapt it for the U2M paths.

This is a structural change with no behavior change. The `Entry` only
holds the token; the point is that it can now grow additional fields
without changing the interface or depending on the SDK. The first such
field will be a config checksum used to invalidate a cached token when
its profile changed, which arrives with a later change that adds caching
for machine-to-machine (M2M) and OIDC tokens.

**Naming note for reviewers:** the new interface is named Store to
distinguish the CLI's durable storage from the SDK's transient Cache.
The concrete types and helpers in this package are intentionally left in
the old vocabulary (fileTokenCache, keyringCache, and so on) in this PR
to keep the diff focused on the structural change. A follow-up PR will
take care of renaming.

## Testing

Existing auth unit and acceptance tests pass; there are no output or
behavior changes.

Author: renaudhartert-db

cmd/auth/in_memory_test.go

@@ -1,39 +1,40 @@
 package auth
 
 import (
-	"github.com/databricks/databricks-sdk-go/credentials/u2m/cache"
+	"github.com/databricks/cli/libs/auth/storage"
 	"golang.org/x/oauth2"
 )
 
 type inMemoryTokenCache struct {
 	Tokens map[string]*oauth2.Token
 }
 
-// Lookup implements TokenCache.
-// Returns a copy to match real (file-backed) cache behavior, where each
-// Lookup deserializes a fresh struct. Without the copy, callers that
+// Lookup returns a copy to match real (file-backed) cache behavior, where
+// each lookup deserializes a fresh struct. Without the copy, callers that
 // mutate the returned token (e.g. clearing RefreshToken) would corrupt
 // entries shared across test cases.
-func (i *inMemoryTokenCache) Lookup(key string) (*oauth2.Token, error) {
+func (i *inMemoryTokenCache) Lookup(key string) (storage.Entry, error) {
 	token, ok := i.Tokens[key]
 	if !ok {
-		return nil, cache.ErrNotFound
+		return storage.Entry{}, storage.ErrNotFound
 	}
 	cp := *token
-	return &cp, nil
+	return storage.Entry{Token: &cp}, nil
 }
 
-// Store implements TokenCache.
-// Stores a copy to prevent callers from mutating cached entries after Store
-// returns (mirrors file-backed cache semantics).
-func (i *inMemoryTokenCache) Store(key string, t *oauth2.Token) error {
-	if t == nil {
-		delete(i.Tokens, key)
-	} else {
-		cp := *t
-		i.Tokens[key] = &cp
-	}
+// Put stores a copy to prevent callers from mutating cached entries after
+// put returns (mirrors file-backed cache semantics).
+func (i *inMemoryTokenCache) Put(key string, e storage.Entry) error {
+	cp := *e.Token
+	i.Tokens[key] = &cp
+	return nil
+}
+
+// Delete deletes the entry under key. Deleting a missing entry is not
+// an error.
+func (i *inMemoryTokenCache) Delete(key string) error {
+	delete(i.Tokens, key)
 	return nil
 }
 
-var _ cache.TokenCache = (*inMemoryTokenCache)(nil)
+var _ storage.Store = (*inMemoryTokenCache)(nil)

cmd/auth/login.go

@@ -22,7 +22,6 @@ import (
 	"github.com/databricks/databricks-sdk-go/config"
 	"github.com/databricks/databricks-sdk-go/config/experimental/auth/authconv"
 	"github.com/databricks/databricks-sdk-go/credentials/u2m"
-	"github.com/databricks/databricks-sdk-go/credentials/u2m/cache"
 	"github.com/spf13/cobra"
 	"golang.org/x/oauth2"
 )
@@ -300,7 +299,7 @@ a new profile is created.
 		persistentAuthOpts := []u2m.PersistentAuthOption{
 			u2m.WithOAuthArgument(oauthArgument),
 			u2m.WithBrowser(getBrowserFunc(cmd)),
-			u2m.WithTokenCache(storage.WrapForOAuthArgument(tokenCache, mode, oauthArgument)),
+			u2m.WithTokenCache(storage.WrapForOAuthArgument(ctx, tokenCache, mode, oauthArgument)),
 		}
 		if len(scopesList) > 0 {
 			persistentAuthOpts = append(persistentAuthOpts, u2m.WithScopes(scopesList))
@@ -629,7 +628,7 @@ type discoveryLoginInputs struct {
 	scopes          string
 	existingProfile *profile.Profile
 	browserFunc     func(string) error
-	tokenCache      cache.TokenCache
+	tokenCache      storage.Store
 	mode            storage.StorageMode
 }
 
@@ -651,7 +650,7 @@ func discoveryLogin(ctx context.Context, in discoveryLoginInputs) error {
 		u2m.WithOAuthArgument(arg),
 		u2m.WithBrowser(in.browserFunc),
 		u2m.WithDiscoveryLogin(),
-		u2m.WithTokenCache(storage.WrapForOAuthArgument(in.tokenCache, in.mode, arg)),
+		u2m.WithTokenCache(storage.WrapForOAuthArgument(ctx, in.tokenCache, in.mode, arg)),
 	}
 	if len(scopesList) > 0 {
 		opts = append(opts, u2m.WithScopes(scopesList))

cmd/auth/login_test.go

@@ -15,12 +15,12 @@ import (
 	"time"
 
 	"github.com/databricks/cli/libs/auth"
+	"github.com/databricks/cli/libs/auth/storage"
 	"github.com/databricks/cli/libs/cmdio"
 	"github.com/databricks/cli/libs/databrickscfg/profile"
 	"github.com/databricks/cli/libs/env"
 	"github.com/databricks/cli/libs/log"
 	"github.com/databricks/databricks-sdk-go/credentials/u2m"
-	"github.com/databricks/databricks-sdk-go/credentials/u2m/cache"
 	"github.com/spf13/cobra"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
@@ -29,7 +29,7 @@ import (
 
 // newTestTokenCache returns an in-memory token cache for tests so that
 // discoveryLogin and other login helpers don't touch ~/.databricks/token-cache.json.
-func newTestTokenCache() cache.TokenCache {
+func newTestTokenCache() storage.Store {
 	return &inMemoryTokenCache{Tokens: map[string]*oauth2.Token{}}
 }
 

cmd/auth/logout.go

@@ -28,7 +28,6 @@ import (
 	"github.com/databricks/cli/libs/databrickscfg/profile"
 	"github.com/databricks/cli/libs/env"
 	"github.com/databricks/databricks-sdk-go/config"
-	"github.com/databricks/databricks-sdk-go/credentials/u2m/cache"
 	"github.com/spf13/cobra"
 )
 
@@ -158,7 +157,7 @@ type logoutArgs struct {
 	autoApprove    bool
 	deleteProfile  bool
 	profiler       profile.Profiler
-	tokenCache     cache.TokenCache
+	tokenCache     storage.Store
 	configFilePath string
 }
 
@@ -270,8 +269,8 @@ func getMatchingProfile(ctx context.Context, profileName string, profiler profil
 //     remaining profile references the same key. For account and unified
 //     profiles, the cache key includes the OIDC path
 //     (host/oidc/accounts/<account_id>).
-func clearTokenCache(ctx context.Context, p profile.Profile, profiler profile.Profiler, tokenCache cache.TokenCache) error {
-	if err := tokenCache.Store(p.Name, nil); err != nil {
+func clearTokenCache(ctx context.Context, p profile.Profile, profiler profile.Profiler, tokenCache storage.Store) error {
+	if err := tokenCache.Delete(p.Name); err != nil {
 		return fmt.Errorf("failed to delete profile-keyed token for profile %q: %w", p.Name, err)
 	}
 
@@ -291,7 +290,7 @@ func clearTokenCache(ctx context.Context, p profile.Profile, profiler profile.Pr
 	}
 
 	if len(otherProfiles) == 0 {
-		if err := tokenCache.Store(hostCacheKey, nil); err != nil {
+		if err := tokenCache.Delete(hostCacheKey); err != nil {
 			return fmt.Errorf("failed to delete host-keyed token for %q: %w", hostCacheKey, err)
 		}
 	}

cmd/auth/token.go

@@ -118,9 +118,9 @@ type loadTokenArgs struct {
 	// profiler is the profiler to use for reading the host and account ID from the .databrickscfg file.
 	profiler profile.Profiler
 
-	// tokenCache is the underlying TokenCache used for OAuth tokens. The caller is
+	// tokenCache is the underlying CLI cache used for OAuth tokens. The caller is
 	// responsible for construction so that tests can substitute an in-memory cache.
-	tokenCache cache.TokenCache
+	tokenCache storage.Store
 
 	// mode is the resolved storage mode. When set to StorageModePlaintext,
 	// login paths mirror freshly minted tokens under the legacy host-based
@@ -264,7 +264,7 @@ func loadToken(ctx context.Context, args loadTokenArgs) (*oauth2.Token, error) {
 	if err != nil {
 		return nil, err
 	}
-	allArgs := append([]u2m.PersistentAuthOption{u2m.WithTokenCache(args.tokenCache)}, args.persistentAuthOpts...)
+	allArgs := append([]u2m.PersistentAuthOption{u2m.WithTokenCache(storage.OAuthTokenCache(ctx, args.tokenCache, args.mode))}, args.persistentAuthOpts...)
 	allArgs = append(allArgs, u2m.WithOAuthArgument(oauthArgument))
 	persistentAuth, err := u2m.NewPersistentAuth(ctx, allArgs...)
 	if err != nil {
@@ -318,7 +318,7 @@ func loadToken(ctx context.Context, args loadTokenArgs) (*oauth2.Token, error) {
 //
 // Returns the resolved profile name and profile (if any). The host and related
 // fields on authArgs are updated in place when resolved via environment variables.
-func resolveNoArgsToken(ctx context.Context, profiler profile.Profiler, authArgs *auth.AuthArguments, tokenCache cache.TokenCache, mode storage.StorageMode) (string, *profile.Profile, error) {
+func resolveNoArgsToken(ctx context.Context, profiler profile.Profiler, authArgs *auth.AuthArguments, tokenCache storage.Store, mode storage.StorageMode) (string, *profile.Profile, error) {
 	// Step 1: Try DATABRICKS_HOST env var (highest priority).
 	if envHost := env.Get(ctx, "DATABRICKS_HOST"); envHost != "" {
 		authArgs.Host = envHost
@@ -394,7 +394,7 @@ func resolveNoArgsToken(ctx context.Context, profiler profile.Profiler, authArgs
 // runInlineLogin runs a minimal interactive login flow: prompts for a profile
 // name and host, performs the OAuth challenge, saves the profile to
 // .databrickscfg, and returns the new profile name and profile.
-func runInlineLogin(ctx context.Context, profiler profile.Profiler, tokenCache cache.TokenCache, mode storage.StorageMode) (string, *profile.Profile, error) {
+func runInlineLogin(ctx context.Context, profiler profile.Profiler, tokenCache storage.Store, mode storage.StorageMode) (string, *profile.Profile, error) {
 	profileName, err := promptForProfile(ctx, "DEFAULT")
 	if err != nil {
 		return "", nil, err
@@ -428,7 +428,7 @@ func runInlineLogin(ctx context.Context, profiler profile.Profiler, tokenCache c
 	persistentAuthOpts := []u2m.PersistentAuthOption{
 		u2m.WithOAuthArgument(oauthArgument),
 		u2m.WithBrowser(func(url string) error { return browser.Open(ctx, url) }),
-		u2m.WithTokenCache(storage.WrapForOAuthArgument(tokenCache, mode, oauthArgument)),
+		u2m.WithTokenCache(storage.WrapForOAuthArgument(ctx, tokenCache, mode, oauthArgument)),
 	}
 	if len(scopesList) > 0 {
 		persistentAuthOpts = append(persistentAuthOpts, u2m.WithScopes(scopesList))