direct: Fix spurious recreate of external volumes with trailing-slash storage_location (#5145)

The UC API strips trailing slashes from storage_location on create,
causing the direct engine to detect drift on every subsequent plan and
trigger a full delete+create cycle.

Fix: add OverrideChangeDesc to ResourceVolume that treats
storage_location values differing only by trailing slashes as equivalent
(alias), matching the Terraform provider's suppressLocationDiff
behavior.

Also update the testserver to accept storage_location on EXTERNAL
volumes and strip the trailing slash, mirroring UC API behavior so the
bug and fix are reproducible locally. Add an invariant no_drift test for
external volumes.

Co-authored-by: Isaac

Author: denik

acceptance/bundle/invariant/configs/volume_external.yml.tmpl

@@ -0,0 +1,11 @@
+bundle:
+  name: test-bundle-$UNIQUE_NAME
+
+resources:
+  volumes:
+    foo:
+      name: test-volume-$UNIQUE_NAME
+      catalog_name: main
+      schema_name: default
+      volume_type: EXTERNAL
+      storage_location: s3://test-bucket/path/

acceptance/bundle/invariant/continue_293/out.test.toml

@@ -57,6 +57,7 @@ EnvMatrix.INPUT_CONFIG = [
     "synced_database_table.yml.tmpl",
     "vector_search_endpoint.yml.tmpl",
     "volume.yml.tmpl",
+    "volume_external.yml.tmpl",
 ]
 
 [EnvMatrixExclude]
@@ -71,6 +72,9 @@ no_postgres_endpoint_on_cloud = ["CONFIG_Cloud=true", "INPUT_CONFIG=postgres_end
 # which are environment-specific, so we only test locally with the mock server
 no_external_location_on_cloud = ["CONFIG_Cloud=true", "INPUT_CONFIG=external_location.yml.tmpl"]
 
+# External volumes reference external locations; excluded from cloud for the same reason
+no_external_volume_on_cloud = ["CONFIG_Cloud=true", "INPUT_CONFIG=volume_external.yml.tmpl"]
+
 # Fake SQL endpoint for local tests
 [[Server]]
 Pattern = "POST /api/2.0/sql/statements/"

acceptance/bundle/invariant/migrate/out.test.toml

@@ -100,6 +100,7 @@ type ChangeDesc struct {
 const (
 	ReasonBackendDefault   = "backend_default"
 	ReasonAlias            = "alias"
+	ReasonURLNormalization = "url_normalization"
 	ReasonRemoteAlreadySet = "remote_already_set"
 	ReasonEmpty            = "empty"
 	ReasonCustom           = "custom"

acceptance/bundle/invariant/no_drift/out.test.toml

@@ -6,7 +6,9 @@ import (
 	"strings"
 
 	"github.com/databricks/cli/bundle/config/resources"
+	"github.com/databricks/cli/bundle/deployplan"
 	"github.com/databricks/cli/libs/log"
+	"github.com/databricks/cli/libs/structs/structpath"
 	"github.com/databricks/cli/libs/utils"
 	"github.com/databricks/databricks-sdk-go"
 	"github.com/databricks/databricks-sdk-go/service/catalog"
@@ -112,6 +114,35 @@ func (r *ResourceVolume) DoDelete(ctx context.Context, id string) error {
 	return r.client.Volumes.DeleteByName(ctx, id)
 }
 
+// OverrideChangeDesc suppresses drift for storage_location when the only difference
+// is a trailing slash. The UC API strips trailing slashes on create, so remote returns
+// "s3://bucket/path" while the config may have "s3://bucket/path/".
+//
+// This matches the Terraform provider's suppressLocationDiff behavior.
+// https://github.com/databricks/terraform-provider-databricks/blob/v1.65.1/catalog/resource_volume.go#L25
+func (*ResourceVolume) OverrideChangeDesc(_ context.Context, path *structpath.PathNode, change *ChangeDesc, _ *catalog.VolumeInfo) error {
+	if change.Action == deployplan.Skip {
+		return nil
+	}
+
+	if path.String() != "storage_location" {
+		return nil
+	}
+
+	newStr, newOk := change.New.(string)
+	remoteStr, remoteOk := change.Remote.(string)
+	if !newOk || !remoteOk {
+		return nil
+	}
+
+	if newStr != remoteStr && strings.TrimRight(newStr, "/") == strings.TrimRight(remoteStr, "/") {
+		change.Action = deployplan.Skip
+		change.Reason = deployplan.ReasonURLNormalization
+	}
+
+	return nil
+}
+
 func getNameFromID(id string) (string, error) {
 	items := strings.Split(id, ".")
 	if len(items) == 0 {

acceptance/bundle/invariant/test.toml

@@ -4,6 +4,7 @@ import (
 	"encoding/json"
 	"fmt"
 	"net/http"
+	"strings"
 
 	"github.com/databricks/databricks-sdk-go/service/catalog"
 	"github.com/databricks/databricks-sdk-go/service/workspace"
@@ -21,7 +22,7 @@ func (s *FakeWorkspace) VolumesCreate(req Request) Response {
 
 	volume.FullName = volume.CatalogName + "." + volume.SchemaName + "." + volume.Name
 
-	if volume.StorageLocation != "" {
+	if volume.StorageLocation != "" && volume.VolumeType != catalog.VolumeTypeExternal {
 		return Response{
 			StatusCode: 400,
 			Body: map[string]string{
@@ -30,8 +31,15 @@ func (s *FakeWorkspace) VolumesCreate(req Request) Response {
 			},
 		}
 	}
+
 	volume.VolumeId = nextUUID()
-	volume.StorageLocation = fmt.Sprintf("s3://%s/metastore/%s/volumes/%s", testMetastoreName, TestMetastore.MetastoreId, volume.VolumeId)
+
+	if volume.StorageLocation != "" {
+		// Strip trailing slash to mimic UC API normalization behavior.
+		volume.StorageLocation = strings.TrimRight(volume.StorageLocation, "/")
+	} else {
+		volume.StorageLocation = fmt.Sprintf("s3://%s/metastore/%s/volumes/%s", testMetastoreName, TestMetastore.MetastoreId, volume.VolumeId)
+	}
 
 	volume.CreatedAt = nowMilli()
 	volume.UpdatedAt = volume.CreatedAt