Revert vector_search_endpoints UUID persistence (#5127, #5192) (#5193)

## Summary

- Reverts #5127 (`Persist endpoint UUID for vector_search_endpoints
drift detection`) and the follow-up changelog entry from #5192.
- The badness #5127 was meant to fix — bundle silently rebinding
permissions to a different backing endpoint after an out-of-band
recreate — was actually addressed by the testserver fix in #5186
(`testserver: 404 on permissions GET when V2 parent is gone`). With the
testserver matching real V2 cloud behavior, bundle correctly observes
that the new endpoint has no permissions and creates them, with no
permanent drift afterwards. UUID persistence in state is no longer
necessary.
- Reworks the `drift/recreated_same_name` acceptance test: keeps
endpoint permissions in `databricks.yml`, drops the obsolete "recreate
detected" assertion, and adds a post-deploy `bundle plan` to confirm
there is no permanent drift.

## Test plan

- [x] `./task build` clean.
- [x] `go test ./acceptance -run
'TestAccept/bundle/resources/vector_search_endpoints/drift'` — all green
(terraform + direct).
- [x] `go test ./bundle/direct/dresources/...` — green.
- [x] `./task lint-q` — clean.
- [x] Verified post-deploy plan shows `Plan: 0 to add, 0 to change, 0 to
delete, 2 unchanged` after an out-of-band endpoint recreate, so
permissions don't end up in permanent drift even without UUID-based
recreate detection.

This pull request and its description were written by Isaac.

Author: janniklasrose

NEXT_CHANGELOG.md

@@ -10,7 +10,6 @@
 ### Bundles
 * Validate that resource keys do not contain variable references ([#5169](https://github.com/databricks/cli/pull/5169))
 * engine/direct: Drop the deployment state entry on a recreate before the follow-up `Create`, so a `Create` failure no longer leaves a broken state with `invalid state: empty id` on the next `bundle plan` ([#5173](https://github.com/databricks/cli/pull/5173)).
-* engine/direct: vector search endpoints: trigger recreate when endpoint is recreated out of band ([#5127](https://github.com/databricks/cli/pull/5127))
 
 ### Dependency updates
 

acceptance/bundle/refschema/out.fields.txt

@@ -3042,7 +3042,7 @@ resources.vector_search_endpoints.*.endpoint_status	*vectorsearch.EndpointStatus
 resources.vector_search_endpoints.*.endpoint_status.message	string	REMOTE
 resources.vector_search_endpoints.*.endpoint_status.state	vectorsearch.EndpointStatusState	REMOTE
 resources.vector_search_endpoints.*.endpoint_type	vectorsearch.EndpointType	ALL
-resources.vector_search_endpoints.*.endpoint_uuid	string	REMOTE	STATE
+resources.vector_search_endpoints.*.endpoint_uuid	string	REMOTE
 resources.vector_search_endpoints.*.id	string	INPUT	REMOTE
 resources.vector_search_endpoints.*.last_updated_timestamp	int64	REMOTE
 resources.vector_search_endpoints.*.last_updated_user	string	REMOTE

acceptance/bundle/resources/vector_search_endpoints/drift/min_qps/out.plan.direct.json

@@ -3,12 +3,6 @@
     "resources.vector_search_endpoints.my_endpoint": {
       "action": "update",
       "changes": {
-        "endpoint_uuid": {
-          "action": "skip",
-          "reason": "custom",
-          "old": "[MY_ENDPOINT_UUID]",
-          "remote": "[MY_ENDPOINT_UUID]"
-        },
         "min_qps": {
           "action": "update",
           "old": 1,

acceptance/bundle/resources/vector_search_endpoints/drift/min_qps/output.txt

@@ -15,7 +15,7 @@ Deployment complete!
     "state": "ONLINE"
   },
   "endpoint_type": "STANDARD",
-  "id": "[MY_ENDPOINT_UUID]",
+  "id": "[UUID]",
   "last_updated_timestamp": [UNIX_TIME_MILLIS][1],
   "last_updated_user": "[USERNAME]",
   "name": "vs-endpoint-[UNIQUE_NAME]",

acceptance/bundle/resources/vector_search_endpoints/drift/min_qps/script

@@ -11,11 +11,6 @@ trace $CLI bundle deploy
 
 endpoint_name="vs-endpoint-${UNIQUE_NAME}"
 
-# Register a stable label for the endpoint UUID so the plan output shows the
-# same token for both saved (old) and remote, confirming they match.
-endpoint_uuid=$($CLI vector-search-endpoints get-endpoint "${endpoint_name}" | jq -r '.id')
-add_repl.py "$endpoint_uuid" "MY_ENDPOINT_UUID"
-
 title "Simulate remote drift: change min_qps to 5 outside the bundle"
 trace $CLI vector-search-endpoints patch-endpoint "${endpoint_name}" --min-qps 5
 

acceptance/bundle/resources/vector_search_endpoints/drift/recreated_same_name/output.txt

@@ -13,9 +13,6 @@ Deployment complete!
   "endpoint_type": "STANDARD"
 }
 
->>> print_state.py
-"/vector-search-endpoints/[ORIGINAL_ENDPOINT_UUID]"
-
 === Delete and recreate remotely with the same name
 >>> [CLI] vector-search-endpoints delete-endpoint vs-endpoint-[UNIQUE_NAME]
 
@@ -35,14 +32,12 @@ Deployment complete!
 Original endpoint UUID: [ORIGINAL_ENDPOINT_UUID]
 Remote recreated endpoint UUID: [REMOTE_RECREATED_ENDPOINT_UUID]
 
-=== Plan detects the UUID change and proposes recreate
+=== Plan after out-of-band recreate
 >>> [CLI] bundle plan
-recreate vector_search_endpoints.my_endpoint
 create vector_search_endpoints.my_endpoint.permissions
 
-Plan: 2 to add, 0 to change, 1 to delete, 0 unchanged
+Plan: 1 to add, 0 to change, 0 to delete, 1 unchanged
 
-=== Deploy recreates the endpoint and rebinds permissions to the new UUID
 >>> [CLI] bundle deploy
 Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/drift-vs-endpoint-recreated-same-name-[UNIQUE_NAME]/default/files...
 Deploying resources...
@@ -51,12 +46,14 @@ Deployment complete!
 
 >>> [CLI] vector-search-endpoints get-endpoint vs-endpoint-[UNIQUE_NAME]
 {
+  "id": "[REMOTE_RECREATED_ENDPOINT_UUID]",
   "name": "vs-endpoint-[UNIQUE_NAME]",
   "endpoint_type": "STANDARD"
 }
 
->>> print_state.py
-"/vector-search-endpoints/[UUID]"
+=== Verify no permanent drift after deploy
+>>> [CLI] bundle plan
+Plan: 0 to add, 0 to change, 0 to delete, 2 unchanged
 
 >>> [CLI] bundle destroy --auto-approve
 The following resources will be deleted:

acceptance/bundle/resources/vector_search_endpoints/drift/recreated_same_name/script

@@ -14,7 +14,6 @@ trace $CLI bundle deploy
 original_endpoint_uuid=$($CLI vector-search-endpoints get-endpoint "${endpoint_name}" | jq -r '.id')
 add_repl.py "$original_endpoint_uuid" "ORIGINAL_ENDPOINT_UUID"
 trace $CLI vector-search-endpoints get-endpoint "${endpoint_name}" | jq '{id, name, endpoint_type}'
-trace print_state.py | jq '.state."resources.vector_search_endpoints.my_endpoint.permissions".state.object_id'
 
 title "Delete and recreate remotely with the same name"
 trace $CLI vector-search-endpoints delete-endpoint "${endpoint_name}"
@@ -32,10 +31,11 @@ if [ "$original_endpoint_uuid" = "$remote_recreated_endpoint_uuid" ]; then
     exit 1
 fi
 
-title "Plan detects the UUID change and proposes recreate"
-trace $CLI bundle plan | contains.py "recreate vector_search_endpoints.my_endpoint" "create vector_search_endpoints.my_endpoint.permissions"
+title "Plan after out-of-band recreate"
+trace $CLI bundle plan
 
-title "Deploy recreates the endpoint and rebinds permissions to the new UUID"
 trace $CLI bundle deploy
-trace $CLI vector-search-endpoints get-endpoint "${endpoint_name}" | jq '{name, endpoint_type}'
-trace print_state.py | jq '.state."resources.vector_search_endpoints.my_endpoint.permissions".state.object_id'
+trace $CLI vector-search-endpoints get-endpoint "${endpoint_name}" | jq '{id, name, endpoint_type}'
+
+title "Verify no permanent drift after deploy"
+trace $CLI bundle plan | contains.py "Plan: 0 to add, 0 to change, 0 to delete"

bundle/direct/dresources/vector_search_endpoint.go

@@ -5,11 +5,9 @@ import (
 	"time"
 
 	"github.com/databricks/cli/bundle/config/resources"
-	"github.com/databricks/cli/bundle/deployplan"
 	"github.com/databricks/cli/libs/structs/structpath"
 	"github.com/databricks/cli/libs/utils"
 	"github.com/databricks/databricks-sdk-go"
-	"github.com/databricks/databricks-sdk-go/marshal"
 	"github.com/databricks/databricks-sdk-go/service/vectorsearch"
 )
 
@@ -18,23 +16,6 @@ var (
 	pathMinQps         = structpath.MustParsePath("min_qps")
 )
 
-// VectorSearchEndpointState is persisted in deployment state. endpoint_uuid is
-// tracked so out-of-band replacement of an endpoint with the same name can be
-// detected: when saved UUID differs from remote UUID, the endpoint is recreated.
-type VectorSearchEndpointState struct {
-	vectorsearch.CreateEndpoint
-	EndpointUuid string `json:"endpoint_uuid,omitempty"`
-}
-
-// Custom marshalers required because embedded CreateEndpoint has its own.
-func (s *VectorSearchEndpointState) UnmarshalJSON(b []byte) error {
-	return marshal.Unmarshal(b, s)
-}
-
-func (s VectorSearchEndpointState) MarshalJSON() ([]byte, error) {
-	return marshal.Marshal(s)
-}
-
 // VectorSearchEndpointRemote is remote state for a vector search endpoint. It embeds API response
 // fields for drift comparison and adds endpoint_uuid for permissions; deployment state id remains the endpoint name.
 type VectorSearchEndpointRemote struct {
@@ -60,28 +41,22 @@ func (*ResourceVectorSearchEndpoint) New(client *databricks.WorkspaceClient) *Re
 	return &ResourceVectorSearchEndpoint{client: client}
 }
 
-func (*ResourceVectorSearchEndpoint) PrepareState(input *resources.VectorSearchEndpoint) *VectorSearchEndpointState {
-	return &VectorSearchEndpointState{
-		CreateEndpoint: input.CreateEndpoint,
-		EndpointUuid:   "",
-	}
+func (*ResourceVectorSearchEndpoint) PrepareState(input *resources.VectorSearchEndpoint) *vectorsearch.CreateEndpoint {
+	return &input.CreateEndpoint
 }
 
-func (*ResourceVectorSearchEndpoint) RemapState(remote *VectorSearchEndpointRemote) *VectorSearchEndpointState {
+func (*ResourceVectorSearchEndpoint) RemapState(remote *VectorSearchEndpointRemote) *vectorsearch.CreateEndpoint {
 	var minQps int64
 	if remote.ScalingInfo != nil {
 		minQps = remote.ScalingInfo.RequestedMinQps
 	}
-	return &VectorSearchEndpointState{
-		CreateEndpoint: vectorsearch.CreateEndpoint{
-			Name:            remote.Name,
-			EndpointType:    remote.EndpointType,
-			BudgetPolicyId:  remote.BudgetPolicyId,
-			UsagePolicyId:   "", // Missing in remote
-			MinQps:          minQps,
-			ForceSendFields: utils.FilterFields[vectorsearch.CreateEndpoint](remote.ForceSendFields, "UsagePolicyId"),
-		},
-		EndpointUuid: remote.EndpointUuid,
+	return &vectorsearch.CreateEndpoint{
+		Name:            remote.Name,
+		EndpointType:    remote.EndpointType,
+		BudgetPolicyId:  remote.BudgetPolicyId,
+		UsagePolicyId:   "", // Missing in remote
+		MinQps:          minQps,
+		ForceSendFields: utils.FilterFields[vectorsearch.CreateEndpoint](remote.ForceSendFields, "UsagePolicyId"),
 	}
 }
 
@@ -93,27 +68,24 @@ func (r *ResourceVectorSearchEndpoint) DoRead(ctx context.Context, id string) (*
 	return newVectorSearchEndpointRemote(info), nil
 }
 
-func (r *ResourceVectorSearchEndpoint) DoCreate(ctx context.Context, config *VectorSearchEndpointState) (string, *VectorSearchEndpointRemote, error) {
-	waiter, err := r.client.VectorSearchEndpoints.CreateEndpoint(ctx, config.CreateEndpoint)
+func (r *ResourceVectorSearchEndpoint) DoCreate(ctx context.Context, config *vectorsearch.CreateEndpoint) (string, *VectorSearchEndpointRemote, error) {
+	waiter, err := r.client.VectorSearchEndpoints.CreateEndpoint(ctx, *config)
 	if err != nil {
 		return "", nil, err
 	}
 	id := config.Name
-	if waiter.Response != nil {
-		config.EndpointUuid = waiter.Response.Id
-	}
 	return id, newVectorSearchEndpointRemote(waiter.Response), nil
 }
 
-func (r *ResourceVectorSearchEndpoint) WaitAfterCreate(ctx context.Context, config *VectorSearchEndpointState) (*VectorSearchEndpointRemote, error) {
+func (r *ResourceVectorSearchEndpoint) WaitAfterCreate(ctx context.Context, config *vectorsearch.CreateEndpoint) (*VectorSearchEndpointRemote, error) {
 	info, err := r.client.VectorSearchEndpoints.WaitGetEndpointVectorSearchEndpointOnline(ctx, config.Name, 60*time.Minute, nil)
 	if err != nil {
 		return nil, err
 	}
 	return newVectorSearchEndpointRemote(info), nil
 }
 
-func (r *ResourceVectorSearchEndpoint) DoUpdate(ctx context.Context, id string, config *VectorSearchEndpointState, entry *PlanEntry) (*VectorSearchEndpointRemote, error) {
+func (r *ResourceVectorSearchEndpoint) DoUpdate(ctx context.Context, id string, config *vectorsearch.CreateEndpoint, entry *PlanEntry) (*VectorSearchEndpointRemote, error) {
 	if entry.Changes.HasChange(pathBudgetPolicyId) {
 		_, err := r.client.VectorSearchEndpoints.UpdateEndpointBudgetPolicy(ctx, vectorsearch.PatchEndpointBudgetPolicyRequest{
 			EndpointName:   id,
@@ -135,36 +107,9 @@ func (r *ResourceVectorSearchEndpoint) DoUpdate(ctx context.Context, id string,
 		}
 	}
 
-	// Preserve endpoint_uuid in saved state: PrepareState leaves it empty because
-	// it isn't in config, so copy from remote before SaveState writes newState.
-	if remote, ok := entry.RemoteState.(*VectorSearchEndpointRemote); ok && remote != nil {
-		config.EndpointUuid = remote.EndpointUuid
-	}
-
 	return nil, nil
 }
 
 func (r *ResourceVectorSearchEndpoint) DoDelete(ctx context.Context, id string) error {
 	return r.client.VectorSearchEndpoints.DeleteEndpointByEndpointName(ctx, id)
 }
-
-// OverrideChangeDesc classifies endpoint_uuid drift: Recreate when saved UUID
-// differs from remote (endpoint replaced out-of-band), Skip otherwise. The
-// field is not in config, so a synthetic diff between saved state and an empty
-// newState is expected on every plan.
-func (*ResourceVectorSearchEndpoint) OverrideChangeDesc(_ context.Context, path *structpath.PathNode, change *ChangeDesc, remote *VectorSearchEndpointRemote) error {
-	if path.String() != "endpoint_uuid" {
-		return nil
-	}
-	savedUuid, _ := change.Old.(string)
-	var remoteUuid string
-	if remote != nil {
-		remoteUuid = remote.EndpointUuid
-	}
-	if savedUuid != "" && remoteUuid != "" && savedUuid != remoteUuid {
-		change.Action = deployplan.Recreate
-	} else {
-		change.Action = deployplan.Skip
-	}
-	return nil
-}