Drop the vector-search recreate-cascade HACK from bundle_plan

The HACK in bundle_plan.go force-cascaded Recreate from
vector_search_endpoints to dependent vector_search_indexes. Remove it
and surface the gap explicitly in the acceptance test: after recreating
the endpoint, a second plan/deploy is required to reconcile the index's
stored endpoint_uuid. A generic per-resource cascade rule is the proper
fix and will land in a follow-up.

Also have the testserver capture the endpoint UUID on indexes at create
time (fakeVectorSearchIndex wrapper, omitted from JSON responses) so
future tests can reason about the orphan scenario without inventing
state out of thin air.

Co-authored-by: Isaac

Author: janniklasrose

acceptance/bundle/resources/vector_search_indexes/recreate/with_endpoint/output.txt

@@ -6,14 +6,13 @@ Deploying resources...
 Updating deployment state...
 Deployment complete!
 
-=== Change endpoint_type (should recreate endpoint AND its dependent index)
+=== Change endpoint_type (recreates endpoint; index recreate cascade is not yet implemented)
 >>> update_file.py databricks.yml endpoint_type: STANDARD endpoint_type: STORAGE_OPTIMIZED
 
 >>> [CLI] bundle plan
 recreate vector_search_endpoints.my_endpoint
-recreate vector_search_indexes.my_index
 
-Plan: 2 to add, 0 to change, 2 to delete, 0 unchanged
+Plan: 1 to add, 0 to change, 1 to delete, 1 unchanged
 
 >>> [CLI] bundle plan --output json
 {
@@ -60,18 +59,7 @@ Plan: 2 to add, 0 to change, 2 to delete, 0 unchanged
           "label": "${resources.vector_search_endpoints.my_endpoint.name}"
         }
       ],
-      "action": "recreate",
-      "new_state": {
-        "value": {
-          "direct_access_index_spec": {
-            "schema_json": "{\"columns\":[{\"name\":\"id\",\"type\":\"integer\"}]}"
-          },
-          "endpoint_name": "vs-endpoint-[UNIQUE_NAME]",
-          "index_type": "DIRECT_ACCESS",
-          "name": "vs-index-[UNIQUE_NAME]",
-          "primary_key": "id"
-        }
-      },
+      "action": "skip",
       "remote_state": {
         "creator": "[USERNAME]",
         "direct_access_index_spec": {
@@ -100,11 +88,6 @@ Plan: 2 to add, 0 to change, 2 to delete, 0 unchanged
 
 >>> [CLI] bundle deploy --auto-approve
 Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/deploy-vs-index-with-endpoint-[UNIQUE_NAME]/default/files...
-
-This action will result in the deletion or recreation of the following Vector Search indexes.
-Recreating a Delta Sync index re-runs the full embedding pipeline; recreating a Direct Access
-index drops all upserted vectors. Both can be expensive to rebuild:
-  recreate resources.vector_search_indexes.my_index
 Deploying resources...
 Updating deployment state...
 Deployment complete!
@@ -123,6 +106,23 @@ Deployment complete!
   "primary_key": "id"
 }
 
+=== Re-plan exposes the leftover drift: the index still points at the old endpoint UUID
+>>> [CLI] bundle plan
+recreate vector_search_indexes.my_index
+
+Plan: 1 to add, 0 to change, 1 to delete, 1 unchanged
+
+>>> [CLI] bundle deploy --auto-approve
+Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/deploy-vs-index-with-endpoint-[UNIQUE_NAME]/default/files...
+
+This action will result in the deletion or recreation of the following Vector Search indexes.
+Recreating a Delta Sync index re-runs the full embedding pipeline; recreating a Direct Access
+index drops all upserted vectors. Both can be expensive to rebuild:
+  recreate resources.vector_search_indexes.my_index
+Deploying resources...
+Updating deployment state...
+Deployment complete!
+
 >>> [CLI] bundle destroy --auto-approve
 The following resources will be deleted:
   delete resources.vector_search_endpoints.my_endpoint

acceptance/bundle/resources/vector_search_indexes/recreate/with_endpoint/script

@@ -10,7 +10,7 @@ title "Initial deployment with STANDARD endpoint_type"
 rm -f out.requests.txt
 trace $CLI bundle deploy
 
-title "Change endpoint_type (should recreate endpoint AND its dependent index)"
+title "Change endpoint_type (recreates endpoint; index recreate cascade is not yet implemented)"
 trace update_file.py databricks.yml "endpoint_type: STANDARD" "endpoint_type: STORAGE_OPTIMIZED"
 
 trace $CLI bundle plan
@@ -23,3 +23,7 @@ endpoint_name="vs-endpoint-${UNIQUE_NAME}"
 index_name="vs-index-${UNIQUE_NAME}"
 trace $CLI vector-search-endpoints get-endpoint "${endpoint_name}" | jq '{name, endpoint_type}'
 trace $CLI vector-search-indexes get-index "${index_name}" | jq '{name, endpoint_name, index_type, primary_key}'
+
+title "Re-plan exposes the leftover drift: the index still points at the old endpoint UUID"
+trace $CLI bundle plan
+trace $CLI bundle deploy --auto-approve

acceptance/bundle/resources/vector_search_indexes/recreate/with_endpoint/test.toml

@@ -1 +1,3 @@
 Cloud = false
+
+Badness = "Recreating a vector_search_endpoint does not cascade to its dependent vector_search_indexes. The first deploy recreates the endpoint and leaves the index attached to a stale endpoint UUID; a second plan/deploy is required to reconcile the index. Follow-up will add a generic per-resource recreate cascade rule to the framework."

bundle/direct/bundle_plan.go

@@ -283,31 +283,6 @@ func (b *DeploymentBundle) CalculatePlan(ctx context.Context, client *databricks
 		return nil, errors.New("planning failed")
 	}
 
-	// HACK: cascade Recreate from vector_search_endpoints to dependent
-	// vector_search_indexes. The framework has no per-resource recreate
-	// cascade rule yet; without this, recreating an endpoint orphans its
-	// indexes (or fails because indexes are still attached on delete).
-	// Replace with a generic rule when the framework grows one.
-	for resourceKey, entry := range plan.Plan {
-		if config.GetResourceTypeFromKey(resourceKey) != "vector_search_indexes" {
-			continue
-		}
-		if entry.Action == deployplan.Create || entry.Action == deployplan.Recreate || entry.Action == deployplan.Delete {
-			continue
-		}
-		for _, dep := range entry.DependsOn {
-			if config.GetResourceTypeFromKey(dep.Node) != "vector_search_endpoints" {
-				continue
-			}
-			parent, ok := plan.Plan[dep.Node]
-			if !ok || parent.Action != deployplan.Recreate {
-				continue
-			}
-			entry.Action = deployplan.Recreate
-			break
-		}
-	}
-
 	for _, entry := range plan.Plan {
 		if entry.Action == deployplan.Skip {
 			entry.NewState = nil

libs/testserver/fake_workspace.go

@@ -152,7 +152,7 @@ type FakeWorkspace struct {
 	RegisteredModels      map[string]catalog.RegisteredModelInfo
 	ServingEndpoints      map[string]serving.ServingEndpointDetailed
 	VectorSearchEndpoints map[string]vectorsearch.EndpointInfo
-	VectorSearchIndexes   map[string]vectorsearch.VectorIndex
+	VectorSearchIndexes   map[string]fakeVectorSearchIndex
 
 	SecretScopes map[string]workspace.SecretScope
 	Secrets      map[string]map[string]string // scope -> key -> value
@@ -288,7 +288,7 @@ func NewFakeWorkspace(url, token string) *FakeWorkspace {
 		},
 		ServingEndpoints:      map[string]serving.ServingEndpointDetailed{},
 		VectorSearchEndpoints: map[string]vectorsearch.EndpointInfo{},
-		VectorSearchIndexes:   map[string]vectorsearch.VectorIndex{},
+		VectorSearchIndexes:   map[string]fakeVectorSearchIndex{},
 		Repos:                 map[string]workspace.RepoInfo{},
 		SecretScopes:          map[string]workspace.SecretScope{},
 		Secrets:               map[string]map[string]string{},

libs/testserver/vector_search_indexes.go

@@ -8,6 +8,18 @@ import (
 	"github.com/databricks/databricks-sdk-go/service/vectorsearch"
 )
 
+// fakeVectorSearchIndex captures the endpoint's UUID at index creation time.
+// On the real backend an index is bound to a specific endpoint instance, not
+// just the name: deleting and recreating an endpoint with the same name yields
+// a different UUID, and the existing index keeps pointing at the OLD UUID
+// (i.e. is orphaned). Tracking this here lets tests reason about that drift.
+// The field is omitted from JSON responses since the real API doesn't return
+// it on the index path; the CLI looks it up via GetEndpointByEndpointName.
+type fakeVectorSearchIndex struct {
+	vectorsearch.VectorIndex
+	EndpointUuid string `json:"-"`
+}
+
 func (s *FakeWorkspace) VectorSearchIndexCreate(req Request) Response {
 	defer s.LockUnlock()()
 
@@ -25,7 +37,8 @@ func (s *FakeWorkspace) VectorSearchIndexCreate(req Request) Response {
 			Body:       map[string]string{"error_code": "RESOURCE_ALREADY_EXISTS", "message": fmt.Sprintf("Vector search index with name %s already exists", createReq.Name)},
 		}
 	}
-	if _, exists := s.VectorSearchEndpoints[createReq.EndpointName]; !exists {
+	endpoint, exists := s.VectorSearchEndpoints[createReq.EndpointName]
+	if !exists {
 		return Response{
 			StatusCode: http.StatusNotFound,
 			Body: map[string]string{
@@ -35,17 +48,20 @@ func (s *FakeWorkspace) VectorSearchIndexCreate(req Request) Response {
 		}
 	}
 
-	index := vectorsearch.VectorIndex{
-		Creator:               s.CurrentUser().UserName,
-		EndpointName:          createReq.EndpointName,
-		IndexType:             createReq.IndexType,
-		Name:                  createReq.Name,
-		PrimaryKey:            createReq.PrimaryKey,
-		DeltaSyncIndexSpec:    remapDeltaSyncSpec(createReq.DeltaSyncIndexSpec),
-		DirectAccessIndexSpec: createReq.DirectAccessIndexSpec,
-		Status: &vectorsearch.VectorIndexStatus{
-			Ready: true,
+	index := fakeVectorSearchIndex{
+		VectorIndex: vectorsearch.VectorIndex{
+			Creator:               s.CurrentUser().UserName,
+			EndpointName:          createReq.EndpointName,
+			IndexType:             createReq.IndexType,
+			Name:                  createReq.Name,
+			PrimaryKey:            createReq.PrimaryKey,
+			DeltaSyncIndexSpec:    remapDeltaSyncSpec(createReq.DeltaSyncIndexSpec),
+			DirectAccessIndexSpec: createReq.DirectAccessIndexSpec,
+			Status: &vectorsearch.VectorIndexStatus{
+				Ready: true,
+			},
 		},
+		EndpointUuid: endpoint.Id,
 	}
 
 	s.VectorSearchIndexes[createReq.Name] = index