Update vector search index test fixtures based on cloud (#5366)

## Changes
- Add missing/Fix parameters to DIRECT_ACCESS indexes
    - Enforce index name (3-part UC identifier) in test server
- Change DELTA_SYNC to DIRECT_ACCESS
- DELTA_SYNC requires a table, which is significant overhead for tests
    - Also change recreate test to recreate based on different parameter

## Why
Cloud tests are failing

## Tests
<!-- How have you tested the changes? -->

<!-- If your PR needs to be included in the release notes for next
release,
add a separate entry in NEXT_CHANGELOG.md as part of your PR. -->

Author: janniklasrose

acceptance/bundle/deployment/bind/vector_search_index/databricks.yml.tmpl

@@ -12,4 +12,7 @@ resources:
       primary_key: id
       index_type: DIRECT_ACCESS
       direct_access_index_spec:
-        schema_json: '{"columns":[{"name":"id","type":"integer"}]}'
+        schema_json: '{"id":"integer","vector":"array<float>"}'
+        embedding_vector_columns:
+          - name: vector
+            embedding_dimension: 768

acceptance/bundle/deployment/bind/vector_search_index/out.test.toml

@@ -5,7 +5,7 @@
   "endpoint_type": "STANDARD"
 }
 
->>> [CLI] vector-search-indexes create-index --json {"name":"main.default.test_vs_index_[UNIQUE_NAME]","endpoint_name":"test-vs-endpoint-[UNIQUE_NAME]","primary_key":"id","index_type":"DIRECT_ACCESS","direct_access_index_spec":{"schema_json":"{\"columns\":[{\"name\":\"id\",\"type\":\"integer\"}]}"}}
+>>> [CLI] vector-search-indexes create-index --json {"name":"main.default.test_vs_index_[UNIQUE_NAME]","endpoint_name":"test-vs-endpoint-[UNIQUE_NAME]","primary_key":"id","index_type":"DIRECT_ACCESS","direct_access_index_spec":{"schema_json":"{\"id\":\"integer\",\"vector\":\"array<float>\"}","embedding_vector_columns":[{"name":"vector","embedding_dimension":768}]}}
 {
   "name": "main.default.test_vs_index_[UNIQUE_NAME]",
   "endpoint_name": "test-vs-endpoint-[UNIQUE_NAME]",

acceptance/bundle/deployment/bind/vector_search_index/output.txt

@@ -11,7 +11,7 @@ trap cleanup EXIT
 
 trace $CLI vector-search-endpoints create-endpoint "${ENDPOINT_NAME}" STANDARD | jq '{name, endpoint_type}'
 
-trace $CLI vector-search-indexes create-index --json "{\"name\":\"${INDEX_NAME}\",\"endpoint_name\":\"${ENDPOINT_NAME}\",\"primary_key\":\"id\",\"index_type\":\"DIRECT_ACCESS\",\"direct_access_index_spec\":{\"schema_json\":\"{\\\"columns\\\":[{\\\"name\\\":\\\"id\\\",\\\"type\\\":\\\"integer\\\"}]}\"}}" | jq '{name, endpoint_name, index_type, primary_key}'
+trace $CLI vector-search-indexes create-index --json "{\"name\":\"${INDEX_NAME}\",\"endpoint_name\":\"${ENDPOINT_NAME}\",\"primary_key\":\"id\",\"index_type\":\"DIRECT_ACCESS\",\"direct_access_index_spec\":{\"schema_json\":\"{\\\"id\\\":\\\"integer\\\",\\\"vector\\\":\\\"array<float>\\\"}\",\"embedding_vector_columns\":[{\"name\":\"vector\",\"embedding_dimension\":768}]}}" | jq '{name, endpoint_name, index_type, primary_key}'
 
 trace $CLI bundle deployment bind index1 "${INDEX_NAME}" --auto-approve
 

acceptance/bundle/deployment/bind/vector_search_index/script

@@ -2,6 +2,9 @@ Local = true
 Cloud = true
 RequiresUnityCatalog = true
 
+# Vector Search indexes are slow to create, so only run them in the nightly (non-short) cloud job.
+CloudSlow = true
+
 Ignore = [
     ".databricks",
     "databricks.yml",

acceptance/bundle/deployment/bind/vector_search_index/test.toml

@@ -8,10 +8,12 @@ resources:
       endpoint_type: STANDARD
   vector_search_indexes:
     foo:
-      name: test-index-$UNIQUE_NAME
+      name: main.default.test_index_$UNIQUE_NAME
       endpoint_name: ${resources.vector_search_endpoints.bar.name}
       primary_key: id
-      index_type: DELTA_SYNC
-      delta_sync_index_spec:
-        source_table: main.default.source_$UNIQUE_NAME
-        pipeline_type: TRIGGERED
+      index_type: DIRECT_ACCESS
+      direct_access_index_spec:
+        schema_json: '{"id":"integer","vector":"array<float>"}'
+        embedding_vector_columns:
+          - name: vector
+            embedding_dimension: 768

acceptance/bundle/invariant/configs/vector_search_index.yml.tmpl

@@ -11,10 +11,12 @@ resources:
       endpoint_type: STANDARD
   vector_search_indexes:
     my_index:
-      name: vs-index-$UNIQUE_NAME
+      name: main.default.vs_index_$UNIQUE_NAME
       endpoint_name: ${resources.vector_search_endpoints.my_endpoint.name}
       primary_key: id
-      index_type: DELTA_SYNC
-      delta_sync_index_spec:
-        source_table: main.default.source_$UNIQUE_NAME
-        pipeline_type: TRIGGERED
+      index_type: DIRECT_ACCESS
+      direct_access_index_spec:
+        schema_json: '{"id":"integer","vector":"array<float>"}'
+        embedding_vector_columns:
+          - name: vector
+            embedding_dimension: 768

acceptance/bundle/resources/vector_search_indexes/basic/databricks.yml.tmpl

@@ -2,13 +2,18 @@
   "method": "POST",
   "path": "/api/2.0/vector-search/indexes",
   "body": {
-    "delta_sync_index_spec": {
-      "pipeline_type": "TRIGGERED",
-      "source_table": "main.default.source_[UNIQUE_NAME]"
+    "direct_access_index_spec": {
+      "embedding_vector_columns": [
+        {
+          "embedding_dimension": 768,
+          "name": "vector"
+        }
+      ],
+      "schema_json": "{\"id\":\"integer\",\"vector\":\"array<float>\"}"
     },
     "endpoint_name": "vs-endpoint-[UNIQUE_NAME]",
-    "index_type": "DELTA_SYNC",
-    "name": "vs-index-[UNIQUE_NAME]",
+    "index_type": "DIRECT_ACCESS",
+    "name": "main.default.vs_index_[UNIQUE_NAME]",
     "primary_key": "id"
   }
 }

acceptance/bundle/resources/vector_search_indexes/basic/out.requests.direct.json

@@ -21,20 +21,20 @@ Resources:
       URL:  [DATABRICKS_URL]/compute/vector-search/vs-endpoint-[UNIQUE_NAME]?w=[NUMID]
   Vector Search Indexes:
     my_index:
-      Name: vs-index-[UNIQUE_NAME]
-      URL:  (not deployed)
+      Name: main.default.vs_index_[UNIQUE_NAME]
+      URL:  [DATABRICKS_URL]/explore/data/main/default/vs_index_[UNIQUE_NAME]?w=[NUMID]
 
 >>> [CLI] bundle deploy
 Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/deploy-vs-index-[UNIQUE_NAME]/default/files...
 Deploying resources...
 Updating deployment state...
 Deployment complete!
 
->>> [CLI] vector-search-indexes get-index vs-index-[UNIQUE_NAME]
+>>> [CLI] vector-search-indexes get-index main.default.vs_index_[UNIQUE_NAME]
 {
-  "name": "vs-index-[UNIQUE_NAME]",
+  "name": "main.default.vs_index_[UNIQUE_NAME]",
   "endpoint_name": "vs-endpoint-[UNIQUE_NAME]",
-  "index_type": "DELTA_SYNC",
+  "index_type": "DIRECT_ACCESS",
   "primary_key": "id"
 }
 
@@ -51,8 +51,8 @@ Resources:
       URL:  [DATABRICKS_URL]/compute/vector-search/vs-endpoint-[UNIQUE_NAME]?w=[NUMID]
   Vector Search Indexes:
     my_index:
-      Name: vs-index-[UNIQUE_NAME]
-      URL:  (not deployed)
+      Name: main.default.vs_index_[UNIQUE_NAME]
+      URL:  [DATABRICKS_URL]/explore/data/main/default/vs_index_[UNIQUE_NAME]?w=[NUMID]
 
 >>> print_requests.py //vector-search/indexes