Merge branch 'main' into simonfaltum/b39-python-tmpdir-leak

Author: simonfaltum

.agent/rules/auto-generated-files.md

@@ -14,7 +14,6 @@ globs:
   - "internal/genkit/tagging.py"
   - "internal/mocks/**/*.go"
   - "bundle/direct/dresources/*.generated.yml"
-  - "bundle/docsgen/output/**/*.md"
   - "bundle/internal/schema/annotations_openapi.yml"
   - "bundle/internal/validation/generated/*.go"
   - "bundle/schema/jsonschema.json"
@@ -36,7 +35,6 @@ paths:
   - "internal/genkit/tagging.py"
   - "internal/mocks/**/*.go"
   - "bundle/direct/dresources/*.generated.yml"
-  - "bundle/docsgen/output/**/*.md"
   - "bundle/internal/schema/annotations_openapi.yml"
   - "bundle/internal/validation/generated/*.go"
   - "bundle/schema/jsonschema.json"
@@ -75,8 +73,6 @@ Files matching this rule's glob pattern are most likely generated artifacts. Aut
   - `./task generate-schema`
   - `./task generate-schema-docs`
   - This can also refresh `bundle/internal/schema/annotations_openapi.yml` when OpenAPI annotation extraction is enabled.
-- Bundle docs:
-  - `./task generate-docs`
 - Validation generated code:
   - `./task generate-validation`
 - Mock files:

.golangci.yaml

@@ -62,7 +62,6 @@ linters:
         no-stdlib-log:
           files:
             - "**"
-            - "!**/bundle/docsgen/**"
             - "!**/bundle/internal/schema/**"
             - "!**/bundle/internal/tf/codegen/**"
             - "!**/bundle/internal/validation/**"

.wsignore

@@ -26,8 +26,3 @@ python/databricks/bundles/*/_models/*.py
 
 # Developed elsewhere:
 internal/genkit/tagging.py
-
-# Docsgen:
-bundle/docsgen/output/resources.md
-bundle/docsgen/output/reference.md
-bundle/docsgen/testdata/anchors.md

AGENTS.md

@@ -40,7 +40,6 @@ This is the Databricks CLI, a command-line interface for interacting with Databr
 ### Specialized Commands
 
 - `./task generate-schema` - Generate bundle JSON schema
-- `./task generate-docs` - Generate bundle documentation
 - `./task generate-cligen` - Regenerate CLI command stubs from the checked-in .codegen/cli.json
 - `./task generate-clijson` - Refresh .codegen/cli.json from the OpenAPI spec via genkit (requires universe repo)
 - `./task generate` - Run all generators

NEXT_CHANGELOG.md

@@ -11,6 +11,7 @@
 * Remove API enum values and types that are still in development from the `databricks-bundles` Python package; these were never accepted by the backend ([#5484](https://github.com/databricks/cli/pull/5484)).
 * direct: Fix resolving a resource reference that is used more than once within the same field ([#5558](https://github.com/databricks/cli/pull/5558)).
 * Bundle variable references now accept Unicode letters in path segments (e.g. `${var.变量}`). ([#5532](https://github.com/databricks/cli/pull/5532))
+* Ignore remote changes for vector search direct_access_index_spec.schema_json to prevent drift when the backend normalizes the schema ([#5481](https://github.com/databricks/cli/pull/5481)).
 
 ### Dependency updates
 

Taskfile.yml

@@ -57,7 +57,6 @@ tasks:
       - task: generate-schema-map
       - task: generate-schema-docs
       - task: generate-validation
-      - task: generate-docs
       - task: generate-direct
       - task: pydabs-codegen
       - task: pydabs-lint
@@ -743,7 +742,6 @@ tasks:
       - task: generate-schema
       - task: generate-schema-docs
       - task: generate-validation
-      - task: generate-docs
       - task: generate-direct
       - task: pydabs-codegen
 
@@ -765,7 +763,6 @@ tasks:
       - task: generate-refschema
       - task: generate-schema
       - task: generate-validation
-      - task: generate-docs
       - task: generate-direct
       - task: pydabs-codegen
 
@@ -861,12 +858,12 @@ tasks:
 
   # Regenerates the OpenAPI annotation files (annotations_openapi*.yml) from the
   # checked-in .codegen/cli.json. This is the step that propagates a cli.json
-  # change into the schema/docs artifacts: generate-schema, generate-schema-docs
-  # and generate-docs all depend on it, so editing cli.json and running
-  # `task generate` percolates into jsonschema.json, the docs and pydabs.
+  # change into the schema/docs artifacts: generate-schema and generate-schema-docs
+  # both depend on it, so editing cli.json and running `task generate`
+  # percolates into jsonschema.json, jsonschema_for_docs.json and pydabs.
   generate-annotations:
     desc: Regenerate annotation files from .codegen/cli.json
-    # Dep of generate-schema, generate-schema-docs and generate-docs; `run: once`
+    # Dep of generate-schema and generate-schema-docs; `run: once`
     # plus the fingerprint keep `task generate` from re-running it per parent.
     run: once
     sources:
@@ -913,23 +910,6 @@ tasks:
       - git fetch origin 'refs/tags/v*:refs/tags/v*'
       - "go run ./bundle/internal/schema ./bundle/internal/schema ./bundle/schema/jsonschema_for_docs.json --docs"
 
-  generate-docs:
-    desc: Generate bundle documentation
-    deps: ['generate-annotations']
-    sources:
-      - "**/*.go"
-      - bundle/docsgen/templates/**
-      - bundle/internal/schema/annotations*.yml
-      - exclude: "**/*_test.go"
-      - go.mod
-      - go.sum
-      - "{{.EMBED_SOURCES}}"
-    generates:
-      - bundle/docsgen/output/reference.md
-      - bundle/docsgen/output/resources.md
-    cmds:
-      - "go run ./bundle/docsgen ./bundle/internal/schema ./bundle/docsgen"
-
   generate-validation:
     desc: Generate enum and required field validation code
     sources:

acceptance/bundle/deployment/bind/vector_search_index/databricks.yml.tmpl

@@ -12,7 +12,7 @@ resources:
       primary_key: id
       index_type: DIRECT_ACCESS
       direct_access_index_spec:
-        schema_json: '{"id":"integer","vector":"array<float>"}'
+        schema_json: '{"id":"int","vector":"array<float>"}'
         embedding_vector_columns:
           - name: vector
             embedding_dimension: 768

acceptance/bundle/deployment/bind/vector_search_index/output.txt

@@ -5,7 +5,7 @@
   "endpoint_type": "STANDARD"
 }
 
->>> [CLI] vector-search-indexes create-index --json {"name":"main.default.test_vs_index_[UNIQUE_NAME]","endpoint_name":"test-vs-endpoint-[UNIQUE_NAME]","primary_key":"id","index_type":"DIRECT_ACCESS","direct_access_index_spec":{"schema_json":"{\"id\":\"integer\",\"vector\":\"array<float>\"}","embedding_vector_columns":[{"name":"vector","embedding_dimension":768}]}}
+>>> [CLI] vector-search-indexes create-index --json {"name":"main.default.test_vs_index_[UNIQUE_NAME]","endpoint_name":"test-vs-endpoint-[UNIQUE_NAME]","primary_key":"id","index_type":"DIRECT_ACCESS","direct_access_index_spec":{"schema_json":"{\"id\":\"int\",\"vector\":\"array<float>\"}","embedding_vector_columns":[{"name":"vector","embedding_dimension":768}]}}
 {
   "name": "main.default.test_vs_index_[UNIQUE_NAME]",
   "endpoint_name": "test-vs-endpoint-[UNIQUE_NAME]",

acceptance/bundle/deployment/bind/vector_search_index/script

@@ -11,7 +11,7 @@ trap cleanup EXIT
 
 trace $CLI vector-search-endpoints create-endpoint "${ENDPOINT_NAME}" STANDARD | jq '{name, endpoint_type}'
 
-trace $CLI vector-search-indexes create-index --json "{\"name\":\"${INDEX_NAME}\",\"endpoint_name\":\"${ENDPOINT_NAME}\",\"primary_key\":\"id\",\"index_type\":\"DIRECT_ACCESS\",\"direct_access_index_spec\":{\"schema_json\":\"{\\\"id\\\":\\\"integer\\\",\\\"vector\\\":\\\"array<float>\\\"}\",\"embedding_vector_columns\":[{\"name\":\"vector\",\"embedding_dimension\":768}]}}" | jq '{name, endpoint_name, index_type, primary_key}'
+trace $CLI vector-search-indexes create-index --json "{\"name\":\"${INDEX_NAME}\",\"endpoint_name\":\"${ENDPOINT_NAME}\",\"primary_key\":\"id\",\"index_type\":\"DIRECT_ACCESS\",\"direct_access_index_spec\":{\"schema_json\":\"{\\\"id\\\":\\\"int\\\",\\\"vector\\\":\\\"array<float>\\\"}\",\"embedding_vector_columns\":[{\"name\":\"vector\",\"embedding_dimension\":768}]}}" | jq '{name, endpoint_name, index_type, primary_key}'
 
 trace $CLI bundle deployment bind index1 "${INDEX_NAME}" --auto-approve
 

acceptance/bundle/resources/vector_search_indexes/recreate/with_endpoint/output.txt

@@ -69,7 +69,7 @@ Plan: 1 to add, 0 to change, 1 to delete, 1 unchanged
               "name": "vector"
             }
           ],
-          "schema_json": "{\"id\":\"integer\",\"vector\":\"array\u003cfloat\u003e\"}"
+          "schema_json": "{\"id\":\"int\",\"vector\":\"array\u003cfloat\u003e\"}"
         },
         "endpoint_name": "vs-endpoint-[UNIQUE_NAME]",
         "endpoint_uuid": "[UUID]",
@@ -82,6 +82,13 @@ Plan: 1 to add, 0 to change, 1 to delete, 1 unchanged
         }
       },
       "changes": {
+        "direct_access_index_spec.schema_json": {
+          "action": "skip",
+          "reason": "normalized_by_backend",
+          "old": "{\"id\":\"integer\",\"vector\":\"array\u003cfloat\u003e\"}",
+          "new": "{\"id\":\"integer\",\"vector\":\"array\u003cfloat\u003e\"}",
+          "remote": "{\"id\":\"int\",\"vector\":\"array\u003cfloat\u003e\"}"
+        },
         "endpoint_uuid": {
           "action": "skip",
           "reason": "state-only field",