Consolidate bundle schema annotations into a single tree-structured file (#5574)

## Changes
- Replace the three type-name-keyed annotation files
(`annotations_openapi.yml`, `annotations_openapi_overrides.yml`,
`annotations.yml`) with one `annotations.yml` structured like the bundle
config tree.
- Derive upstream docs (descriptions, enums, launch stages, field
behaviors) from `.codegen/cli.json` in memory at generation time,
instead of checking in ~7k lines of generated YAML. The file now holds
only CLI-owned docs and overrides.
- Structural keys are `$type`/`$fields`; absent entries inherit from
cli.json, and `description: PLACEHOLDER` flags undocumented fields.
- `task generate-schema` keeps the file in lockstep with the config
structure: it adds placeholders for new fields and drops entries for
fields the schema no longer emits.

## Why
The old files duplicated cli.json data that went stale between
regenerations and were keyed by Go type name, which hid where a field
actually lives. One structured file gives a single place for CLI-owned
docs, no duplicated upstream content, and a layout that reads like the
config it documents.

## Tests
- `bundle/schema/jsonschema.json` and `jsonschema_for_docs.json` are
byte-for-byte identical to `main` — this is purely an internal
representation change.
- New unit tests for the type graph and the file codec; existing
annotation tests carried over.

_This PR was written by Isaac._

Author: pietern

.agent/rules/auto-generated-files.md

@@ -14,7 +14,6 @@ globs:
   - "internal/genkit/tagging.py"
   - "internal/mocks/**/*.go"
   - "bundle/direct/dresources/*.generated.yml"
-  - "bundle/internal/schema/annotations_openapi.yml"
   - "bundle/internal/validation/generated/*.go"
   - "bundle/schema/jsonschema.json"
   - "bundle/schema/jsonschema_for_docs.json"
@@ -35,7 +34,6 @@ paths:
   - "internal/genkit/tagging.py"
   - "internal/mocks/**/*.go"
   - "bundle/direct/dresources/*.generated.yml"
-  - "bundle/internal/schema/annotations_openapi.yml"
   - "bundle/internal/validation/generated/*.go"
   - "bundle/schema/jsonschema.json"
   - "bundle/schema/jsonschema_for_docs.json"
@@ -72,7 +70,7 @@ Files matching this rule's glob pattern are most likely generated artifacts. Aut
 - Bundle schemas:
   - `./task generate-schema`
   - `./task generate-schema-docs`
-  - This can also refresh `bundle/internal/schema/annotations_openapi.yml` when OpenAPI annotation extraction is enabled.
+  - Both rewrite `bundle/internal/schema/annotations.yml` in place: upstream docs are sourced from `.codegen/cli.json` at generation time, and the file is synced with the config structure (placeholders added, stale entries dropped).
 - Validation generated code:
   - `./task generate-validation`
 - Mock files:

Taskfile.yml

@@ -856,35 +856,17 @@ tasks:
     cmds:
       - go test ./acceptance -run TestAccept/bundle/refschema -update
 
-  # Regenerates the OpenAPI annotation files (annotations_openapi*.yml) from the
-  # checked-in .codegen/cli.json. This is the step that propagates a cli.json
-  # change into the schema/docs artifacts: generate-schema and generate-schema-docs
-  # both depend on it, so editing cli.json and running `task generate`
-  # percolates into jsonschema.json, jsonschema_for_docs.json and pydabs.
-  generate-annotations:
-    desc: Regenerate annotation files from .codegen/cli.json
-    # Dep of generate-schema and generate-schema-docs; `run: once`
-    # plus the fingerprint keep `task generate` from re-running it per parent.
-    run: once
-    sources:
-      - .codegen/cli.json
-      - bundle/internal/schema/**/*.go
-      - bundle/internal/schema/annotations*.yml
-      - go.mod
-      - go.sum
-    generates:
-      - bundle/internal/schema/annotations_openapi.yml
-      - bundle/internal/schema/annotations.yml
-      - bundle/schema/jsonschema.json
-    cmds:
-      - "DATABRICKS_CLI_JSON=.codegen/cli.json go run ./bundle/internal/schema ./bundle/internal/schema ./bundle/schema/jsonschema.json"
-
+  # Upstream field documentation comes from the checked-in .codegen/cli.json;
+  # bundle/internal/schema/annotations.yml carries the CLI-owned docs and
+  # overrides and is rewritten in place (synced with the config structure).
+  # Editing either input and running `task generate` percolates into
+  # jsonschema.json, jsonschema_for_docs.json and pydabs.
   generate-schema:
     desc: Generate bundle JSON schema
-    deps: ['generate-annotations']
     sources: &SCHEMA_SOURCES
       - "**/*.go"
-      - bundle/internal/schema/annotations*.yml
+      - .codegen/cli.json
+      - bundle/internal/schema/annotations.yml
       - exclude: "**/*_test.go"
       - go.mod
       - go.sum
@@ -893,11 +875,10 @@ tasks:
       - bundle/schema/jsonschema.json
       - bundle/internal/schema/annotations.yml
     cmds:
-      - "go run ./bundle/internal/schema ./bundle/internal/schema ./bundle/schema/jsonschema.json"
+      - "go run ./bundle/internal/schema ./bundle/internal/schema ./bundle/schema/jsonschema.json .codegen/cli.json"
 
   generate-schema-docs:
     desc: Generate bundle JSON schema for documentation
-    deps: ['generate-annotations']
     sources: *SCHEMA_SOURCES
     generates:
       - bundle/schema/jsonschema_for_docs.json
@@ -908,7 +889,7 @@ tasks:
       # silently dropped from the output. Restore the fetch that lived in the
       # old tools/post-generate.sh.
       - git fetch origin 'refs/tags/v*:refs/tags/v*'
-      - "go run ./bundle/internal/schema ./bundle/internal/schema ./bundle/schema/jsonschema_for_docs.json --docs"
+      - "go run ./bundle/internal/schema ./bundle/internal/schema ./bundle/schema/jsonschema_for_docs.json .codegen/cli.json --docs"
 
   generate-validation:
     desc: Generate enum and required field validation code

bundle/internal/annotation/file.go

@@ -1,44 +1,32 @@
 package annotation
 
-import (
-	"bytes"
-	"os"
-
-	"github.com/databricks/cli/libs/dyn"
-	"github.com/databricks/cli/libs/dyn/convert"
-	"github.com/databricks/cli/libs/dyn/merge"
-	"github.com/databricks/cli/libs/dyn/yamlloader"
-)
+// TypeAnnotation holds the documentation for one Go type. Self documents the
+// type itself — it is applied to the type's JSON-schema $defs entry and is
+// where enum values live — and Fields documents each of the type's fields by
+// JSON name.
+type TypeAnnotation struct {
+	Self   Descriptor            `json:"type,omitempty"`
+	Fields map[string]Descriptor `json:"fields,omitempty"`
+}
 
-// Parsed file with annotations, expected format:
-// github.com/databricks/cli/bundle/config.Bundle:
-//
-//	cluster_id:
-//	   description: "Description"
-type File map[string]map[string]Descriptor
+// File is the in-memory annotations, keyed by Go type path, e.g.
+// "github.com/databricks/cli/bundle/config.Bundle".
+type File map[string]TypeAnnotation
 
-func LoadAndMerge(sources []string) (File, error) {
-	prev := dyn.NilValue
-	for _, path := range sources {
-		b, err := os.ReadFile(path)
-		if err != nil {
-			return nil, err
-		}
-		generated, err := yamlloader.LoadYAML(path, bytes.NewBuffer(b))
-		if err != nil {
-			return nil, err
-		}
-		prev, err = merge.Merge(prev, generated)
-		if err != nil {
-			return nil, err
-		}
+// SetField stores a descriptor for a field of typeKey, allocating the entry
+// and its field map as needed.
+func (f File) SetField(typeKey, name string, d Descriptor) {
+	ta := f[typeKey]
+	if ta.Fields == nil {
+		ta.Fields = map[string]Descriptor{}
 	}
+	ta.Fields[name] = d
+	f[typeKey] = ta
+}
 
-	var data File
-
-	err := convert.ToTyped(&data, prev)
-	if err != nil {
-		return nil, err
-	}
-	return data, nil
+// SetSelf stores the descriptor for the type itself.
+func (f File) SetSelf(typeKey string, d Descriptor) {
+	ta := f[typeKey]
+	ta.Self = d
+	f[typeKey] = ta
 }

bundle/internal/annotation/file_test.go

@@ -0,0 +1,36 @@
+package annotation
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestFileSetField(t *testing.T) {
+	f := File{}
+
+	// SetField allocates the type entry and its field map on first use.
+	f.SetField("pkg.Type", "a", Descriptor{Description: "A"})
+	f.SetField("pkg.Type", "b", Descriptor{Description: "B"})
+
+	assert.Equal(t, "A", f["pkg.Type"].Fields["a"].Description)
+	assert.Equal(t, "B", f["pkg.Type"].Fields["b"].Description)
+
+	// A later SetField overwrites the field, leaving siblings intact.
+	f.SetField("pkg.Type", "a", Descriptor{Description: "A2"})
+	assert.Equal(t, "A2", f["pkg.Type"].Fields["a"].Description)
+	assert.Equal(t, "B", f["pkg.Type"].Fields["b"].Description)
+}
+
+func TestFileSetSelf(t *testing.T) {
+	f := File{}
+
+	f.SetSelf("pkg.Type", Descriptor{Description: "the type"})
+	assert.Equal(t, "the type", f["pkg.Type"].Self.Description)
+
+	// SetSelf and SetField populate the same entry without clobbering each
+	// other.
+	f.SetField("pkg.Type", "a", Descriptor{Description: "A"})
+	assert.Equal(t, "the type", f["pkg.Type"].Self.Description)
+	assert.Equal(t, "A", f["pkg.Type"].Fields["a"].Description)
+}