bundle: reject and hide deployment_id/version_id on jobs and pipelines

deployment_id and version_id on a job/pipeline deployment block are set by
the CLI to track the bundle deployment in the Deployment Metadata Service.
They are not user-configurable, so this change:

- Errors at validation time if either is set in bundle configuration
  (new validate.ValidateDeploymentFields mutator).
- Adds a hide_from_plan lifecycle annotation to the direct engine that
  removes a field from the plan diff (display only), and applies it to
  deployment.version_id for jobs and pipelines (version_id is set on every
  deploy, so showing it is pure noise). version_id also gets
  ignore_local_changes/ignore_remote_changes, which are what keep its churn
  from driving an update or showing as drift. deployment_id is intentionally
  left to show in the plan.
- Removes deployment_id/version_id from the generated JSON schema.

Co-authored-by: Isaac

Author: shreyas-goenka

acceptance/bundle/validate/reserved_deployment_fields/databricks.yml

@@ -0,0 +1,18 @@
+bundle:
+  name: test-bundle
+
+resources:
+  jobs:
+    my_job:
+      name: my_job
+      deployment:
+        kind: BUNDLE
+        deployment_id: "dep-123"
+        version_id: "ver-456"
+  pipelines:
+    my_pipeline:
+      name: my_pipeline
+      deployment:
+        kind: BUNDLE
+        deployment_id: "dep-789"
+        version_id: "ver-012"

acceptance/bundle/validate/reserved_deployment_fields/out.test.toml

@@ -0,0 +1,27 @@
+
+>>> [CLI] bundle validate
+Error: deployment_id must not be set in bundle configuration; it is managed by Databricks Asset Bundles
+  at resources.jobs.my_job.deployment.deployment_id
+  in databricks.yml:10:24
+
+Error: deployment_id must not be set in bundle configuration; it is managed by Databricks Asset Bundles
+  at resources.pipelines.my_pipeline.deployment.deployment_id
+  in databricks.yml:17:24
+
+Error: version_id must not be set in bundle configuration; it is managed by Databricks Asset Bundles
+  at resources.jobs.my_job.deployment.version_id
+  in databricks.yml:11:21
+
+Error: version_id must not be set in bundle configuration; it is managed by Databricks Asset Bundles
+  at resources.pipelines.my_pipeline.deployment.version_id
+  in databricks.yml:18:21
+
+Name: test-bundle
+Target: default
+Workspace:
+  User: [USERNAME]
+  Path: /Workspace/Users/[USERNAME]/.bundle/test-bundle/default
+
+Found 4 errors
+
+Exit code: 1

acceptance/bundle/validate/reserved_deployment_fields/output.txt

@@ -0,0 +1 @@
+trace $CLI bundle validate

acceptance/bundle/validate/reserved_deployment_fields/script

@@ -0,0 +1,66 @@
+package validate
+
+import (
+	"cmp"
+	"context"
+	"fmt"
+	"slices"
+
+	"github.com/databricks/cli/bundle"
+	"github.com/databricks/cli/libs/diag"
+	"github.com/databricks/cli/libs/dyn"
+)
+
+// reservedDeploymentFields are fields on a resource's "deployment" block that
+// the CLI sets on the user's behalf: they identify the bundle deployment and
+// its version in the Deployment Metadata Service. A value set by hand would be
+// overwritten at deploy time, so we reject it up front with a clear error
+// rather than silently dropping it.
+var reservedDeploymentFields = []string{"deployment_id", "version_id"}
+
+// reservedDeploymentResources are the resource types whose deployment block
+// carries the reserved fields above (jobs.JobDeployment, pipelines.PipelineDeployment).
+var reservedDeploymentResources = []string{"jobs", "pipelines"}
+
+func ValidateDeploymentFields() bundle.ReadOnlyMutator {
+	return &validateDeploymentFields{}
+}
+
+type validateDeploymentFields struct{ bundle.RO }
+
+func (v *validateDeploymentFields) Name() string {
+	return "validate:validate_deployment_fields"
+}
+
+func (v *validateDeploymentFields) Apply(_ context.Context, b *bundle.Bundle) diag.Diagnostics {
+	var diags diag.Diagnostics
+
+	for _, resourceType := range reservedDeploymentResources {
+		for _, field := range reservedDeploymentFields {
+			pattern := dyn.NewPattern(dyn.Key("resources"), dyn.Key(resourceType), dyn.AnyKey(), dyn.Key("deployment"), dyn.Key(field))
+			_, err := dyn.MapByPattern(b.Config.Value(), pattern, func(p dyn.Path, val dyn.Value) (dyn.Value, error) {
+				diags = append(diags, diag.Diagnostic{
+					Severity:  diag.Error,
+					Summary:   field + " must not be set in bundle configuration; it is managed by Databricks Asset Bundles",
+					Paths:     []dyn.Path{p},
+					Locations: val.Locations(),
+				})
+				return val, nil
+			})
+			if err != nil {
+				diags = diags.Extend(diag.FromErr(err))
+			}
+		}
+	}
+
+	// MapByPattern visits resources in map order, which is randomized; sort so
+	// the reported diagnostics are stable across runs.
+	slices.SortFunc(diags, func(a, b diag.Diagnostic) int {
+		if n := cmp.Compare(a.Summary, b.Summary); n != 0 {
+			return n
+		}
+		return cmp.Compare(fmt.Sprintf("%v", a.Paths), fmt.Sprintf("%v", b.Paths))
+	})
+
+	return diags
+}

bundle/config/validate/validate_deployment_fields.go

@@ -0,0 +1,80 @@
+package validate
+
+import (
+	"testing"
+
+	"github.com/databricks/cli/bundle"
+	"github.com/databricks/cli/bundle/config"
+	"github.com/databricks/cli/bundle/config/resources"
+	"github.com/databricks/cli/bundle/internal/bundletest"
+	"github.com/databricks/cli/libs/diag"
+	"github.com/databricks/cli/libs/dyn"
+	"github.com/databricks/databricks-sdk-go/service/jobs"
+	"github.com/databricks/databricks-sdk-go/service/pipelines"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+func deploymentFieldsBundle(t *testing.T) *bundle.Bundle {
+	t.Helper()
+	return &bundle.Bundle{
+		Config: config.Root{
+			Resources: config.Resources{
+				Jobs: map[string]*resources.Job{
+					"my_job": {JobSettings: jobs.JobSettings{Name: "my_job"}},
+				},
+				Pipelines: map[string]*resources.Pipeline{
+					"my_pipeline": {CreatePipeline: pipelines.CreatePipeline{Name: "my_pipeline"}},
+				},
+			},
+		},
+	}
+}
+
+func TestValidateDeploymentFieldsRejectsReservedFields(t *testing.T) {
+	tests := []struct {
+		name         string
+		resourcePath string
+		field        string
+		want         string
+	}{
+		{"job deployment_id", "resources.jobs.my_job.deployment", "deployment_id", "deployment_id must not be set in bundle configuration; it is managed by Databricks Asset Bundles"},
+		{"job version_id", "resources.jobs.my_job.deployment", "version_id", "version_id must not be set in bundle configuration; it is managed by Databricks Asset Bundles"},
+		{"pipeline deployment_id", "resources.pipelines.my_pipeline.deployment", "deployment_id", "deployment_id must not be set in bundle configuration; it is managed by Databricks Asset Bundles"},
+		{"pipeline version_id", "resources.pipelines.my_pipeline.deployment", "version_id", "version_id must not be set in bundle configuration; it is managed by Databricks Asset Bundles"},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			b := deploymentFieldsBundle(t)
+			bundletest.Mutate(t, b, func(v dyn.Value) (dyn.Value, error) {
+				return dyn.Set(v, tt.resourcePath, dyn.V(map[string]dyn.Value{
+					tt.field: dyn.V("should-not-be-here"),
+				}))
+			})
+
+			diags := ValidateDeploymentFields().Apply(t.Context(), b)
+			require.Len(t, diags, 1)
+			assert.Equal(t, diag.Error, diags[0].Severity)
+			assert.Equal(t, tt.want, diags[0].Summary)
+		})
+	}
+}
+
+func TestValidateDeploymentFieldsReportsAllOffenders(t *testing.T) {
+	b := deploymentFieldsBundle(t)
+	bundletest.Mutate(t, b, func(v dyn.Value) (dyn.Value, error) {
+		v, err := dyn.Set(v, "resources.jobs.my_job.deployment", dyn.V(map[string]dyn.Value{
+			"deployment_id": dyn.V("a"),
+		}))
+		if err != nil {
+			return v, err
+		}
+		return dyn.Set(v, "resources.pipelines.my_pipeline.deployment", dyn.V(map[string]dyn.Value{
+			"version_id": dyn.V("b"),
+		}))
+	})
+
+	diags := ValidateDeploymentFields().Apply(t.Context(), b)
+	require.Len(t, diags, 2)
+}

bundle/config/validate/validate_deployment_fields_test.go

@@ -357,7 +357,12 @@ func addPerFieldActions(ctx context.Context, adapter *dresources.Adapter, change
 			return err
 		}
 
-		if structdiff.IsEqual(ch.Remote, ch.New) {
+		if shouldDropFromPlan(cfg, path) || shouldDropFromPlan(generatedCfg, path) {
+			// Drop the field from the plan entirely. ReasonDrop removes the entry
+			// below so it never surfaces as a change regardless of its diff state.
+			ch.Action = deployplan.Skip
+			ch.Reason = deployplan.ReasonDrop
+		} else if structdiff.IsEqual(ch.Remote, ch.New) {
 			ch.Action = deployplan.Skip
 			ch.Reason = deployplan.ReasonRemoteAlreadySet
 		} else if allEmpty(ch.Old, ch.New, ch.Remote) {
@@ -434,6 +439,16 @@ func shouldSkip(cfg *dresources.ResourceLifecycleConfig, path *structpath.PathNo
 	return "", false
 }
 
+// shouldDropFromPlan reports whether the field matches a HideFromPlan rule, meaning
+// it should be removed from the plan entirely rather than shown as a change.
+func shouldDropFromPlan(cfg *dresources.ResourceLifecycleConfig, path *structpath.PathNode) bool {
+	if cfg == nil {
+		return false
+	}
+	_, ok := findMatchingRule(path, cfg.HideFromPlan)
+	return ok
+}
+
 func shouldUpdateOrRecreate(cfg *dresources.ResourceLifecycleConfig, path *structpath.PathNode) (deployplan.ActionType, string) {
 	if cfg == nil {
 		return deployplan.Undefined, ""

bundle/direct/bundle_plan.go

@@ -3,8 +3,11 @@ package direct
 import (
 	"testing"
 
+	"github.com/databricks/cli/bundle/direct/dresources"
 	"github.com/databricks/cli/libs/dyn"
+	"github.com/databricks/cli/libs/structs/structpath"
 	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
 )
 
 func TestDynPathToStructPath(t *testing.T) {
@@ -35,3 +38,19 @@ func TestDynPathToStructPath(t *testing.T) {
 		assert.Equal(t, tc.expected, node.String())
 	}
 }
+
+// TestShouldDropFromPlanDeploymentVersion verifies that the DMS deployment
+// version_id is dropped from the plan for jobs and pipelines, while
+// deployment_id is left in (it is stable and fine to show).
+func TestShouldDropFromPlanDeploymentVersion(t *testing.T) {
+	versionID, err := structpath.ParsePath("deployment.version_id")
+	require.NoError(t, err)
+	deploymentID, err := structpath.ParsePath("deployment.deployment_id")
+	require.NoError(t, err)
+
+	for _, resourceType := range []string{"jobs", "pipelines"} {
+		cfg := dresources.GetResourceConfig(resourceType)
+		assert.True(t, shouldDropFromPlan(cfg, versionID), "%s: deployment.version_id should be dropped from plan", resourceType)
+		assert.False(t, shouldDropFromPlan(cfg, deploymentID), "%s: deployment.deployment_id should not be dropped from plan", resourceType)
+	}
+}

bundle/direct/bundle_plan_test.go

@@ -62,6 +62,13 @@ type ResourceLifecycleConfig struct {
 	// BackendDefaults: fields where the backend may set defaults.
 	// When old and new are nil but remote is set, and the remote value matches allowed values (if specified), the change is skipped.
 	BackendDefaults []BackendDefaultRule `yaml:"backend_defaults,omitempty"`
+
+	// HideFromPlan: field patterns removed from the plan diff entirely (never shown as a change).
+	// This only controls display: it does not by itself decide whether a change is applied.
+	// Use for fields the CLI sets on every deploy where the churn is pure noise (e.g.
+	// deployment.version_id). Pair it with ignore_local_changes / ignore_remote_changes, which
+	// are what actually keep the field from driving an update.
+	HideFromPlan []FieldRule `yaml:"hide_from_plan,omitempty"`
 }
 
 // Config is the root configuration structure for resource lifecycle behavior.
@@ -81,6 +88,7 @@ var empty = ResourceLifecycleConfig{
 	RecreateOnChanges:   nil,
 	UpdateIDOnChanges:   nil,
 	BackendDefaults:     nil,
+	HideFromPlan:        nil,
 }
 
 func mustParseConfig(data []byte) func() *Config {

bundle/direct/dresources/config.go

@@ -35,6 +35,7 @@ func categoryRules(c ResourceLifecycleConfig) []struct {
 		{"recreate_on_changes", c.RecreateOnChanges},
 		{"update_id_on_changes", c.UpdateIDOnChanges},
 		{"backend_defaults", backendAsFieldRules},
+		{"hide_from_plan", c.HideFromPlan},
 	}
 }