bundle: reject and hide deployment_id/version_id on jobs and pipelines

deployment_id and version_id on a job/pipeline deployment block are set by
the CLI to track the bundle deployment in the Deployment Metadata Service.
They are not user-configurable, so this change:

- Errors at validation time if either is set in bundle configuration
  (new validate.ValidateDeploymentFields mutator).
- Adds a hide_from_plan lifecycle annotation to the direct engine that
  removes a field from the plan diff (display only), and applies it to
  deployment.version_id for jobs and pipelines (version_id is set on every
  deploy, so showing it is pure noise). version_id also gets
  ignore_local_changes/ignore_remote_changes, which are what keep its churn
  from driving an update or showing as drift. deployment_id is intentionally
  left to show in the plan.
- Removes deployment_id/version_id from the generated JSON schema.

Co-authored-by: Isaac

Author: shreyas-goenka

acceptance/bundle/validate/reserved_deployment_fields/databricks.yml

@@ -0,0 +1,18 @@
+bundle:
+  name: test-bundle
+
+resources:
+  jobs:
+    my_job:
+      name: my_job
+      deployment:
+        kind: BUNDLE
+        deployment_id: "dep-123"
+        version_id: "ver-456"
+  pipelines:
+    my_pipeline:
+      name: my_pipeline
+      deployment:
+        kind: BUNDLE
+        deployment_id: "dep-789"
+        version_id: "ver-012"

acceptance/bundle/validate/reserved_deployment_fields/out.test.toml

@@ -0,0 +1,27 @@
+
+>>> [CLI] bundle validate
+Error: deployment_id must not be set in bundle configuration; it is managed by Databricks Asset Bundles
+  at resources.jobs.my_job.deployment.deployment_id
+  in databricks.yml:10:24
+
+Error: version_id must not be set in bundle configuration; it is managed by Databricks Asset Bundles
+  at resources.jobs.my_job.deployment.version_id
+  in databricks.yml:11:21
+
+Error: deployment_id must not be set in bundle configuration; it is managed by Databricks Asset Bundles
+  at resources.pipelines.my_pipeline.deployment.deployment_id
+  in databricks.yml:17:24
+
+Error: version_id must not be set in bundle configuration; it is managed by Databricks Asset Bundles
+  at resources.pipelines.my_pipeline.deployment.version_id
+  in databricks.yml:18:21
+
+Name: test-bundle
+Target: default
+Workspace:
+  User: [USERNAME]
+  Path: /Workspace/Users/[USERNAME]/.bundle/test-bundle/default
+
+Found 4 errors
+
+Exit code: 1

acceptance/bundle/validate/reserved_deployment_fields/output.txt

@@ -0,0 +1 @@
+trace $CLI bundle validate

acceptance/bundle/validate/reserved_deployment_fields/script

@@ -0,0 +1,72 @@
+package validate
+
+import (
+	"cmp"
+	"context"
+	"slices"
+
+	"github.com/databricks/cli/bundle"
+	"github.com/databricks/cli/libs/diag"
+	"github.com/databricks/cli/libs/dyn"
+)
+
+func ValidateDeploymentFields() bundle.ReadOnlyMutator {
+	return &validateDeploymentFields{}
+}
+
+type validateDeploymentFields struct{ bundle.RO }
+
+func (v *validateDeploymentFields) Name() string {
+	return "validate:validate_deployment_fields"
+}
+
+func (v *validateDeploymentFields) Apply(_ context.Context, b *bundle.Bundle) diag.Diagnostics {
+	var diags diag.Diagnostics
+
+	for name, job := range b.Config.Resources.Jobs {
+		if job.Deployment == nil {
+			continue
+		}
+		diags = diags.Extend(reservedDeploymentDiags(b, "resources.jobs."+name, job.Deployment.DeploymentId, job.Deployment.VersionId))
+	}
+
+	for name, pipeline := range b.Config.Resources.Pipelines {
+		if pipeline.Deployment == nil {
+			continue
+		}
+		diags = diags.Extend(reservedDeploymentDiags(b, "resources.pipelines."+name, pipeline.Deployment.DeploymentId, pipeline.Deployment.VersionId))
+	}
+
+	// Resources are stored in maps with randomized iteration order; sort by path
+	// so the reported diagnostics are stable across runs.
+	slices.SortFunc(diags, func(x, y diag.Diagnostic) int {
+		return cmp.Compare(x.Paths[0].String(), y.Paths[0].String())
+	})
+
+	return diags
+}
+
+// reservedDeploymentDiags returns an error for each reserved deployment field
+// (deployment_id, version_id) set on the resource at resourcePath. These fields
+// are set by the CLI to identify the bundle deployment and its version in the
+// Deployment Metadata Service; a value set by hand would be overwritten at
+// deploy time, so we reject it up front.
+func reservedDeploymentDiags(b *bundle.Bundle, resourcePath, deploymentID, versionID string) diag.Diagnostics {
+	var diags diag.Diagnostics
+	for _, f := range []struct{ name, value string }{
+		{"deployment_id", deploymentID},
+		{"version_id", versionID},
+	} {
+		if f.value == "" {
+			continue
+		}
+		path := resourcePath + ".deployment." + f.name
+		diags = append(diags, diag.Diagnostic{
+			Severity:  diag.Error,
+			Summary:   f.name + " must not be set in bundle configuration; it is managed by Databricks Asset Bundles",
+			Paths:     []dyn.Path{dyn.MustPathFromString(path)},
+			Locations: b.Config.GetLocations(path),
+		})
+	}
+	return diags
+}

bundle/config/validate/validate_deployment_fields.go

@@ -0,0 +1,80 @@
+package validate
+
+import (
+	"testing"
+
+	"github.com/databricks/cli/bundle"
+	"github.com/databricks/cli/bundle/config"
+	"github.com/databricks/cli/bundle/config/resources"
+	"github.com/databricks/cli/libs/diag"
+	"github.com/databricks/databricks-sdk-go/service/jobs"
+	"github.com/databricks/databricks-sdk-go/service/pipelines"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+const reservedFieldMsg = " must not be set in bundle configuration; it is managed by Databricks Asset Bundles"
+
+func jobBundle(d *jobs.JobDeployment) *bundle.Bundle {
+	return &bundle.Bundle{
+		Config: config.Root{
+			Resources: config.Resources{
+				Jobs: map[string]*resources.Job{
+					"my_job": {JobSettings: jobs.JobSettings{Name: "my_job", Deployment: d}},
+				},
+			},
+		},
+	}
+}
+
+func pipelineBundle(d *pipelines.PipelineDeployment) *bundle.Bundle {
+	return &bundle.Bundle{
+		Config: config.Root{
+			Resources: config.Resources{
+				Pipelines: map[string]*resources.Pipeline{
+					"my_pipeline": {CreatePipeline: pipelines.CreatePipeline{Name: "my_pipeline", Deployment: d}},
+				},
+			},
+		},
+	}
+}
+
+func TestValidateDeploymentFieldsRejectsReservedFields(t *testing.T) {
+	tests := []struct {
+		name string
+		b    *bundle.Bundle
+		want string
+	}{
+		{"job deployment_id", jobBundle(&jobs.JobDeployment{DeploymentId: "x"}), "deployment_id" + reservedFieldMsg},
+		{"job version_id", jobBundle(&jobs.JobDeployment{VersionId: "x"}), "version_id" + reservedFieldMsg},
+		{"pipeline deployment_id", pipelineBundle(&pipelines.PipelineDeployment{DeploymentId: "x"}), "deployment_id" + reservedFieldMsg},
+		{"pipeline version_id", pipelineBundle(&pipelines.PipelineDeployment{VersionId: "x"}), "version_id" + reservedFieldMsg},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			diags := ValidateDeploymentFields().Apply(t.Context(), tt.b)
+			require.Len(t, diags, 1)
+			assert.Equal(t, diag.Error, diags[0].Severity)
+			assert.Equal(t, tt.want, diags[0].Summary)
+		})
+	}
+}
+
+func TestValidateDeploymentFieldsReportsAllOffenders(t *testing.T) {
+	b := &bundle.Bundle{
+		Config: config.Root{
+			Resources: config.Resources{
+				Jobs: map[string]*resources.Job{
+					"my_job": {JobSettings: jobs.JobSettings{Deployment: &jobs.JobDeployment{DeploymentId: "a"}}},
+				},
+				Pipelines: map[string]*resources.Pipeline{
+					"my_pipeline": {CreatePipeline: pipelines.CreatePipeline{Deployment: &pipelines.PipelineDeployment{VersionId: "b"}}},
+				},
+			},
+		},
+	}
+
+	diags := ValidateDeploymentFields().Apply(t.Context(), b)
+	require.Len(t, diags, 2)
+}

bundle/config/validate/validate_deployment_fields_test.go

@@ -357,7 +357,12 @@ func addPerFieldActions(ctx context.Context, adapter *dresources.Adapter, change
 			return err
 		}
 
-		if structdiff.IsEqual(ch.Remote, ch.New) {
+		if shouldDropFromPlan(cfg, path) || shouldDropFromPlan(generatedCfg, path) {
+			// Drop the field from the plan entirely. ReasonDrop removes the entry
+			// below so it never surfaces as a change regardless of its diff state.
+			ch.Action = deployplan.Skip
+			ch.Reason = deployplan.ReasonDrop
+		} else if structdiff.IsEqual(ch.Remote, ch.New) {
 			ch.Action = deployplan.Skip
 			ch.Reason = deployplan.ReasonRemoteAlreadySet
 		} else if allEmpty(ch.Old, ch.New, ch.Remote) {
@@ -434,6 +439,16 @@ func shouldSkip(cfg *dresources.ResourceLifecycleConfig, path *structpath.PathNo
 	return "", false
 }
 
+// shouldDropFromPlan reports whether the field matches a HideFromPlan rule, meaning
+// it should be removed from the plan entirely rather than shown as a change.
+func shouldDropFromPlan(cfg *dresources.ResourceLifecycleConfig, path *structpath.PathNode) bool {
+	if cfg == nil {
+		return false
+	}
+	_, ok := findMatchingRule(path, cfg.HideFromPlan)
+	return ok
+}
+
 func shouldUpdateOrRecreate(cfg *dresources.ResourceLifecycleConfig, path *structpath.PathNode) (deployplan.ActionType, string) {
 	if cfg == nil {
 		return deployplan.Undefined, ""

bundle/direct/bundle_plan.go

@@ -3,8 +3,11 @@ package direct
 import (
 	"testing"
 
+	"github.com/databricks/cli/bundle/direct/dresources"
 	"github.com/databricks/cli/libs/dyn"
+	"github.com/databricks/cli/libs/structs/structpath"
 	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
 )
 
 func TestDynPathToStructPath(t *testing.T) {
@@ -35,3 +38,19 @@ func TestDynPathToStructPath(t *testing.T) {
 		assert.Equal(t, tc.expected, node.String())
 	}
 }
+
+// TestShouldDropFromPlanDeploymentVersion verifies that the DMS deployment
+// version_id is dropped from the plan for jobs and pipelines, while
+// deployment_id is left in (it is stable and fine to show).
+func TestShouldDropFromPlanDeploymentVersion(t *testing.T) {
+	versionID, err := structpath.ParsePath("deployment.version_id")
+	require.NoError(t, err)
+	deploymentID, err := structpath.ParsePath("deployment.deployment_id")
+	require.NoError(t, err)
+
+	for _, resourceType := range []string{"jobs", "pipelines"} {
+		cfg := dresources.GetResourceConfig(resourceType)
+		assert.True(t, shouldDropFromPlan(cfg, versionID), "%s: deployment.version_id should be dropped from plan", resourceType)
+		assert.False(t, shouldDropFromPlan(cfg, deploymentID), "%s: deployment.deployment_id should not be dropped from plan", resourceType)
+	}
+}

bundle/direct/bundle_plan_test.go

@@ -62,6 +62,13 @@ type ResourceLifecycleConfig struct {
 	// BackendDefaults: fields where the backend may set defaults.
 	// When old and new are nil but remote is set, and the remote value matches allowed values (if specified), the change is skipped.
 	BackendDefaults []BackendDefaultRule `yaml:"backend_defaults,omitempty"`
+
+	// HideFromPlan: field patterns removed from the plan diff entirely (never shown as a change).
+	// This only controls display: it does not by itself decide whether a change is applied.
+	// Use for fields the CLI sets on every deploy where the churn is pure noise (e.g.
+	// deployment.version_id). Pair it with ignore_local_changes / ignore_remote_changes, which
+	// are what actually keep the field from driving an update.
+	HideFromPlan []FieldRule `yaml:"hide_from_plan,omitempty"`
 }
 
 // Config is the root configuration structure for resource lifecycle behavior.
@@ -81,6 +88,7 @@ var empty = ResourceLifecycleConfig{
 	RecreateOnChanges:   nil,
 	UpdateIDOnChanges:   nil,
 	BackendDefaults:     nil,
+	HideFromPlan:        nil,
 }
 
 func mustParseConfig(data []byte) func() *Config {

bundle/direct/dresources/config.go

@@ -35,6 +35,7 @@ func categoryRules(c ResourceLifecycleConfig) []struct {
 		{"recreate_on_changes", c.RecreateOnChanges},
 		{"update_id_on_changes", c.UpdateIDOnChanges},
 		{"backend_defaults", backendAsFieldRules},
+		{"hide_from_plan", c.HideFromPlan},
 	}
 }