Add acceptance test pinning plan suppression of policy-injected cluster fields

Author: ilyakuz-db

acceptance/bundle/resources/jobs/policy_injected_cluster_fields/databricks.yml.tmpl

@@ -0,0 +1,29 @@
+bundle:
+  name: test-bundle-$UNIQUE_NAME
+
+targets:
+  default:
+    mode: development
+
+resources:
+  jobs:
+    job1:
+      max_concurrent_runs: 1
+      job_clusters:
+        - job_cluster_key: shared
+          new_cluster:
+            spark_version: $DEFAULT_SPARK_VERSION
+            node_type_id: $NODE_TYPE_ID
+            num_workers: 1
+      tasks:
+        - task_key: shared_cluster_task
+          job_cluster_key: shared
+          notebook_task:
+            notebook_path: /Users/{{workspace_user_name}}/notebook
+        - task_key: own_cluster_task
+          notebook_task:
+            notebook_path: /Users/{{workspace_user_name}}/notebook
+          new_cluster:
+            spark_version: $DEFAULT_SPARK_VERSION
+            node_type_id: $NODE_TYPE_ID
+            num_workers: 1

acceptance/bundle/resources/jobs/policy_injected_cluster_fields/out.test.toml

@@ -0,0 +1,26 @@
+Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle-[UNIQUE_NAME]/default/files...
+Deploying resources...
+Updating deployment state...
+Deployment complete!
+
+=== Simulate a cluster policy injecting custom_tags and cluster_log_conf
+These fields exist only remotely (the user never set them in config).
+
+=== Plan after injection: policy-injected fields alone must not produce an update
+>>> [CLI] bundle plan
+Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged
+
+=== A real remote edit must still be detected
+>>> [CLI] bundle plan
+update jobs.job1
+
+Plan: 0 to add, 1 to change, 0 to delete, 0 unchanged
+
+>>> [CLI] bundle destroy --auto-approve
+The following resources will be deleted:
+  delete resources.jobs.job1
+
+All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/test-bundle-[UNIQUE_NAME]/default
+
+Deleting files...
+Destroy complete!

acceptance/bundle/resources/jobs/policy_injected_cluster_fields/output.txt

@@ -0,0 +1,40 @@
+#!/bin/bash
+
+envsubst < databricks.yml.tmpl > databricks.yml
+
+cleanup() {
+  trace $CLI bundle destroy --auto-approve
+}
+trap cleanup EXIT
+
+$CLI bundle deploy
+job_id="$(read_id.py job1)"
+
+title "Simulate a cluster policy injecting custom_tags and cluster_log_conf"
+echo
+echo "These fields exist only remotely (the user never set them in config)."
+edit_resource.py jobs $job_id <<'PYEOF'
+# A DBFS destination is used instead of S3: the real backend rejects an S3
+# cluster log destination without a per-cluster instance-profile ARN.
+policy_tags = {"CostCenter": "dev-1234", "Team": "finops"}
+policy_log_conf = {"dbfs": {"destination": "dbfs:/cluster-logs/dev"}}
+
+for task in r.get("tasks", []):
+    if "new_cluster" in task:
+        task["new_cluster"]["custom_tags"] = dict(policy_tags)
+        task["new_cluster"]["cluster_log_conf"] = dict(policy_log_conf)
+
+for jc in r.get("job_clusters", []):
+    jc["new_cluster"]["custom_tags"] = dict(policy_tags)
+    jc["new_cluster"]["cluster_log_conf"] = dict(policy_log_conf)
+PYEOF
+
+title "Plan after injection: policy-injected fields alone must not produce an update"
+trace $CLI bundle plan
+
+title "A real remote edit must still be detected"
+edit_resource.py jobs $job_id <<'PYEOF'
+r["max_concurrent_runs"] = 5
+PYEOF
+
+trace $CLI bundle plan

acceptance/bundle/resources/jobs/policy_injected_cluster_fields/script

@@ -0,0 +1,9 @@
+Cloud = true
+
+RecordRequests = false
+Ignore = [".databricks", "databricks.yml"]
+
+# The suppression under test is the direct engine's backend_defaults
+# classification of policy-injected fields; the terraform engine delegates
+# these diffs to the terraform provider.
+EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"]