Merge branch 'main' of github.com:databricks/cli into config-sync-migrate-to-resource-yml

Author: ilyakuz-db

.codegen/_openapi_sha

@@ -1 +1 @@
-2f10a6ab001e2fa2552f6b25f7fbb4a08014a43f
+a7c320a6b531263c8fa45619c1565b63849750e5

.gitattributes

@@ -77,6 +77,7 @@ cmd/workspace/current-user/current-user.go linguist-generated=true
 cmd/workspace/dashboard-email-subscriptions/dashboard-email-subscriptions.go linguist-generated=true
 cmd/workspace/dashboard-widgets/dashboard-widgets.go linguist-generated=true
 cmd/workspace/dashboards/dashboards.go linguist-generated=true
+cmd/workspace/data-classification/data-classification.go linguist-generated=true
 cmd/workspace/data-quality/data-quality.go linguist-generated=true
 cmd/workspace/data-sources/data-sources.go linguist-generated=true
 cmd/workspace/database/database.go linguist-generated=true
@@ -107,6 +108,7 @@ cmd/workspace/instance-pools/instance-pools.go linguist-generated=true
 cmd/workspace/instance-profiles/instance-profiles.go linguist-generated=true
 cmd/workspace/ip-access-lists/ip-access-lists.go linguist-generated=true
 cmd/workspace/jobs/jobs.go linguist-generated=true
+cmd/workspace/knowledge-assistants/knowledge-assistants.go linguist-generated=true
 cmd/workspace/lakeview-embedded/lakeview-embedded.go linguist-generated=true
 cmd/workspace/lakeview/lakeview.go linguist-generated=true
 cmd/workspace/libraries/libraries.go linguist-generated=true

.github/workflows/tagging.yml

@@ -6,10 +6,10 @@ on:
   workflow_dispatch:
   # No inputs are required for the manual dispatch.
 
-  # Runs at 8:00 UTC on Tuesday, Wednesday, and Thursday. To enable automated
+  # Runs at 8:00 UTC on Monday, Tuesday, Wednesday, and Thursday. To enable automated
   # tagging for a repository, simply add it to the if block of the tag job.
   schedule:
-    - cron: '0 8 * * TUE,WED,THU'
+    - cron: '0 8 * * MON,TUE,WED,THU'
 
 # Ensure that only a single instance of the workflow is running at a time.
 concurrency:

.gitignore

@@ -57,3 +57,5 @@ go.work.sum
 
 # Fetched based on recorded hash
 .codegen/openapi.json
+
+.claude/settings.local.json

Makefile

@@ -182,6 +182,7 @@ generate:
 	@echo "Generating CLI code..."
 	$(GENKIT_BINARY) update-sdk
 	cat .gitattributes.manual .gitattributes > .gitattributes.tmp && mv .gitattributes.tmp .gitattributes
+	-go test ./acceptance -run TestAccept/bundle/refschema -update &> /dev/null
 	@echo "Updating direct engine config..."
 	make generate-direct
 	go test ./bundle/internal/schema

NEXT_CHANGELOG.md

@@ -8,5 +8,6 @@
 * Modify grants to use SDK types ([#4666](https://github.com/databricks/cli/pull/4666))
 
 ### Dependency updates
+* Bump databricks-sdk-go from v0.112.0 to v0.119.0 ([#4631](https://github.com/databricks/cli/pull/4631), [#4695](https://github.com/databricks/cli/pull/4695))
 
 ### API Changes

acceptance/bundle/config-remote-sync/job_fields/databricks.yml.tmpl

@@ -22,8 +22,18 @@ resources:
             environment_version: "3"
             dependencies:
               - ./*.whl
+      job_clusters:
+        - job_cluster_key: test_cluster
+          new_cluster:
+            spark_version: $DEFAULT_SPARK_VERSION
+            node_type_id: $NODE_TYPE_ID
+            num_workers: 1
       tasks:
         - task_key: main
+          notebook_task:
+            notebook_path: /Users/{{workspace_user_name}}/notebook
+          job_cluster_key: test_cluster
+        - task_key: inline_cluster_task
           notebook_task:
             notebook_path: /Users/{{workspace_user_name}}/notebook
           new_cluster:

acceptance/bundle/config-remote-sync/job_fields/output.txt

@@ -12,8 +12,11 @@ Detected changes in 1 resource(s):
 Resource: resources.jobs.my_job
   email_notifications.no_alert_for_skipped_runs: add
   email_notifications.on_failure: add
+  job_clusters[0].job_cluster_key: replace
   parameters: replace
   tags['team']: add
+  tasks[task_key='inline_cluster_task'].new_cluster.num_workers: replace
+  tasks[task_key='main'].job_cluster_key: replace
   trigger.pause_status: add
   trigger.periodic: remove
   trigger.table_update: add
@@ -52,9 +55,25 @@ Resource: resources.jobs.my_job
 +            - samples.nyctaxi.trips
        environments:
          - environment_key: default
-@@ -31,4 +37,6 @@
+@@ -24,5 +30,5 @@
+               - ./*.whl
+       job_clusters:
+-        - job_cluster_key: test_cluster
++        - job_cluster_key: test_cluster_renamed
+           new_cluster:
+             spark_version: [[DEFAULT_SPARK_VERSION]]
+@@ -33,5 +39,5 @@
+           notebook_task:
+             notebook_path: /Users/{{workspace_user_name}}/notebook
+-          job_cluster_key: test_cluster
++          job_cluster_key: test_cluster_renamed
+         - task_key: inline_cluster_task
+           notebook_task:
+@@ -40,5 +46,7 @@
+             spark_version: [[DEFAULT_SPARK_VERSION]]
              node_type_id: [NODE_TYPE_ID]
-             num_workers: 1
+-            num_workers: 1
++            num_workers: 2
 +      tags:
 +        team: data
 

acceptance/bundle/config-remote-sync/job_fields/script

@@ -1,6 +1,8 @@
 #!/bin/bash
 
 envsubst < databricks.yml.tmpl > databricks.yml
+add_repl.py "$DEFAULT_SPARK_VERSION" "[DEFAULT_SPARK_VERSION]"
+add_repl.py "$NODE_TYPE_ID" "[NODE_TYPE_ID]"
 
 cleanup() {
   trace $CLI bundle destroy --auto-approve
@@ -20,6 +22,16 @@ r["email_notifications"]["no_alert_for_skipped_runs"] = True
 r["parameters"].append({"name": "region", "default": "us-east-1"})
 r["trigger"] = {"pause_status": "UNPAUSED", "table_update": {"table_names": ["samples.nyctaxi.trips"]}}
 
+for cluster in r.get("job_clusters", []):
+    if cluster.get("job_cluster_key") == "test_cluster":
+        cluster["job_cluster_key"] = "test_cluster_renamed"
+for task in r.get("tasks", []):
+    if task.get("task_key") == "inline_cluster_task" and "new_cluster" in task:
+        task["new_cluster"]["num_workers"] = 2
+for task in r.get("tasks", []):
+    if task.get("job_cluster_key") == "test_cluster":
+        task["job_cluster_key"] = "test_cluster_renamed"
+
 if "tags" not in r:
     r["tags"] = {}
 r["tags"]["team"] = "data"

acceptance/bundle/config-remote-sync/validation_errors/databricks.yml.tmpl

@@ -0,0 +1,26 @@
+bundle:
+  name: test-bundle-$UNIQUE_NAME
+
+resources:
+  pipelines:
+    my_pipeline:
+      name: test-pipeline-$UNIQUE_NAME
+      root_path: ./pipeline_root
+      libraries:
+        - notebook:
+            path: /Users/{{workspace_user_name}}/notebook
+
+  jobs:
+    my_job:
+      tasks:
+        - task_key: main
+          notebook_task:
+            notebook_path: ./src/notebook.py
+          new_cluster:
+            spark_version: $DEFAULT_SPARK_VERSION
+            node_type_id: $NODE_TYPE_ID
+            num_workers: 1
+
+targets:
+  default:
+    mode: development