Fix task renaming in config-remote-sync when tasks are defined in multiple files (#4459)

## Changes

Respect index-shifting occurs when tasks (or other sequences that may
have merging behavior) are merged from multiple files. The previous
logic incorrectly assumed that they could only be defined in one file.

Also, new logic includes generic sorting issues when tasks are sorted by
mutators, and it's not in sync with the way the defined in the YAML

I also updated tests to simulate such issues

## Why
Sync was failing when tasks were defined in multiple files, and sorting
after mutators was different from what we have in the config

## Tests
Added more tests for tasks, also updated multi-file  test case

<!-- If your PR needs to be included in the release notes for next
release,
add a separate entry in NEXT_CHANGELOG.md as part of your PR. -->

Author: ilyakuz-db

acceptance/bundle/config-remote-sync/job_multiple_tasks/databricks.yml.tmpl

@@ -5,56 +5,72 @@ resources:
   jobs:
     my_job:
       tasks:
-        - task_key: task1
+        - task_key: b_task
           notebook_task:
-            notebook_path: /Users/{{workspace_user_name}}/task1
+            notebook_path: /Users/{{workspace_user_name}}/b_task
           new_cluster:
             spark_version: $DEFAULT_SPARK_VERSION
             node_type_id: $NODE_TYPE_ID
             num_workers: 1
-        - task_key: task2
+        - task_key: d_task
           notebook_task:
-            notebook_path: /Users/{{workspace_user_name}}/task2
+            notebook_path: /Users/{{workspace_user_name}}/d_task
           new_cluster:
             spark_version: $DEFAULT_SPARK_VERSION
             node_type_id: $NODE_TYPE_ID
             num_workers: 2
           depends_on:
-            - task_key: task1
-        - task_key: task3
+            - task_key: b_task
+        - task_key: c_task
           notebook_task:
-            notebook_path: /Users/{{workspace_user_name}}/task3
+            notebook_path: /Users/{{workspace_user_name}}/c_task
           new_cluster:
             spark_version: $DEFAULT_SPARK_VERSION
             node_type_id: $NODE_TYPE_ID
             num_workers: 2
           depends_on:
-            - task_key: task2
-        - task_key: task4
+            - task_key: d_task
+        - task_key: a_task
           notebook_task:
-            notebook_path: /Users/{{workspace_user_name}}/task4
+            notebook_path: /Users/{{workspace_user_name}}/a_task
           new_cluster:
             spark_version: $DEFAULT_SPARK_VERSION
             node_type_id: $NODE_TYPE_ID
             num_workers: 1
           depends_on:
-            - task_key: task3
+            - task_key: c_task
 
     rename_task_job:
       tasks:
-        - task_key: task_rename_1
+        - task_key: b_task
           notebook_task:
-            notebook_path: /Users/{{workspace_user_name}}/rename_task_1
+            notebook_path: /Users/{{workspace_user_name}}/b_task
           new_cluster:
             spark_version: $DEFAULT_SPARK_VERSION
             node_type_id: $NODE_TYPE_ID
             num_workers: 1
-        - task_key: task_rename_2
+        - task_key: d_task
+          depends_on:
+            - task_key: b_task
           notebook_task:
-            notebook_path: /Users/{{workspace_user_name}}/rename_task_2
+            notebook_path: /Users/{{workspace_user_name}}/d_task
           new_cluster:
             spark_version: $DEFAULT_SPARK_VERSION
             node_type_id: $NODE_TYPE_ID
             num_workers: 1
+        - task_key: c_task
           depends_on:
-            - task_key: task_rename_1
+            - task_key: b_task
+          notebook_task:
+            notebook_path: /Users/{{workspace_user_name}}/c_task
+          new_cluster:
+            spark_version: $DEFAULT_SPARK_VERSION
+            node_type_id: $NODE_TYPE_ID
+            num_workers: 1
+        - task_key: a_task
+          notebook_task:
+            notebook_path: /Users/{{workspace_user_name}}/a_task
+          new_cluster:
+            spark_version: $DEFAULT_SPARK_VERSION
+            node_type_id: $NODE_TYPE_ID
+            num_workers: 1

acceptance/bundle/config-remote-sync/job_multiple_tasks/output.txt

@@ -3,20 +3,20 @@ Deploying resources...
 Updating deployment state...
 Deployment complete!
 
-=== Modify only 'process' task num_workers and add timeout
+=== Modify c_task, remove d_task, add e_task
 === Detect and save changes
 Detected changes in 1 resource(s):
 
 Resource: resources.jobs.my_job
-  tasks[task_key='new_task']: add
-  tasks[task_key='task2']: remove
-  tasks[task_key='task3'].depends_on[0].task_key: replace
-  tasks[task_key='task3'].new_cluster.num_workers: replace
-  tasks[task_key='task3'].timeout_seconds: add
+  tasks[task_key='c_task'].depends_on[0].task_key: replace
+  tasks[task_key='c_task'].new_cluster.num_workers: replace
+  tasks[task_key='c_task'].timeout_seconds: add
+  tasks[task_key='d_task']: remove
+  tasks[task_key='e_task']: add
 
 
 
-=== Configuration changes for new task
+=== Configuration changes
 
 >>> diff.py databricks.yml.backup databricks.yml
 --- databricks.yml.backup
@@ -30,37 +30,37 @@ Resource: resources.jobs.my_job
 @@ -13,13 +12,11 @@
              node_type_id: [NODE_TYPE_ID]
              num_workers: 1
--        - task_key: task2
+-        - task_key: d_task
 +        - new_cluster:
 +            node_type_id: [NODE_TYPE_ID]
 +            num_workers: 1
 +            spark_version: 13.3.x-snapshot-scala2.12
            notebook_task:
--            notebook_path: /Users/{{workspace_user_name}}/task2
+-            notebook_path: /Users/{{workspace_user_name}}/d_task
 -          new_cluster:
 -            spark_version: 13.3.x-snapshot-scala2.12
 -            node_type_id: [NODE_TYPE_ID]
 -            num_workers: 2
 -          depends_on:
--            - task_key: task1
-+            notebook_path: /Users/[USERNAME]/new_task
-+          task_key: new_task
-         - task_key: task3
+-            - task_key: b_task
++            notebook_path: /Users/[USERNAME]/e_task
++          task_key: e_task
+         - task_key: c_task
            notebook_task:
 @@ -28,7 +25,8 @@
              spark_version: 13.3.x-snapshot-scala2.12
              node_type_id: [NODE_TYPE_ID]
 -            num_workers: 2
 +            num_workers: 5
            depends_on:
--            - task_key: task2
-+            - task_key: task1
+-            - task_key: d_task
++            - task_key: b_task
 +          timeout_seconds: 3600
-         - task_key: task4
+         - task_key: a_task
            notebook_task:
 @@ -40,5 +38,4 @@
            depends_on:
-             - task_key: task3
+             - task_key: c_task
 -
      rename_task_job:
        tasks:
@@ -69,14 +69,15 @@ Deploying resources...
 Updating deployment state...
 Deployment complete!
 
-=== Rename task_rename_1 to task_rename_new
+=== Rename b_task to b_task_renamed (4 tasks, 2 with depends_on, 1 without)
 === Detect task key rename
 Detected changes in 1 resource(s):
 
 Resource: resources.jobs.rename_task_job
-  tasks[task_key='task_rename_1']: remove
-  tasks[task_key='task_rename_2'].depends_on[0].task_key: replace
-  tasks[task_key='task_rename_new']: add
+  tasks[task_key='b_task']: remove
+  tasks[task_key='b_task_renamed']: add
+  tasks[task_key='c_task'].depends_on[0].task_key: replace
+  tasks[task_key='d_task'].depends_on[0].task_key: replace
 
 
 
@@ -85,25 +86,31 @@ Resource: resources.jobs.rename_task_job
 >>> diff.py databricks.yml.backup2 databricks.yml
 --- databricks.yml.backup2
 +++ databricks.yml
-@@ -40,11 +40,11 @@
+@@ -40,14 +40,14 @@
      rename_task_job:
        tasks:
--        - task_key: task_rename_1
+-        - task_key: b_task
 +        - new_cluster:
 +            node_type_id: [NODE_TYPE_ID]
 +            num_workers: 1
 +            spark_version: 13.3.x-snapshot-scala2.12
            notebook_task:
-             notebook_path: /Users/{{workspace_user_name}}/rename_task_1
+             notebook_path: /Users/{{workspace_user_name}}/b_task
 -          new_cluster:
 -            spark_version: 13.3.x-snapshot-scala2.12
 -            node_type_id: [NODE_TYPE_ID]
 -            num_workers: 1
-+          task_key: task_rename_new
-         - task_key: task_rename_2
++          task_key: b_task_renamed
+         - task_key: d_task
+           depends_on:
+-            - task_key: b_task
++            - task_key: b_task_renamed
            notebook_task:
-@@ -55,3 +55,3 @@
-             num_workers: 1
+             notebook_path: /Users/{{workspace_user_name}}/d_task
+@@ -58,5 +58,5 @@
+         - task_key: c_task
            depends_on:
--            - task_key: task_rename_1
-+            - task_key: task_rename_new
+-            - task_key: b_task
++            - task_key: b_task_renamed
+           notebook_task:
+             notebook_path: /Users/{{workspace_user_name}}/c_task

acceptance/bundle/config-remote-sync/job_multiple_tasks/script

@@ -7,20 +7,20 @@ $CLI bundle deploy
 job_id="$(read_id.py my_job)"
 
 
-title "Modify only 'process' task num_workers and add timeout"
+title "Modify c_task, remove d_task, add e_task"
 edit_resource.py jobs $job_id <<EOF
 for task in r["tasks"]:
-    if task["task_key"] == "task3":
+    if task["task_key"] == "c_task":
         task["new_cluster"]["num_workers"] = 5
         task["timeout_seconds"] = 3600
-        task["depends_on"] = [{"task_key": "task1"}]
+        task["depends_on"] = [{"task_key": "b_task"}]
 
-r["tasks"] = [task for task in r["tasks"] if task["task_key"] != "task2"]
+r["tasks"] = [task for task in r["tasks"] if task["task_key"] != "d_task"]
 
 r["tasks"].append({
-    "task_key": "new_task",
+    "task_key": "e_task",
     "notebook_task": {
-        "notebook_path": "/Users/${CURRENT_USER_NAME}/new_task"
+        "notebook_path": "/Users/${CURRENT_USER_NAME}/e_task"
     },
     "new_cluster": {
         "spark_version": "${DEFAULT_SPARK_VERSION}",
@@ -35,7 +35,7 @@ echo
 cp databricks.yml databricks.yml.backup
 $CLI bundle config-remote-sync --save
 
-title "Configuration changes for new task"
+title "Configuration changes"
 echo
 trace diff.py databricks.yml.backup databricks.yml
 rm databricks.yml.backup
@@ -47,27 +47,24 @@ mv databricks.yml.resolved databricks.yml
 # Deploy the updated configuration to sync state
 $CLI bundle deploy
 
-title "Rename task_rename_1 to task_rename_new"
+title "Rename b_task to b_task_renamed (4 tasks, 2 with depends_on, 1 without)"
 rename_job_id="$(read_id.py rename_task_job)"
 edit_resource.py jobs $rename_job_id <<'EOF'
 for task in r["tasks"]:
-    if task["task_key"] == "task_rename_1":
-        task["task_key"] = "task_rename_new"
-    # Update dependencies that reference the old key
+    if task["task_key"] == "b_task":
+        task["task_key"] = "b_task_renamed"
     if "depends_on" in task:
         for dep in task["depends_on"]:
-            if dep["task_key"] == "task_rename_1":
-                dep["task_key"] = "task_rename_new"
+            if dep["task_key"] == "b_task":
+                dep["task_key"] = "b_task_renamed"
 EOF
 
 title "Detect task key rename"
 echo
 cp databricks.yml databricks.yml.backup2
-$CLI bundle config-remote-sync --save || true
+$CLI bundle config-remote-sync --save
 
 title "Configuration changes for task key rename"
 echo
-if [ -f databricks.yml.backup2 ]; then
-    trace diff.py databricks.yml.backup2 databricks.yml || true
-    rm databricks.yml.backup2
-fi
+trace diff.py databricks.yml.backup2 databricks.yml
+rm databricks.yml.backup2

acceptance/bundle/config-remote-sync/multiple_files/databricks.yml.tmpl

@@ -3,3 +3,18 @@ bundle:
 
 include:
   - "resources/*.yml"
+
+targets:
+  dev:
+    resources:
+      jobs:
+        job_one:
+          max_concurrent_runs: 1
+          tasks:
+            - task_key: b_task
+              notebook_task:
+                notebook_path: /Users/{{workspace_user_name}}/b_task
+              new_cluster:
+                spark_version: $DEFAULT_SPARK_VERSION
+                node_type_id: $NODE_TYPE_ID
+                num_workers: 1