infra updates

Author: jeffreyaven

retail-job/databricks.yml

@@ -95,13 +95,28 @@ resources:
             - task_key: "remove_duplicates_task"
               outcome: "true"
 
+        # - task_key: "process_state_data_task"
+        #   description: "Process orders data by state"
+        #   job_cluster_key: "main_cluster"
+        #   for_each_task:
+        #     inputs: '["CA", "NY", "TX", "FL"]'
+        #     task:
+        #       task_key: "process_single_state"
+        #       notebook_task:
+        #         notebook_path: "./Task Files/05_state_processing/process_orders_by_state.py"
+        #         source: WORKSPACE
+        #         base_parameters:
+        #           state: "{{item}}"
+        #   depends_on:
+        #     - task_key: "join_customers_orders_task"
+
         - task_key: "process_state_data_task"
           description: "Process orders data by state"
-          job_cluster_key: "main_cluster"
           for_each_task:
             inputs: '["CA", "NY", "TX", "FL"]'
             task:
               task_key: "process_single_state"
+              job_cluster_key: "main_cluster"
               notebook_task:
                 notebook_path: "./Task Files/05_state_processing/process_orders_by_state.py"
                 source: WORKSPACE