direct: save state before wait/publish in sql_warehouse DoCreate and dashboard DoUpdate

Two more cases where state could be persisted earlier to avoid orphaning or
stale-etag conflicts:

sql_warehouse DoCreate: the warehouse is created, then DoCreate polls for RUNNING
(and may Stop it). If interrupted during the wait, the warehouse was orphaned.
Now SaveState is called right after Create returns the id, before the wait.

dashboard DoUpdate: Update() bumps the server-side etag, then publishDashboard()
can fail. Mirror the DoCreate fix — save the new etag with Published=false before
publishing. This keeps the etag in sync (a stale etag makes the next Update fail
with a conflict) and records published=false so the planner re-publishes next
deploy. Resolves the pre-existing TODO.

Add publish-failure-retry-on-update acceptance test: deploy, then trigger an
update with an injected publish failure, and verify the update issued a PATCH
(no CREATE) and the plan records published old=false.

Co-authored-by: Denis Bilenko

Author: denik

acceptance/bundle/resources/dashboards/publish-failure-retry-on-update/dashboard.lvdash.json

@@ -0,0 +1 @@
+{"pages":[{"name":"test-page","displayName":"Test Dashboard"}]}

acceptance/bundle/resources/dashboards/publish-failure-retry-on-update/databricks.yml.tmpl

@@ -0,0 +1,9 @@
+bundle:
+  name: publish-failure-retry-on-update
+
+resources:
+  dashboards:
+    dashboard1:
+      display_name: my dashboard
+      warehouse_id: someid
+      file_path: ./dashboard.lvdash.json

acceptance/bundle/resources/dashboards/publish-failure-retry-on-update/out.test.toml

@@ -0,0 +1,56 @@
+
+>>> [CLI] bundle deploy
+Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/publish-failure-retry-on-update/default/files...
+Deploying resources...
+Updating deployment state...
+Deployment complete!
+
+>>> [CLI] bundle deploy
+Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/publish-failure-retry-on-update/default/files...
+Deploying resources...
+Error: cannot update resources.dashboards.dashboard1: updating id=[DASHBOARD1_ID]: Fault injected by test. (400 INJECTED)
+
+Endpoint: POST [DATABRICKS_URL]/api/2.0/lakeview/dashboards/[DASHBOARD1_ID]/published
+HTTP Status: 400 Bad Request
+API error_code: INJECTED
+API message: Fault injected by test.
+
+Updating deployment state...
+
+Exit code: 1
+{
+  "method": "PATCH",
+  "path": "/api/2.0/lakeview/dashboards/[DASHBOARD1_ID]",
+  "body": {
+    "display_name": "my dashboard renamed",
+    "parent_path": "/Workspace/Users/[USERNAME]/.bundle/publish-failure-retry-on-update/default/resources",
+    "serialized_dashboard": "{\"pages\":[{\"name\":\"test-page\",\"displayName\":\"Test Dashboard\"}]}\n",
+    "warehouse_id": "someid"
+  }
+}
+{
+  "method": "POST",
+  "path": "/api/2.0/lakeview/dashboards/[DASHBOARD1_ID]/published",
+  "body": {
+    "embed_credentials": false,
+    "warehouse_id": "someid"
+  }
+}
+
+>>> [CLI] bundle plan -o json
+{
+  "action": "skip",
+  "reason": "remote_already_set",
+  "old": false,
+  "new": true,
+  "remote": true
+}
+
+>>> [CLI] bundle destroy --auto-approve
+The following resources will be deleted:
+  delete resources.dashboards.dashboard1
+
+All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/publish-failure-retry-on-update/default
+
+Deleting files...
+Destroy complete!

acceptance/bundle/resources/dashboards/publish-failure-retry-on-update/output.txt

@@ -0,0 +1,33 @@
+envsubst < databricks.yml.tmpl > databricks.yml
+
+cleanup() {
+    trace $CLI bundle destroy --auto-approve
+    rm -f out.requests.txt
+}
+trap cleanup EXIT
+
+# unset MSYS_NO_PATHCONV so MSYS2 converts the script path to a Windows path
+# when invoking the Python interpreter (required for fault.py to be found on Windows).
+unset MSYS_NO_PATHCONV
+
+# First deploy succeeds fully: the dashboard is created and published.
+trace $CLI bundle deploy
+replace_ids.py
+rm out.requests.txt
+
+# Inject a single publish failure for the update below.
+fault.py "POST /api/2.0/lakeview/dashboards/*" 400 0 1
+
+# Change the dashboard to trigger an Update, which bumps the server-side etag.
+update_file.py databricks.yml "my dashboard" "my dashboard renamed"
+
+# Deploy: Update (PATCH) succeeds and bumps the etag, but the publish (POST) fails.
+errcode trace $CLI bundle deploy
+
+# The failed deploy issued a PATCH (update) and a POST (publish), but no CREATE
+# (POST /api/2.0/lakeview/dashboards): the existing dashboard was updated in place.
+print_requests.py //lakeview/dashboards
+
+# Plan shows published old=false: state was saved with published=false (and the bumped
+# etag) before the failed publish, so the next deploy knows publishing is still pending.
+trace $CLI bundle plan -o json | jq '.plan["resources.dashboards.dashboard1"].changes.published'

acceptance/bundle/resources/dashboards/publish-failure-retry-on-update/script

@@ -0,0 +1,13 @@
+Cloud = false
+Local = true
+RecordRequests = true
+
+# Only run with the direct engine: the test verifies direct engine's SaveState
+# behavior during DoUpdate (new etag + published=false persisted before a failed publish).
+[EnvMatrix]
+DATABRICKS_BUNDLE_ENGINE = ["direct"]
+
+# Dashboard published URLs use ?o= (local testserver) or ?w= (cloud) for the workspace/org ID.
+[[Repls]]
+Old = '\?[ow]=\d+'
+New = "?[WSPARAM]=[NUMID]"

acceptance/bundle/resources/dashboards/publish-failure-retry-on-update/test.toml

@@ -319,7 +319,7 @@ func (r *ResourceDashboard) DoCreate(ctx context.Context, engine *Engine, config
 	return createResp.DashboardId, responseToState(createResp, publishResp, dashboard.SerializedDashboard, config.Published), nil
 }
 
-func (r *ResourceDashboard) DoUpdate(ctx context.Context, _ *Engine, id string, config *DashboardState, _ *PlanEntry) (*DashboardState, error) {
+func (r *ResourceDashboard) DoUpdate(ctx context.Context, engine *Engine, id string, config *DashboardState, _ *PlanEntry) (*DashboardState, error) {
 	dashboard, err := prepareDashboardRequest(config)
 	if err != nil {
 		return nil, err
@@ -339,15 +339,21 @@ func (r *ResourceDashboard) DoUpdate(ctx context.Context, _ *Engine, id string,
 		return nil, err
 	}
 
-	// Persist the etag in state.
+	// Persist the new etag with Published=false before publishing. Update() bumps the
+	// etag on the server; if a subsequent publish fails, saving here keeps the etag in
+	// sync (a stale etag would make the next Update fail with a conflict) and records
+	// published=false so the planner re-publishes on the next deploy.
 	config.Etag = updateResp.Etag
+	savedPublished := config.Published
+	config.Published = false
+	engine.SaveState(ctx, id, config)
+	config.Published = savedPublished
 
 	var publishResp *dashboards.PublishedDashboard
 	// Note, today config.Published is always true (we do not have this field in input config).
 	if config.Published {
 		publishResp, err = r.publishDashboard(ctx, id, config)
 		if err != nil {
-			// TODO: store partial state with published=false?
 			return nil, err
 		}
 	}

bundle/direct/dresources/dashboard.go

@@ -118,13 +118,17 @@ func (r *ResourceSqlWarehouse) DoRead(ctx context.Context, id string) (*SqlWareh
 }
 
 // DoCreate creates the warehouse and returns its id.
-func (r *ResourceSqlWarehouse) DoCreate(ctx context.Context, _ *Engine, config *SqlWarehouseState) (string, *SqlWarehouseRemote, error) {
+func (r *ResourceSqlWarehouse) DoCreate(ctx context.Context, engine *Engine, config *SqlWarehouseState) (string, *SqlWarehouseRemote, error) {
 	waiter, err := r.client.Warehouses.Create(ctx, config.CreateWarehouseRequest)
 	if err != nil {
 		return "", nil, err
 	}
 	id := waiter.Id
 
+	// Save state immediately after the warehouse is created so it is not orphaned
+	// if the subsequent wait or stop is interrupted.
+	engine.SaveState(ctx, id, config)
+
 	if config.Lifecycle == nil || config.Lifecycle.Started == nil {
 		return id, nil, nil
 	}