acc: simulate dashboard eventual consistency and harden dashboard tests (databricks#5724)

denik · web-flow · commit e8aa56f79dba · 2026-07-06T18:44:21.000+02:00
Follow-up to databricks#5694. We've seen the dashboard API be eventually consistent: a GET right after a create can 404 before the write propagates. The dashboard acceptance tests run on cloud (`Cloud = true`) and read a dashboard immediately after deploy, so they are exposed to this. - Add a testserver EC mode for dashboards: under the eventual-consistency token (opt-in via `TESTS_STALE_ONCE=1`, direct engine only) the first GET of a dashboard after it is created returns 404. This reproduces the cloud window deterministically. - Enable it for the dashboards directory and wrap the first `lakeview get` after each create with `retry.py`, so reads right after deploy are retried rather than assumed to succeed. Test-only: no engine changes. Making the direct engine itself resilient to the same 404 (plan/refresh reads) is a separate follow-up. This pull request and its description were written by Isaac.
diff --git a/acceptance/acceptance_test.go b/acceptance/acceptance_test.go
@@ -767,7 +767,7 @@ func runTest(t *testing.T,
 	args := []string{"bash", "-euo", "pipefail", EntryPointScript}
 	cmd := exec.CommandContext(ctx, args[0], args[1:]...)
 
-	cfg, user := internal.PrepareServerAndClient(t, config, LogRequests, tmpDir)
+	cfg, user := internal.PrepareServerAndClient(t, config, LogRequests, tmpDir, testEnv)
 	testdiff.PrepareReplacementsUser(t, &repls, user)
 	testdiff.PrepareReplacementsWorkspaceConfig(t, &repls, cfg)
 
diff --git a/acceptance/bundle/resources/dashboards/change-embed-credentials/output.txt b/acceptance/bundle/resources/dashboards/change-embed-credentials/output.txt
@@ -9,8 +9,6 @@ Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/change-embed-crede
 Deploying resources...
 Updating deployment state...
 Deployment complete!
-
->>> [CLI] lakeview get [DASHBOARD_ID]
 {
   "display_name": "test dashboard",
   "lifecycle_state": "ACTIVE",
@@ -36,8 +34,6 @@ Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/change-embed-crede
 Deploying resources...
 Updating deployment state...
 Deployment complete!
-
->>> [CLI] lakeview get [DASHBOARD_ID]
 {
   "display_name": "test dashboard",
   "lifecycle_state": "ACTIVE",
diff --git a/acceptance/bundle/resources/dashboards/change-embed-credentials/script b/acceptance/bundle/resources/dashboards/change-embed-credentials/script
@@ -15,18 +15,16 @@ deploy_dashboard() {
     trace $CLI bundle deploy
     dashboard_id=$($CLI bundle summary --output json | jq -r '.resources.dashboards.my_dashboard.id')
 
-    trace $CLI lakeview get $dashboard_id | jq "{display_name, lifecycle_state, parent_path, path, serialized_dashboard, warehouse_id}"
+    DASHBOARD=$(retry $CLI lakeview get $dashboard_id)
+    echo "$dashboard_id:DASHBOARD_ID" >> ACC_REPLS
+    echo "$(echo "$DASHBOARD" | jq -r '.etag'):ETAG" >> ACC_REPLS
+    echo "$DASHBOARD" | jq "{display_name, lifecycle_state, parent_path, path, serialized_dashboard, warehouse_id}"
     trace $CLI lakeview get-published $dashboard_id | jq "{warehouse_id, embed_credentials}"
 }
 
 # Create the dashboard with embed_credentials=false
 deploy_dashboard
 
-# Capture the dashboard ID as a replacement.
-dashboard_id=$($CLI bundle summary --output json | jq -r '.resources.dashboards.my_dashboard.id')
-echo "$dashboard_id:DASHBOARD_ID" >> ACC_REPLS
-echo "$($CLI lakeview get $dashboard_id | jq -r '.etag'):ETAG" >> ACC_REPLS
-
 # Change embed_credentials to true - this should trigger an update
 export EMBED_CREDENTIALS="true"
 deploy_dashboard
diff --git a/acceptance/bundle/resources/dashboards/change-name/output.txt b/acceptance/bundle/resources/dashboards/change-name/output.txt
@@ -9,8 +9,6 @@ Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/change-name-[UNIQU
 Deploying resources...
 Updating deployment state...
 Deployment complete!
-
->>> [CLI] lakeview get [DASHBOARD_ID]
 {
   "display_name": "dashboard1",
   "lifecycle_state": "ACTIVE",
@@ -30,8 +28,6 @@ Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/change-name-[UNIQU
 Deploying resources...
 Updating deployment state...
 Deployment complete!
-
->>> [CLI] lakeview get [DASHBOARD_ID]
 {
   "display_name": "dashboard2",
   "lifecycle_state": "ACTIVE",
diff --git a/acceptance/bundle/resources/dashboards/change-name/script b/acceptance/bundle/resources/dashboards/change-name/script
@@ -12,7 +12,10 @@ deploy_dashboard() {
     trace $CLI bundle plan
     trace $CLI bundle deploy
     dashboard_id=$($CLI bundle summary --output json | jq -r '.resources.dashboards.my_dashboard.id')
-    trace $CLI lakeview get $dashboard_id | jq '{display_name, lifecycle_state, parent_path, path, serialized_dashboard, warehouse_id}'
+    # Wait out eventual consistency: retry until the current name is visible (the read
+    # 404s on create and returns the stale name right after a rename).
+    DASHBOARD=$(retry --until "$NAME" $CLI lakeview get $dashboard_id)
+    echo "$DASHBOARD" | jq '{display_name, lifecycle_state, parent_path, path, serialized_dashboard, warehouse_id}'
 }
 
 # Create the dashboard
@@ -21,7 +24,7 @@ deploy_dashboard
 # Capture the dashboard ID as a replacement.
 dashboard_id=$($CLI bundle summary --output json | jq -r '.resources.dashboards.my_dashboard.id')
 echo "$dashboard_id:DASHBOARD_ID" >> ACC_REPLS
-echo "$($CLI lakeview get $dashboard_id | jq -r '.etag'):ETAG" >> ACC_REPLS
+echo "$(echo "$DASHBOARD" | jq -r '.etag'):ETAG" >> ACC_REPLS
 
 # Change the name
 export NAME="dashboard2"
diff --git a/acceptance/bundle/resources/dashboards/change-parent-path/output.txt b/acceptance/bundle/resources/dashboards/change-parent-path/output.txt
@@ -10,7 +10,7 @@ Deploying resources...
 Updating deployment state...
 Deployment complete!
 
->>> [CLI] lakeview get [ORIGINAL_DASHBOARD_ID]
+>>> retry [CLI] lakeview get [ORIGINAL_DASHBOARD_ID]
 {
   "display_name": "test dashboard",
   "lifecycle_state": "ACTIVE",
@@ -35,7 +35,7 @@ Deploying resources...
 Updating deployment state...
 Deployment complete!
 
->>> [CLI] lakeview get [NEW_DASHBOARD_ID]
+>>> retry [CLI] lakeview get [NEW_DASHBOARD_ID]
 {
   "display_name": "test dashboard",
   "lifecycle_state": "ACTIVE",
diff --git a/acceptance/bundle/resources/dashboards/change-parent-path/script b/acceptance/bundle/resources/dashboards/change-parent-path/script
@@ -14,7 +14,7 @@ deploy_dashboard() {
     trace $CLI bundle plan
     trace $CLI bundle deploy --auto-approve
     dashboard_id=$($CLI bundle summary --output json | jq -r '.resources.dashboards.my_dashboard.id')
-    trace $CLI lakeview get $dashboard_id | jq '{display_name, lifecycle_state, parent_path, path, serialized_dashboard, warehouse_id}'
+    trace retry $CLI lakeview get $dashboard_id | jq '{display_name, lifecycle_state, parent_path, path, serialized_dashboard, warehouse_id}'
 }
 
 # Create the dashboard with initial parent_path
diff --git a/acceptance/bundle/resources/dashboards/change-serialized-dashboard/output.txt b/acceptance/bundle/resources/dashboards/change-serialized-dashboard/output.txt
@@ -9,17 +9,13 @@ Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/change-serialized-
 Deploying resources...
 Updating deployment state...
 Deployment complete!
-
->>> [CLI] lakeview get [DASHBOARD_ID]
 {
   "display_name": "test dashboard",
   "lifecycle_state": "ACTIVE",
   "parent_path": "/Users/[USERNAME]/.bundle/change-serialized-dashboard-[UNIQUE_NAME]/default/resources",
   "path": "/Users/[USERNAME]/.bundle/change-serialized-dashboard-[UNIQUE_NAME]/default/resources/test dashboard.lvdash.json",
   "warehouse_id": "[TEST_DEFAULT_WAREHOUSE_ID]"
 }
-
->>> [CLI] lakeview get [DASHBOARD_ID]
 {
   "displayName": "name1",
   "name": "name1",
@@ -36,17 +32,13 @@ Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/change-serialized-
 Deploying resources...
 Updating deployment state...
 Deployment complete!
-
->>> [CLI] lakeview get [DASHBOARD_ID]
 {
   "display_name": "test dashboard",
   "lifecycle_state": "ACTIVE",
   "parent_path": "/Users/[USERNAME]/.bundle/change-serialized-dashboard-[UNIQUE_NAME]/default/resources",
   "path": "/Users/[USERNAME]/.bundle/change-serialized-dashboard-[UNIQUE_NAME]/default/resources/test dashboard.lvdash.json",
   "warehouse_id": "[TEST_DEFAULT_WAREHOUSE_ID]"
 }
-
->>> [CLI] lakeview get [DASHBOARD_ID]
 {
   "displayName": "name2",
   "name": "name2",
diff --git a/acceptance/bundle/resources/dashboards/change-serialized-dashboard/script b/acceptance/bundle/resources/dashboards/change-serialized-dashboard/script
@@ -12,8 +12,11 @@ deploy_dashboard() {
     trace $CLI bundle deploy
     dashboard_id=$($CLI bundle summary --output json | jq -r '.resources.dashboards.my_dashboard.id')
 
-    trace $CLI lakeview get $dashboard_id | jq '{display_name, lifecycle_state, parent_path, path, warehouse_id}'
-    trace $CLI lakeview get $dashboard_id | jq '.serialized_dashboard | fromjson' | jq -S '.pages[0]'
+    # Wait out eventual consistency: retry until the current page is visible (the read
+    # 404s on create and returns the stale page right after the serialized update).
+    DASHBOARD=$(retry --until "$PAGE_NAME" $CLI lakeview get $dashboard_id)
+    echo "$DASHBOARD" | jq '{display_name, lifecycle_state, parent_path, path, warehouse_id}'
+    echo "$DASHBOARD" | jq '.serialized_dashboard | fromjson' | jq -S '.pages[0]'
 }
 
 # Create the dashboard with initial serialized_dashboard
diff --git a/acceptance/bundle/resources/dashboards/dataset-catalog-schema/output.txt b/acceptance/bundle/resources/dashboards/dataset-catalog-schema/output.txt
@@ -4,15 +4,11 @@ Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/deploy-dashboard-d
 Deploying resources...
 Updating deployment state...
 Deployment complete!
-
->>> [CLI] lakeview get [DASHBOARD_ID]
 {
   "lifecycle_state": "ACTIVE",
   "parent_path": "/Users/[USERNAME]",
   "path": "/Users/[USERNAME]/test bundle-deploy-dashboard-dataset [UUID].lvdash.json"
 }
-
->>> [CLI] lakeview get [DASHBOARD_ID]
 {
   "catalog": "main",
   "schema": "default"
diff --git a/acceptance/bundle/resources/dashboards/dataset-catalog-schema/script b/acceptance/bundle/resources/dashboards/dataset-catalog-schema/script
@@ -19,13 +19,14 @@ DASHBOARD_ID=$($CLI bundle summary --output json | jq -r '.resources.dashboards.
 
 # Capture the dashboard ID as a replacement.
 echo "$DASHBOARD_ID:DASHBOARD_ID" >> ACC_REPLS
-echo "$($CLI lakeview get $DASHBOARD_ID | jq -r '.etag'):ETAG" >> ACC_REPLS
+DASHBOARD=$(retry $CLI lakeview get $DASHBOARD_ID)
+echo "$(echo "$DASHBOARD" | jq -r '.etag'):ETAG" >> ACC_REPLS
 
-trace $CLI lakeview get $DASHBOARD_ID | jq '{lifecycle_state, parent_path, path}'
+echo "$DASHBOARD" | jq '{lifecycle_state, parent_path, path}'
 
 # Verify that the serialized_dashboard datasets have the overridden catalog/schema values.
 # The dataset_catalog and dataset_schema parameters should override the values in the datasets.
-trace $CLI lakeview get $DASHBOARD_ID | jq '.serialized_dashboard | fromjson | .datasets[] | {catalog, schema}'
+echo "$DASHBOARD" | jq '.serialized_dashboard | fromjson | .datasets[] | {catalog, schema}'
 
 # Verify that there is no drift right after deploy.
 trace $CLI bundle plan -o json > out.plan.$DATABRICKS_BUNDLE_ENGINE.json
diff --git a/acceptance/bundle/resources/dashboards/delete-trashed-out-of-band/output.txt b/acceptance/bundle/resources/dashboards/delete-trashed-out-of-band/output.txt
@@ -4,8 +4,6 @@ Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/delete-trashed-out
 Deploying resources...
 Updating deployment state...
 Deployment complete!
-
->>> [CLI] lakeview get [DASHBOARD_ID]
 {
   "lifecycle_state": "ACTIVE",
   "parent_path": "/Users/[USERNAME]/.bundle/delete-trashed-out-of-band-[UNIQUE_NAME]/default/resources"
diff --git a/acceptance/bundle/resources/dashboards/delete-trashed-out-of-band/script b/acceptance/bundle/resources/dashboards/delete-trashed-out-of-band/script
@@ -13,10 +13,11 @@ DASHBOARD_ID=$($CLI bundle summary --output json | jq -r '.resources.dashboards.
 
 # Capture the dashboard ID as a replacement.
 echo "$DASHBOARD_ID:DASHBOARD_ID" >> ACC_REPLS
-echo "$($CLI lakeview get $DASHBOARD_ID | jq -r '.etag'):ETAG" >> ACC_REPLS
+DASHBOARD=$(retry $CLI lakeview get $DASHBOARD_ID)
+echo "$(echo "$DASHBOARD" | jq -r '.etag'):ETAG" >> ACC_REPLS
 
 # Verify dashboard was deployed
-trace $CLI lakeview get $DASHBOARD_ID | jq '{lifecycle_state, parent_path}'
+echo "$DASHBOARD" | jq '{lifecycle_state, parent_path}'
 
 # Trash the dashboard out of band (simulating external deletion)
 # This should succeed
diff --git a/acceptance/bundle/resources/dashboards/destroy/output.txt b/acceptance/bundle/resources/dashboards/destroy/output.txt
@@ -5,7 +5,7 @@ Deploying resources...
 Updating deployment state...
 Deployment complete!
 
->>> [CLI] lakeview get [DASHBOARD_ID]
+>>> retry [CLI] lakeview get [DASHBOARD_ID]
 {
   "lifecycle_state": "ACTIVE",
   "parent_path": "/Users/[USERNAME]/.bundle/destroy-dashboard-[UNIQUE_NAME]/default/resources"
diff --git a/acceptance/bundle/resources/dashboards/destroy/script b/acceptance/bundle/resources/dashboards/destroy/script
@@ -14,7 +14,7 @@ DASHBOARD_ID=$($CLI bundle summary --output json | jq -r '.resources.dashboards.
 echo "$DASHBOARD_ID:DASHBOARD_ID" >> ACC_REPLS
 
 # Verify dashboard was deployed
-trace $CLI lakeview get $DASHBOARD_ID | jq '{lifecycle_state, parent_path}'
+trace retry $CLI lakeview get $DASHBOARD_ID | jq '{lifecycle_state, parent_path}'
 
 trace $CLI bundle destroy --auto-approve
 
diff --git a/acceptance/bundle/resources/dashboards/detect-change/script b/acceptance/bundle/resources/dashboards/detect-change/script
@@ -22,11 +22,12 @@ DASHBOARD_ID=$(jq -r '.id' out.summary.json)
 
 # Capture the dashboard ID as a replacement.
 echo "$DASHBOARD_ID:DASHBOARD_ID" >> ACC_REPLS
-echo "$($CLI lakeview get "$DASHBOARD_ID" | jq -r '.etag'):ETAG_1" >> ACC_REPLS
+DASHBOARD=$(retry $CLI lakeview get "$DASHBOARD_ID")
+echo "$(echo "$DASHBOARD" | jq -r '.etag'):ETAG_1" >> ACC_REPLS
 
 rm out.summary.json
 title "Load the dashboard by its ID and confirm its display name: "
-$CLI lakeview get "${DASHBOARD_ID}" | jq '{display_name,page_display_name: (.serialized_dashboard | fromjson | .pages[0].displayName)}'
+echo "$DASHBOARD" | jq '{display_name,page_display_name: (.serialized_dashboard | fromjson | .pages[0].displayName)}'
 
 title "Make an out of band modification to the dashboard and confirm that it is detected:\n"
 RESOURCE_ID=$($CLI workspace get-status "${DASHBOARD_PATH}" | jq -r '.resource_id')
diff --git a/acceptance/bundle/resources/dashboards/detect-change/test.toml b/acceptance/bundle/resources/dashboards/detect-change/test.toml
@@ -11,6 +11,12 @@ Ignore = [
 # Setting this environment variable prevents that conversion on windows.
 MSYS_NO_PATHCONV = "1"
 
+# This test makes an out-of-band update and then expects `bundle plan` to detect it.
+# Under eventual consistency the plan's own read (CheckDashboardsModifiedRemotely)
+# would observe the stale pre-update value and miss the change -- an engine-level read
+# that no test-level retry can fix -- so opt out of the simulation here.
+INJECT_STALE_ON_DIRECT = "0"
+
 [[Repls]]
 Old = "[0-9a-z]{16,}"
 New = "[ALPHANUMID]"
diff --git a/acceptance/bundle/resources/dashboards/generate_inplace/script b/acceptance/bundle/resources/dashboards/generate_inplace/script
@@ -12,7 +12,7 @@ trap cleanup EXIT
 dashboard_id=$($CLI bundle summary -o json | jq .resources.dashboards.already_exists.id -r)
 
 title "Get current dashboard\n"
-$CLI lakeview get $dashboard_id | jq .serialized_dashboard
+retry $CLI lakeview get $dashboard_id | jq .serialized_dashboard
 
 title "Update dashboard\n"
 $CLI lakeview update $dashboard_id --display-name "Updated dashboard" | jq .serialized_dashboard
diff --git a/acceptance/bundle/resources/dashboards/nested-folders/output.txt b/acceptance/bundle/resources/dashboards/nested-folders/output.txt
@@ -5,7 +5,7 @@ Deploying resources...
 Updating deployment state...
 Deployment complete!
 
->>> [CLI] lakeview get [DASHBOARD_ID]
+>>> retry [CLI] lakeview get [DASHBOARD_ID]
 {
   "lifecycle_state": "ACTIVE",
   "parent_path": "/Users/[USERNAME]",
diff --git a/acceptance/bundle/resources/dashboards/nested-folders/script b/acceptance/bundle/resources/dashboards/nested-folders/script
@@ -18,4 +18,4 @@ DASHBOARD_ID=$($CLI bundle summary --output json | jq -r '.resources.dashboards.
 # Capture the dashboard ID as a replacement.
 echo "$DASHBOARD_ID:DASHBOARD_ID" >> ACC_REPLS
 
-trace $CLI lakeview get $DASHBOARD_ID | jq '{lifecycle_state, parent_path, path, serialized_dashboard: (.serialized_dashboard | fromjson | {pages: (.pages | map({name, displayName, pageType}))})}'
+trace retry $CLI lakeview get $DASHBOARD_ID | jq '{lifecycle_state, parent_path, path, serialized_dashboard: (.serialized_dashboard | fromjson | {pages: (.pages | map({name, displayName, pageType}))})}'
diff --git a/acceptance/bundle/resources/dashboards/publish-failure-stale-content/output.txt b/acceptance/bundle/resources/dashboards/publish-failure-stale-content/output.txt
@@ -4,8 +4,6 @@ Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/update-publish-fai
 Deploying resources...
 Updating deployment state...
 Deployment complete!
-
->>> [CLI] lakeview get [DASHBOARD1_ID]
 {
   "display_name": "my dashboard",
   "etag": "[ETAG_1]"
diff --git a/acceptance/bundle/resources/dashboards/publish-failure-stale-content/script b/acceptance/bundle/resources/dashboards/publish-failure-stale-content/script
@@ -12,9 +12,10 @@ unset MSYS_NO_PATHCONV
 trace $CLI bundle deploy
 replace_ids.py
 DASHBOARD_ID=$($CLI bundle summary --output json | jq -r '.resources.dashboards.dashboard1.id')
-ETAG_1=$($CLI lakeview get $DASHBOARD_ID | jq -r '.etag')
+DASHBOARD=$(retry $CLI lakeview get $DASHBOARD_ID)
+ETAG_1=$(echo "$DASHBOARD" | jq -r '.etag')
 add_repl.py "$ETAG_1" ETAG_1
-trace $CLI lakeview get $DASHBOARD_ID | jq '{display_name, etag}'
+echo "$DASHBOARD" | jq '{display_name, etag}'
 trace $CLI lakeview get-published $DASHBOARD_ID | jq '{display_name}'
 trace $CLI bundle plan -o json | gron.py | grep -E "etag|published"
 rm out.requests.txt
@@ -30,7 +31,8 @@ update_file.py databricks.yml "my dashboard" "my dashboard renamed"
 errcode trace $CLI bundle deploy
 trace print_requests.py //lakeview/dashboards
 # The PATCH bumped the remote etag to ETAG_2; retry until it is visible (eventual consistency).
-add_repl.py "$(retry.py --until-not "$ETAG_1" $CLI lakeview get $DASHBOARD_ID | jq -r '.etag')" ETAG_2
+ETAG_2=$(retry --until-not "$ETAG_1" $CLI lakeview get $DASHBOARD_ID | jq -r '.etag')
+add_repl.py "$ETAG_2" ETAG_2
 trace $CLI lakeview get $DASHBOARD_ID | jq '{display_name, etag}'
 trace $CLI lakeview get-published $DASHBOARD_ID | jq '{display_name}'
 trace $CLI bundle plan -o json | gron.py | grep -E "etag|published"
@@ -47,7 +49,9 @@ trace $CLI lakeview get-published $DASHBOARD_ID | jq '{display_name}'
 # which fixes the stale published content.
 trace $CLI bundle deploy --force
 trace print_requests.py //lakeview/dashboards
-add_repl.py "$($CLI lakeview get $DASHBOARD_ID | jq -r '.etag')" ETAG_3
+# --force did another PATCH; retry until the new etag is visible (eventual consistency).
+ETAG_3=$(retry --until-not "$ETAG_2" $CLI lakeview get $DASHBOARD_ID | jq -r '.etag')
+add_repl.py "$ETAG_3" ETAG_3
 trace $CLI lakeview get $DASHBOARD_ID | jq '{display_name, etag}'
 trace $CLI lakeview get-published $DASHBOARD_ID | jq '{display_name}'
 trace $CLI bundle plan
diff --git a/acceptance/bundle/resources/dashboards/simple/output.txt b/acceptance/bundle/resources/dashboards/simple/output.txt
@@ -4,8 +4,6 @@ Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/deploy-dashboard-t
 Deploying resources...
 Updating deployment state...
 Deployment complete!
-
->>> [CLI] lakeview get [DASHBOARD_ID]
 {
   "lifecycle_state": "ACTIVE",
   "parent_path": "/Users/[USERNAME]",
diff --git a/acceptance/bundle/resources/dashboards/simple/script b/acceptance/bundle/resources/dashboards/simple/script
@@ -17,9 +17,10 @@ DASHBOARD_ID=$($CLI bundle summary --output json | jq -r '.resources.dashboards.
 
 # Capture the dashboard ID as a replacement.
 echo "$DASHBOARD_ID:DASHBOARD_ID" >> ACC_REPLS
-echo "$($CLI lakeview get $DASHBOARD_ID | jq -r '.etag'):ETAG" >> ACC_REPLS
+DASHBOARD=$(retry $CLI lakeview get $DASHBOARD_ID)
+echo "$(echo "$DASHBOARD" | jq -r '.etag'):ETAG" >> ACC_REPLS
 
-trace $CLI lakeview get $DASHBOARD_ID | jq '{lifecycle_state, parent_path, path, serialized_dashboard}'
+echo "$DASHBOARD" | jq '{lifecycle_state, parent_path, path, serialized_dashboard}'
 
 # Verify that there is no drift right after deploy. We use |= to update the serialized_dashboard field in place to account
 # for formatting that the cloud server applies to the serialized_dashboard field.
diff --git a/acceptance/bundle/resources/dashboards/simple_outside_bundle_root/output.txt b/acceptance/bundle/resources/dashboards/simple_outside_bundle_root/output.txt
@@ -5,7 +5,7 @@ Deploying resources...
 Updating deployment state...
 Deployment complete!
 
->>> [CLI] lakeview get [DASHBOARD_ID]
+>>> retry [CLI] lakeview get [DASHBOARD_ID]
 {
   "lifecycle_state": "ACTIVE",
   "parent_path": "/Users/[USERNAME]",
diff --git a/acceptance/bundle/resources/dashboards/simple_outside_bundle_root/script b/acceptance/bundle/resources/dashboards/simple_outside_bundle_root/script
@@ -18,4 +18,4 @@ DASHBOARD_ID=$($CLI bundle summary --output json | jq -r '.resources.dashboards.
 # Capture the dashboard ID as a replacement.
 echo "$DASHBOARD_ID:DASHBOARD_ID" >> ACC_REPLS
 
-trace $CLI lakeview get $DASHBOARD_ID | jq '{lifecycle_state, parent_path, path, serialized_dashboard}'
+trace retry $CLI lakeview get $DASHBOARD_ID | jq '{lifecycle_state, parent_path, path, serialized_dashboard}'
diff --git a/acceptance/bundle/resources/dashboards/simple_syncroot/output.txt b/acceptance/bundle/resources/dashboards/simple_syncroot/output.txt
@@ -5,7 +5,7 @@ Deploying resources...
 Updating deployment state...
 Deployment complete!
 
->>> [CLI] lakeview get [DASHBOARD_ID]
+>>> retry [CLI] lakeview get [DASHBOARD_ID]
 {
   "lifecycle_state": "ACTIVE",
   "parent_path": "/Users/[USERNAME]",
diff --git a/acceptance/bundle/resources/dashboards/simple_syncroot/script b/acceptance/bundle/resources/dashboards/simple_syncroot/script
@@ -19,4 +19,4 @@ DASHBOARD_ID=$($CLI bundle summary --output json | jq -r '.resources.dashboards.
 # Capture the dashboard ID as a replacement.
 echo "$DASHBOARD_ID:DASHBOARD_ID" >> ACC_REPLS
 
-trace $CLI lakeview get $DASHBOARD_ID | jq '{lifecycle_state, parent_path, path, serialized_dashboard}'
+trace retry $CLI lakeview get $DASHBOARD_ID | jq '{lifecycle_state, parent_path, path, serialized_dashboard}'
diff --git a/acceptance/bundle/resources/dashboards/test.toml b/acceptance/bundle/resources/dashboards/test.toml
@@ -10,3 +10,8 @@ RequiresWarehouse = true
 # C:/Program Files/Git/Users/$username/UNIQUE_NAME before passing it to the CLI
 # Setting this environment variable prevents that conversion on windows.
 MSYS_NO_PATHCONV = "1"
+
+# Direct-engine eventual consistency: the first GET of a freshly created dashboard
+# returns 404, so reads right after deploy must be retried (retry.py). Skipped on
+# terraform, whose provider reads the dashboard back after create and fails on the 404.
+INJECT_STALE_ON_DIRECT = "1"
diff --git a/acceptance/bundle/resources/dashboards/unpublish-out-of-band/script b/acceptance/bundle/resources/dashboards/unpublish-out-of-band/script
diff --git a/acceptance/internal/prepare_server.go b/acceptance/internal/prepare_server.go
diff --git a/acceptance/script.prepare b/acceptance/script.prepare
diff --git a/libs/testserver/dashboards.go b/libs/testserver/dashboards.go
diff --git a/libs/testserver/eventual.go b/libs/testserver/eventual.go
diff --git a/libs/testserver/fake_workspace.go b/libs/testserver/fake_workspace.go
diff --git a/libs/testserver/handlers.go b/libs/testserver/handlers.go

Original file line number	Diff line number	Diff line change
`@@ -14,7 +14,7 @@ deploy_dashboard() {`
`14`	`14`	`trace $CLI bundle plan`
`15`	`15`	`trace $CLI bundle deploy --auto-approve`
`16`	`16`	`dashboard_id=$($CLI bundle summary --output json \| jq -r '.resources.dashboards.my_dashboard.id')`
`17`		`- trace $CLI lakeview get $dashboard_id \| jq '{display_name, lifecycle_state, parent_path, path, serialized_dashboard, warehouse_id}'`
	`17`	`+ trace retry $CLI lakeview get $dashboard_id \| jq '{display_name, lifecycle_state, parent_path, path, serialized_dashboard, warehouse_id}'`
`18`	`18`	`}`
`19`	`19`
`20`	`20`	`# Create the dashboard with initial parent_path`
Original file line number	Diff line number	Diff line change
`@@ -4,15 +4,11 @@ Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/deploy-dashboard-d`
`4`	`4`	`Deploying resources...`
`5`	`5`	`Updating deployment state...`
`6`	`6`	`Deployment complete!`
`7`		`-`
`8`		`->>> [CLI] lakeview get [DASHBOARD_ID]`
`9`	`7`	`{`
`10`	`8`	`"lifecycle_state": "ACTIVE",`
`11`	`9`	`"parent_path": "/Users/[USERNAME]",`
`12`	`10`	`"path": "/Users/[USERNAME]/test bundle-deploy-dashboard-dataset [UUID].lvdash.json"`
`13`	`11`	`}`
`14`		`-`
`15`		`->>> [CLI] lakeview get [DASHBOARD_ID]`
`16`	`12`	`{`
`17`	`13`	`"catalog": "main",`
`18`	`14`	`"schema": "default"`