acc: add retry.py helper and avoid eventual-consistency-prone gets (#5694)

Test-only improvements toward making the direct-engine dashboard tests
robust
against eventual consistency.

- Add `acceptance/bin/retry.py`: retry a command until it succeeds, with
`--until SUBSTR` / `--until-not SUBSTR` to also gate on stdout content.
- `publish-failure-stale-content`: wait for the bumped etag to become
visible
  via `retry.py --until-not` instead of a single `lakeview get`.
- `detect-change`: read the etag from the update response instead of a
separate
  `lakeview get`.

This pull request and its description were written by Isaac.

Author: denik

acceptance/bin/retry.py

@@ -0,0 +1,53 @@
+#!/usr/bin/env python3
+"""Retry a command until it succeeds and its output matches expectations.
+
+Usage: retry.py [--until SUBSTR] [--until-not SUBSTR] CMD [ARGS...]
+
+Retries CMD up to 5 times (configurable via RETRY_MAX_ATTEMPTS env var),
+sleeping RETRY_INTERVAL_MS milliseconds (default 500) between attempts.
+An attempt is considered successful when the command exits with code 0 and:
+  --until SUBSTR     SUBSTR appears in stdout
+  --until-not SUBSTR SUBSTR does not appear in stdout
+"""
+
+import argparse
+import os
+import subprocess
+import sys
+import time
+
+
+def main():
+    parser = argparse.ArgumentParser(prog="retry.py")
+    parser.add_argument("--until")
+    parser.add_argument("--until-not")
+    parser.add_argument("cmd", nargs=argparse.REMAINDER)
+    args = parser.parse_args()
+    if not args.cmd:
+        parser.error("no command given")
+    until = args.until
+    until_not = args.until_not
+    argv = args.cmd
+
+    interval = float(os.environ.get("RETRY_INTERVAL_MS", "500")) / 1000.0
+    max_attempts = int(os.environ.get("RETRY_MAX_ATTEMPTS", "5"))
+
+    result = subprocess.run(argv, capture_output=True)
+    for _ in range(1, max_attempts):
+        success = (
+            result.returncode == 0
+            and (until is None or until.encode() in result.stdout)
+            and (until_not is None or until_not.encode() not in result.stdout)
+        )
+        if success:
+            break
+        time.sleep(interval)
+        result = subprocess.run(argv, capture_output=True)
+
+    sys.stdout.buffer.write(result.stdout)
+    sys.stderr.buffer.write(result.stderr)
+    sys.exit(result.returncode)
+
+
+if __name__ == "__main__":
+    main()

acceptance/bundle/resources/dashboards/detect-change/script

@@ -31,8 +31,9 @@ $CLI lakeview get "${DASHBOARD_ID}" | jq '{display_name,page_display_name: (.ser
 title "Make an out of band modification to the dashboard and confirm that it is detected:\n"
 RESOURCE_ID=$($CLI workspace get-status "${DASHBOARD_PATH}" | jq -r '.resource_id')
 DASHBOARD_JSON="{\"serialized_dashboard\": \"{}\", \"warehouse_id\": \"$TEST_DEFAULT_WAREHOUSE_ID\"}"
-$CLI lakeview update "${RESOURCE_ID}" --json "${DASHBOARD_JSON}" | jq '{lifecycle_state}'
-echo "$($CLI lakeview get "$DASHBOARD_ID" | jq -r '.etag'):ETAG_2" >> ACC_REPLS
+UPDATE_RESP=$($CLI lakeview update "${RESOURCE_ID}" --json "${DASHBOARD_JSON}")
+echo "$UPDATE_RESP" | jq '{lifecycle_state}'
+echo "$(echo "$UPDATE_RESP" | jq -r '.etag'):ETAG_2" >> ACC_REPLS
 
 title "Try to redeploy the bundle and confirm that the out of band modification is detected:"
 trace $CLI bundle plan

acceptance/bundle/resources/dashboards/publish-failure-stale-content/script

@@ -12,7 +12,8 @@ unset MSYS_NO_PATHCONV
 trace $CLI bundle deploy
 replace_ids.py
 DASHBOARD_ID=$($CLI bundle summary --output json | jq -r '.resources.dashboards.dashboard1.id')
-add_repl.py "$($CLI lakeview get $DASHBOARD_ID | jq -r '.etag')" ETAG_1
+ETAG_1=$($CLI lakeview get $DASHBOARD_ID | jq -r '.etag')
+add_repl.py "$ETAG_1" ETAG_1
 trace $CLI lakeview get $DASHBOARD_ID | jq '{display_name, etag}'
 trace $CLI lakeview get-published $DASHBOARD_ID | jq '{display_name}'
 trace $CLI bundle plan -o json | gron.py | grep -E "etag|published"
@@ -28,7 +29,8 @@ update_file.py databricks.yml "my dashboard" "my dashboard renamed"
 # SaveState is only called on success, so state retains the pre-PATCH etag.
 errcode trace $CLI bundle deploy
 trace print_requests.py //lakeview/dashboards
-add_repl.py "$($CLI lakeview get $DASHBOARD_ID | jq -r '.etag')" ETAG_2
+# The PATCH bumped the remote etag to ETAG_2; retry until it is visible (eventual consistency).
+add_repl.py "$(retry.py --until-not "$ETAG_1" $CLI lakeview get $DASHBOARD_ID | jq -r '.etag')" ETAG_2
 trace $CLI lakeview get $DASHBOARD_ID | jq '{display_name, etag}'
 trace $CLI lakeview get-published $DASHBOARD_ID | jq '{display_name}'
 trace $CLI bundle plan -o json | gron.py | grep -E "etag|published"