Merge branch 'main' into simonfaltum/pydabs-profile-env

Author: simonfaltum

NEXT_CHANGELOG.md

@@ -1,11 +1,12 @@
 # NEXT CHANGELOG
 
-## Release v0.300.0
+## Release v0.299.2
 
 ### CLI
 
 ### Bundles
 
 * Propagate authentication environment (including `DATABRICKS_CONFIG_PROFILE`) to the `experimental.python` subprocess so bundle validate/deploy no longer fails with a multi-profile host ambiguity error when several profiles in `~/.databrickscfg` share the same host.
+* Fixed `--force-pull` on `bundle summary` and `bundle open` so the flag bypasses the local state cache and reads state from the workspace.
 
 ### Dependency updates

acceptance/bundle/invariant/continue_293/test.toml

@@ -11,3 +11,7 @@ EnvMatrixExclude.no_vector_search_endpoint = ["INPUT_CONFIG=vector_search_endpoi
 
 # Dotted pipeline configuration keys are not supported on v0.293.0
 EnvMatrixExclude.no_pipeline_config_dots = ["INPUT_CONFIG=pipeline_config_dots.yml.tmpl"]
+
+# The 1000-task scale case is covered by no_drift. Running it here adds ~1.5 min
+# per variant (two full deploys at 1000 tasks) without incremental coverage.
+EnvMatrixExclude.no_pydabs_1000_tasks = ["INPUT_CONFIG=job_pydabs_1000_tasks.yml.tmpl"]

acceptance/bundle/invariant/migrate/test.toml

@@ -19,3 +19,7 @@ EnvMatrixExclude.no_grant_ref = ["INPUT_CONFIG=schema_grant_ref.yml.tmpl"]
 
 # SQL warehouses currently failing with migration with permanent drift. TODO: fix this.
 EnvMatrixExclude.no_sql_warehouse = ["INPUT_CONFIG=sql_warehouse.yml.tmpl"]
+
+# The 1000-task scale case is covered by no_drift. Running it here adds ~1.5 min
+# per variant (deploy + migrate + plan at 1000 tasks) without incremental coverage.
+EnvMatrixExclude.no_pydabs_1000_tasks = ["INPUT_CONFIG=job_pydabs_1000_tasks.yml.tmpl"]

acceptance/bundle/state/force_pull_commands/databricks.yml

@@ -0,0 +1,16 @@
+bundle:
+  name: force-pull-commands
+
+resources:
+  jobs:
+    foo:
+      name: foo
+      tasks:
+        - task_key: task
+          spark_python_task:
+            python_file: ./foo.py
+          environment_key: default
+      environments:
+        - environment_key: default
+          spec:
+            client: "2"

acceptance/bundle/state/force_pull_commands/foo.py

@@ -0,0 +1 @@
+print("hello")

acceptance/bundle/state/force_pull_commands/open

@@ -0,0 +1,2 @@
+#!/bin/bash
+echo "I AM BROWSER"

acceptance/bundle/state/force_pull_commands/out.test.toml

@@ -0,0 +1,35 @@
+
+>>> [CLI] bundle deploy
+Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/force-pull-commands/default/files...
+Deploying resources...
+Updating deployment state...
+Deployment complete!
+
+=== Modify PATH so that real open is not run
+=== bundle summary without --force-pull: no remote state read
+
+>>> [CLI] bundle summary
+
+=== bundle summary --force-pull: remote state read
+
+>>> [CLI] bundle summary --force-pull
+{
+  "method": "GET",
+  "path": "/api/2.0/workspace-files/Workspace/Users/[USERNAME]/.bundle/force-pull-commands/default/state/STATE_FILENAME"
+}
+
+=== bundle open without --force-pull: no remote state read
+
+>>> [CLI] bundle open foo
+Opening browser at [DATABRICKS_URL]/jobs/[NUMID]?o=[NUMID]
+Error: exec: "open": cannot run executable found relative to current directory
+
+=== bundle open --force-pull: remote state read
+
+>>> [CLI] bundle open foo --force-pull
+Opening browser at [DATABRICKS_URL]/jobs/[NUMID]?o=[NUMID]
+Error: exec: "open": cannot run executable found relative to current directory
+{
+  "method": "GET",
+  "path": "/api/2.0/workspace-files/Workspace/Users/[USERNAME]/.bundle/force-pull-commands/default/state/STATE_FILENAME"
+}

acceptance/bundle/state/force_pull_commands/output.txt

@@ -0,0 +1,21 @@
+trace $CLI bundle deploy > /dev/null
+rm -f out.requests.txt
+
+title "Modify PATH so that real open is not run"
+export PATH=.:$PATH
+
+title "bundle summary without --force-pull: no remote state read\n"
+trace $CLI bundle summary > /dev/null
+print_requests.py --get //workspace-files/
+
+title "bundle summary --force-pull: remote state read\n"
+trace $CLI bundle summary --force-pull > /dev/null
+print_requests.py --get //workspace-files/
+
+title "bundle open without --force-pull: no remote state read\n"
+musterr trace $CLI bundle open foo > /dev/null
+print_requests.py --get //workspace-files/
+
+title "bundle open --force-pull: remote state read\n"
+musterr trace $CLI bundle open foo --force-pull > /dev/null
+print_requests.py --get //workspace-files/

acceptance/bundle/state/force_pull_commands/script

@@ -0,0 +1,17 @@
+Local = true
+Cloud = false
+RecordRequests = true
+
+# bundle open opens a browser; restrict to darwin to get stable output
+# (mirrors acceptance/bundle/open/test.toml).
+GOOS.windows = false
+GOOS.linux = false
+
+Ignore = [".databricks"]
+
+[EnvMatrix]
+  DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"]
+
+[[Repls]]
+Old = '(resources\.json|terraform\.tfstate)'
+New = 'STATE_FILENAME'