Merge branch 'main' into simonfaltum/version-update-notice

Author: simonfaltum

acceptance/acceptance_test.go

@@ -48,6 +48,7 @@ var (
 	WorkspaceTmpDir bool
 	OnlyOutTestToml bool
 	Subset          bool
+	DebugSandbox    bool
 )
 
 // In order to debug CLI running under acceptance test, search for TestInprocessMode and update
@@ -80,6 +81,7 @@ func init() {
 	flag.BoolVar(&WorkspaceTmpDir, "workspace-tmp-dir", false, "Run tests on the workspace file system (For DBR testing).")
 	flag.BoolVar(&OnlyOutTestToml, "only-out-test-toml", false, "Only regenerate out.test.toml files without running tests")
 	flag.BoolVar(&Subset, "subset", false, "Select a subset of EnvMatrix variants that cover all output files. Auto-enabled on -update (unless -run specifies a variant with '=').")
+	flag.BoolVar(&DebugSandbox, "debugsandbox", false, "Use a per-test blocking proxy instead of a shared one; shows exactly which test caused unexpected internet access (slower)")
 }
 
 const (
@@ -328,6 +330,12 @@ func testAccept(t *testing.T, inprocessMode bool, singleTest string) int {
 		}
 	}
 
+	const sharedProxyHint = "; re-run with -debugsandbox to see which test caused this"
+	sandboxProxyURL := ""
+	if cloudEnv == "" && !DebugSandbox {
+		sandboxProxyURL = internal.StartRejectingProxy(t, sharedProxyHint)
+	}
+
 	setReplsForTestEnvVars(t, &repls)
 
 	if cloudEnv != "" && UseVersion == "" {
@@ -455,15 +463,15 @@ func testAccept(t *testing.T, inprocessMode bool, singleTest string) int {
 			// If the matrix expands to a single empty envset, run the test directly
 			// without creating a subtest (avoids the "#00" dummy subtest name).
 			if len(expanded) == 1 && len(expanded[0]) == 0 {
-				runTest(t, dir, 0, coverDir, repls.Clone(), config, nil, envFilters)
+				runTest(t, dir, 0, coverDir, repls.Clone(), config, nil, envFilters, sandboxProxyURL)
 			} else {
 				for ind, envset := range expanded {
 					envname := strings.Join(envset, "/")
 					t.Run(envname, func(t *testing.T) {
 						if runParallel {
 							t.Parallel()
 						}
-						runTest(t, dir, ind, coverDir, repls.Clone(), config, envset, envFilters)
+						runTest(t, dir, ind, coverDir, repls.Clone(), config, envset, envFilters, sandboxProxyURL)
 					})
 				}
 			}
@@ -609,6 +617,7 @@ func runTest(t *testing.T,
 	config internal.TestConfig,
 	customEnv []string,
 	envFilters []string,
+	sharedProxyURL string,
 ) {
 	// Check env filters early, before creating any resources like directories on the file system.
 	// Creating / deleting too many directories causes this error on the workspace FUSE mount:
@@ -763,6 +772,26 @@ func runTest(t *testing.T,
 	cmd.Env = append(cmd.Env, "TESTDIR="+absDir)
 	cmd.Env = append(cmd.Env, "CLOUD_ENV="+cloudEnv)
 	cmd.Env = append(cmd.Env, "CURRENT_USER_NAME="+user.UserName)
+	if !isRunningOnCloud {
+		proxyURL := sharedProxyURL
+		if DebugSandbox {
+			// Per-test proxy: errors are attributed to this subtest's t, making
+			// it immediately clear which test caused the internet access.
+			proxyURL = internal.StartRejectingProxy(t, "")
+		}
+		// Only block HTTPS: the local test server is plain HTTP (http://127.0.0.1:PORT)
+		// so HTTP_PROXY would intercept its traffic. All real external calls use HTTPS.
+		cmd.Env = append(cmd.Env, "HTTPS_PROXY="+proxyURL)
+		// Python's urllib does not automatically bypass the proxy for loopback
+		// addresses the way Go does, so the test-server helper scripts
+		// (kill_after.py, callserver.py, …) would route their requests through
+		// the blocking proxy. NO_PROXY exempts them.
+		cmd.Env = append(cmd.Env, "NO_PROXY=127.0.0.1,localhost")
+		// Terraform phones home to checkpoint-api.hashicorp.com on every run to
+		// check for updates. Disable it so these CONNECT requests don't reach the
+		// blocking proxy and fail every terraform-engine test.
+		cmd.Env = append(cmd.Env, "CHECKPOINT_DISABLE=1")
+	}
 	cmd.Dir = tmpDir
 
 	outputPath := filepath.Join(tmpDir, "output.txt")

acceptance/auth/bundle_and_profile/.databrickscfg

@@ -2,4 +2,4 @@
 host = $DATABRICKS_HOST
 
 [profile_name]
-host = https://test@non.existing.subdomain.databricks.com
+host = https://test@non.existing.subdomain.databricks.test

acceptance/auth/bundle_and_profile/output.txt

@@ -13,14 +13,14 @@
 
 === Inside the bundle, profile flag not matching bundle host. Should use profile from the flag and not the bundle.
 >>> errcode [CLI] current-user me -p profile_name
-Warn: [hostmetadata] failed to fetch host metadata for https://non.existing.subdomain.databricks.com, will skip for 1m0s
-Error: Get "https://non.existing.subdomain.databricks.com/api/2.0/preview/scim/v2/Me": (redacted)
+Warn: [hostmetadata] failed to fetch host metadata for https://non.existing.subdomain.databricks.test, will skip for 1m0s
+Error: Get "https://non.existing.subdomain.databricks.test/api/2.0/preview/scim/v2/Me": (redacted)
 
 Exit code: 1
 
 === Inside the bundle, target and not matching profile
 >>> errcode [CLI] current-user me -t dev -p profile_name
-Error: cannot resolve bundle auth configuration: the host in the profile (https://non.existing.subdomain.databricks.com) doesn’t match the host configured in the bundle ([DATABRICKS_TARGET]). The profile "DEFAULT" has host="[DATABRICKS_TARGET]" that matches host in the bundle. To select it, pass "-p DEFAULT"
+Error: cannot resolve bundle auth configuration: the host in the profile (https://non.existing.subdomain.databricks.test) doesn’t match the host configured in the bundle ([DATABRICKS_TARGET]). The profile "DEFAULT" has host="[DATABRICKS_TARGET]" that matches host in the bundle. To select it, pass "-p DEFAULT"
 
 
 Exit code: 1
@@ -49,7 +49,7 @@ Validation OK!
 
 === Bundle commands load bundle configuration with -p flag, validation not OK (profile host don't match bundle host)
 >>> errcode [CLI] bundle validate -p profile_name
-Error: cannot resolve bundle auth configuration: the host in the profile (https://non.existing.subdomain.databricks.com) doesn’t match the host configured in the bundle ([DATABRICKS_TARGET]). The profile "DEFAULT" has host="[DATABRICKS_TARGET]" that matches host in the bundle. To select it, pass "-p DEFAULT"
+Error: cannot resolve bundle auth configuration: the host in the profile (https://non.existing.subdomain.databricks.test) doesn’t match the host configured in the bundle ([DATABRICKS_TARGET]). The profile "DEFAULT" has host="[DATABRICKS_TARGET]" that matches host in the bundle. To select it, pass "-p DEFAULT"
 
 Name: test-auth
 Target: dev

acceptance/auth/bundle_and_profile/test.toml

@@ -10,5 +10,5 @@ Old='DATABRICKS_URL'
 New='DATABRICKS_TARGET'
 
 [[Repls]]
-Old='Get "https://non.existing.subdomain.databricks.com/api/2.0/preview/scim/v2/Me\??": .*'
-New='Get "https://non.existing.subdomain.databricks.com/api/2.0/preview/scim/v2/Me": (redacted)'
+Old='Get "https://non.existing.subdomain.databricks.test/api/2.0/preview/scim/v2/Me\??": .*'
+New='Get "https://non.existing.subdomain.databricks.test/api/2.0/preview/scim/v2/Me": (redacted)'

acceptance/bundle/deploy/mlops-stacks/out.test.toml

@@ -1,5 +1,5 @@
 Cloud=true
-Local=true
+Local=false
 
 Badness = "the newly initialized bundle from the 'mlops-stacks' template contains two validation warnings in the configuration"
 

acceptance/bundle/deploy/mlops-stacks/test.toml

@@ -7,4 +7,5 @@ resources:
       cluster_name: test-cluster-$UNIQUE_NAME
       spark_version: 13.3.x-scala2.12
       node_type_id: $NODE_TYPE_ID
+      instance_pool_id: $TEST_INSTANCE_POOL_ID
       num_workers: 1

acceptance/bundle/invariant/configs/cluster.yml.tmpl

@@ -12,4 +12,5 @@ resources:
           new_cluster:
             spark_version: $DEFAULT_SPARK_VERSION
             node_type_id: $NODE_TYPE_ID
+            instance_pool_id: $TEST_INSTANCE_POOL_ID
             num_workers: 1

acceptance/bundle/invariant/configs/job_with_task.yml.tmpl

@@ -10,6 +10,7 @@ resources:
       cluster_name: test-cluster-$UNIQUE_NAME
       spark_version: $DEFAULT_SPARK_VERSION
       node_type_id: $NODE_TYPE_ID
+      instance_pool_id: $TEST_INSTANCE_POOL_ID
       num_workers: 2
       spark_conf:
         "spark.executor.memory": "2g"

acceptance/bundle/resources/clusters/deploy/data_security_mode/databricks.yml.tmpl

@@ -10,6 +10,7 @@ resources:
       cluster_name: test-cluster-$UNIQUE_NAME
       spark_version: $DEFAULT_SPARK_VERSION
       node_type_id: $NODE_TYPE_ID
+      instance_pool_id: $TEST_INSTANCE_POOL_ID
       num_workers: 2
       spark_conf:
         "spark.executor.memory": "2g"