Replace old Spark Jar integration tests with an acceptance test (#5191)

## Changes
Replace old Spark Jar integration tests with an acceptance test

## Why
We don't need to test the matric of DBR / Java / Spark versions anymore
and instead can just rely on an acceptance test for LTS DBR

## Tests
Added an acceptance test which pass on both Cloud and Local

<!-- If your PR needs to be included in the release notes for next
release,
add a separate entry in NEXT_CHANGELOG.md as part of your PR. -->

Author: andrewnester

acceptance/bundle/deploy/spark-jar-task/databricks.yml.tmpl

@@ -0,0 +1,26 @@
+bundle:
+  name: spark-jar-task-$UNIQUE_NAME
+
+artifacts:
+  my_java_code:
+    path: ./myjar
+    build: "javac PrintArgs.java && jar cvfm PrintArgs.jar META-INF/MANIFEST.MF PrintArgs.class"
+    files:
+      - source: ./myjar/PrintArgs.jar
+
+resources:
+  jobs:
+    jar_job:
+      name: "[${bundle.target}] Test Spark Jar Job $UNIQUE_NAME"
+      tasks:
+        - task_key: TestSparkJarTask
+          new_cluster:
+            num_workers: 1
+            spark_version: 16.4.x-scala2.12
+            node_type_id: $NODE_TYPE_ID
+            instance_pool_id: $TEST_INSTANCE_POOL_ID
+            data_security_mode: NONE
+          spark_jar_task:
+            main_class_name: PrintArgs
+          libraries:
+            - jar: ./myjar/PrintArgs.jar

…e/{{.project_name}}/META-INF/MANIFEST.MF …park-jar-task/myjar/META-INF/MANIFEST.MFintegration/bundle/bundles/spark_jar_task/template/{{.project_name}}/META-INF/MANIFEST.MF renamed to acceptance/bundle/deploy/spark-jar-task/myjar/META-INF/MANIFEST.MF integration/bundle/bundles/spark_jar_task/template/{{.project_name}}/META-INF/MANIFEST.MF renamed to acceptance/bundle/deploy/spark-jar-task/myjar/META-INF/MANIFEST.MF

@@ -0,0 +1,24 @@
+
+>>> [CLI] bundle deploy
+Building my_java_code...
+Uploading myjar/PrintArgs.jar...
+Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/spark-jar-task-[UNIQUE_NAME]/default/files...
+Deploying resources...
+Updating deployment state...
+Deployment complete!
+
+>>> [CLI] bundle run jar_job
+Run URL: [DATABRICKS_URL]/?o=[NUMID]#job/[NUMID]/run/[NUMID]
+
+[TIMESTAMP] "[default] Test Spark Jar Job [UNIQUE_NAME]" RUNNING
+[TIMESTAMP] "[default] Test Spark Jar Job [UNIQUE_NAME]" TERMINATED SUCCESS
+Hello from Jar!
+[]
+>>> [CLI] bundle destroy --auto-approve
+The following resources will be deleted:
+  delete resources.jobs.jar_job
+
+All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/spark-jar-task-[UNIQUE_NAME]/default
+
+Deleting files...
+Destroy complete!

…emplate/{{.project_name}}/PrintArgs.java …ploy/spark-jar-task/myjar/PrintArgs.javaintegration/bundle/bundles/spark_jar_task/template/{{.project_name}}/PrintArgs.java renamed to acceptance/bundle/deploy/spark-jar-task/myjar/PrintArgs.java integration/bundle/bundles/spark_jar_task/template/{{.project_name}}/PrintArgs.java renamed to acceptance/bundle/deploy/spark-jar-task/myjar/PrintArgs.java

@@ -0,0 +1,4 @@
+envsubst < databricks.yml.tmpl > databricks.yml
+trap "errcode trace '$CLI' bundle destroy --auto-approve" EXIT
+trace $CLI bundle deploy
+trace $CLI bundle run jar_job

acceptance/bundle/deploy/spark-jar-task/out.test.toml

@@ -0,0 +1,17 @@
+Local = true
+Cloud = true
+EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"]
+
+Ignore = [
+    'myjar/PrintArgs.jar',
+    'myjar/PrintArgs.class',
+]
+
+[[Server]]
+Pattern = "GET /api/2.2/jobs/runs/get-output"
+Response.Body = '''
+{
+	"run_id": 1234567890,
+	"logs": "Hello from Jar!\n[]"
+}
+'''

acceptance/bundle/deploy/spark-jar-task/output.txt

@@ -67,13 +67,6 @@ func deployBundle(t testutil.TestingT, ctx context.Context, path string) {
 	require.NoError(t, err)
 }
 
-func runResource(t testutil.TestingT, ctx context.Context, path, key string) (string, error) {
-	ctx = env.Set(ctx, "BUNDLE_ROOT", path)
-	c := testcli.NewRunner(t, ctx, "bundle", "run", key)
-	stdout, _, err := c.Run()
-	return stdout.String(), err
-}
-
 func destroyBundle(t testutil.TestingT, ctx context.Context, path string) {
 	ctx = env.Set(ctx, "BUNDLE_ROOT", path)
 	c := testcli.NewRunner(t, ctx, "bundle", "destroy", "--auto-approve")