acceptance: make contains.py fail the test on a failed assertion (#5466)

`contains.py` (#3345) printed `contains error:` to stderr but exited 0,
so a failed assertion got baked into the golden output instead of
failing the test. It now exits non-zero, so under the harness's `bash
-euo pipefail` the script aborts. This surfaced a stale assertion in
`bundle/state/state_present` (serial asserted as `11`/`12`, but it's
been `13` since the test was added); `selftest/contains` wraps its
failing cases in `errcode` so the script still runs to completion.

This pull request and its description were written by Isaac.

Author: pietern

acceptance/bin/contains.py

@@ -10,6 +10,7 @@
         must_find.append(arg)
 
 found = set()
+failed = False
 
 for line in sys.stdin:
     sys.stdout.write(line)
@@ -19,9 +20,16 @@
     for t in must_not_find:
         if t in line:
             sys.stderr.write(f"contains error: {t!r} was not expected: {line.strip()!r}\n")
+            failed = True
 
 sys.stdout.flush()
 
 not_found = set(must_find) - found
 for item in sorted(not_found):
     sys.stderr.write(f"contains error: {item!r} not found in the output.\n")
+    failed = True
+
+# Exit non-zero so a failed assertion aborts the script (set -e -o pipefail)
+# instead of silently baking the error line into the expected output.
+if failed:
+    sys.exit(1)

acceptance/bundle/state/state_present/output.txt

@@ -71,7 +71,6 @@ Deployment complete!
 >>> print_state.py
 3
 13
-contains error: '11' not found in the output.
 
 >>> DATABRICKS_BUNDLE_ENGINE=terraform [CLI] bundle deploy
 Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files...
@@ -92,7 +91,6 @@ Deployment complete!
 >>> print_state.py
 3
 13
-contains error: '12' not found in the output.
 
 >>> DATABRICKS_BUNDLE_ENGINE= [CLI] bundle debug states
 [TEST_TMP_DIR]/.databricks/bundle/default/terraform/terraform.tfstate: local terraform state serial=3 lineage="test-lineage"

acceptance/bundle/state/state_present/script

@@ -29,14 +29,14 @@ trace DATABRICKS_BUNDLE_ENGINE=terraform $CLI bundle destroy --auto-approve
 rm out.requests.txt
 trace DATABRICKS_BUNDLE_ENGINE=direct $CLI bundle deploy
 trace print_requests.py //api/2.1/unity-catalog/schemas | jq '.headers["User-Agent"][0]' | contains.py 'engine/direct' '!terraform' '!tf-provider'
-trace print_state.py | jq .serial | contains.py "11"
+trace print_state.py | jq .serial | contains.py "13"
 
 trace DATABRICKS_BUNDLE_ENGINE=terraform $CLI bundle deploy
 #trace print_requests.py --get //api/2.1/unity-catalog/schemas | jq '.headers["User-Agent"][0]' | contains.py 'engine/direct' '!terraform' '!tf-provider'
 
 trace DATABRICKS_BUNDLE_ENGINE= $CLI bundle deploy
 trace print_requests.py --get //api/2.1/unity-catalog/schemas | jq '.headers["User-Agent"][0]' | contains.py 'engine/direct' '!terraform' '!tf-provider'
-trace print_state.py | jq .serial | contains.py "12"
+trace print_state.py | jq .serial | contains.py "13"
 
 trace DATABRICKS_BUNDLE_ENGINE= $CLI bundle debug states
 trace DATABRICKS_BUNDLE_ENGINE= $CLI bundle debug states --force-pull

acceptance/selftest/contains/output.txt

@@ -4,11 +4,15 @@
 Hello world
 contains error: 'not_found' not found in the output.
 
+Exit code: 1
+
 === This should complain about not_found not found
 >>> musterr python3 ./failure.py
 Failed script
 contains error: 'not_found' not found in the output.
 
+Exit code: 1
+
 === This should not complain
 >>> python3 ./success.py
 Hello world
@@ -22,6 +26,8 @@ Failed script
 Hello world
 contains error: 'Hello' was not expected: 'Hello world'
 
+Exit code: 1
+
 === This should not complain
 >>> python3 ./success.py
 Hello world

acceptance/selftest/contains/script

@@ -1,8 +1,8 @@
 title "This should complain about not_found not found"
-trace python3 ./success.py | contains.py not_found
+trace python3 ./success.py | errcode contains.py not_found
 
 title "This should complain about not_found not found"
-trace musterr python3 ./failure.py | contains.py not_found
+trace musterr python3 ./failure.py | errcode contains.py not_found
 
 title "This should not complain"
 trace python3 ./success.py | contains.py world Hello
@@ -11,7 +11,7 @@ title "This should not complain"
 trace musterr python3 ./failure.py | contains.py Failed
 
 title "This should complain about Hello present in output"
-trace python3 ./success.py | contains.py !Hello
+trace python3 ./success.py | errcode contains.py !Hello
 
 title "This should not complain"
 trace python3 ./success.py | contains.py !not_found