Don't cache the tests that are part of the e2e testsuite. (#1185)

* Don't cache the tests that are part of the e2e testsuite.

This prevents spurious errors when running the tests repeatedly.

* add a small comment on what '-count=1' means and why we need it.

Author: metlos

make/sandbox-ui.mk

@@ -112,11 +112,12 @@ e2e-run-sandbox-ui:
 	$(GOPATH)/bin/playwright install firefox
 
 	@echo "Running Developer Sandbox UI setup e2e tests..."
-	SANDBOX_UI_NS=${SANDBOX_UI_NS} go test "./test/e2e/sandbox-ui/setup" -v -timeout=10m -failfast
+	SANDBOX_UI_NS=${SANDBOX_UI_NS} go test "./test/e2e/sandbox-ui/setup" -v -timeout=10m -failfast -count=1
 
 	@echo "Running Developer Sandbox UI e2e tests in firefox..."
 	SSO_USERNAME=${SSO_USERNAME} SSO_PASSWORD=${SSO_PASSWORD} BASE_URL=${RHDH} BROWSER=firefox envsubst < deploy/sandbox-ui/ui-e2e-tests/.env > testsupport/sandbox-ui/.env
-	go test "./test/e2e/sandbox-ui" -v -timeout=10m -failfast
+	# NOTE: The "-count=1" is the idiomatic way of turning off the test result cache according to https://pkg.go.dev/cmd/go#hdr-Testing_flags.
+	go test "./test/e2e/sandbox-ui" -v -timeout=10m -failfast -count=1
 	@oc delete usersignup ${SSO_USERNAME} -n ${HOST_NS}
 
 	@echo "The Developer Sandbox UI e2e tests successfully finished"
@@ -159,4 +160,4 @@ test-sandbox-ui-in-container: build-sandbox-ui-e2e-tests
 	  -e TMP=/tmp/ \
 	  -e PUSH_SANDBOX_IMAGE=false \
 	  -e RUNNING_IN_CONTAINER=true \
-	  $(UNIT_TEST_IMAGE_NAME)
+	  $(UNIT_TEST_IMAGE_NAME)

make/test.mk

@@ -164,7 +164,11 @@ execute-tests:
 	@echo "Status of ToolchainStatus"
 	-oc get ToolchainStatus -n ${HOST_NS} -o yaml
 	@echo "Starting test $(shell date)"
-	MEMBER_NS=${MEMBER_NS} MEMBER_NS_2=${MEMBER_NS_2} HOST_NS=${HOST_NS} REGISTRATION_SERVICE_NS=${REGISTRATION_SERVICE_NS} SECOND_MEMBER_MODE=${SECOND_MEMBER_MODE} go test ${TESTS_TO_EXECUTE} -run ${TESTS_RUN_FILTER_REGEXP} -p 1 -v -timeout=90m -failfast || \
+	# NOTE: The "-count=1" is the idiomatic way of turning off the test result cache according to https://pkg.go.dev/cmd/go#hdr-Testing_flags.
+	# One might wonder whether the word "idiomatic" shouldn't have been spelled with 2 letters less there.
+	# We need to turn off the cache because the e2e tests depend on running the migration setup. If the results of the migration tests were
+	# cached, it might happen that the cluster is in an unprepared state when the e2e tests start running.
+	MEMBER_NS=${MEMBER_NS} MEMBER_NS_2=${MEMBER_NS_2} HOST_NS=${HOST_NS} REGISTRATION_SERVICE_NS=${REGISTRATION_SERVICE_NS} SECOND_MEMBER_MODE=${SECOND_MEMBER_MODE} go test ${TESTS_TO_EXECUTE} -run ${TESTS_RUN_FILTER_REGEXP} -p 1 -v -timeout=90m -failfast -count=1 || \
 	($(MAKE) print-logs HOST_NS=${HOST_NS} MEMBER_NS=${MEMBER_NS} MEMBER_NS_2=${MEMBER_NS_2} REGISTRATION_SERVICE_NS=${REGISTRATION_SERVICE_NS} && exit 1)
 
 .PHONY: print-logs

test/e2e/sandbox-ui/README.md

@@ -1,17 +1,19 @@
 # Developer Sandbox UI E2E Tests
+
 The UI E2E tests are executed against the Developer Sandbox UI running in OpenShift.
 
 *Prerequisites*:
 
-1. You need a OCP cluster 
+1. You need a OCP cluster
     - ROSA cluster from ClusterBot will not work since we are not able to modify the OAuth configuration of ROSA clusters created by the ClusterBot.
 2. Ensure you are using Node.js version 22
     - to easily manage it, you can run `nvm use 22`
 3. Ensure you have `yarn` installed
-4. Make sure you can log in at https://sso.devsandbox.dev/auth/realms/sandbox-dev/account using your SSO_USERNAME and SSO_PASSWORD
+4. Make sure you can log in at <https://sso.devsandbox.dev/auth/realms/sandbox-dev/account> using your SSO_USERNAME and SSO_PASSWORD
 5. Make sure you do not have any toolchain resources deployed on your cluster
 
 ### Running UI E2E Tests locally
+
 `make test-ui-e2e SSO_USERNAME=${SSO_USERNAME} SSO_PASSWORD=${SSO_PASSWORD}`
 
 If you want to run and test the Developer Sandbox UI from your local rhdh-plugins repo, run `make test-ui-e2e-local SSO_USERNAME=${SSO_USERNAME} SSO_PASSWORD=${SSO_PASSWORD}`
@@ -23,25 +25,30 @@ For now, the UI E2E tests are running only through the Firefox browser.
 `make test-sandbox-ui-in-container SSO_USERNAME=<SSO_USERNAME> SSO_PASSWORD=<SSO_PASSWORD>`
 
 ### Running UI E2E Tests against dev/stage/prod
+
 If you want to run the UI E2E tests against dev/stage/prod, please follow the next steps:
+
 - have a new Red Hat account or an account that was deactivated
 - fill `testsupport/sandbox-ui/.env`, note that to run the UI E2E tests against dev/stage/prod, you need to set the ENVIRONMENT to dev.
+
 ```
 SSO_USERNAME=<your-username>
 SSO_PASSWORD=<your-password>
 BASE_URL=https://sandbox.redhat.com/
 ENVIRONMENT=dev
 BROWSER=firefox
 ```
-- run `go test "./test/e2e/sandbox-ui" -v -timeout=10m -failfast`
+
+- run `go test "./test/e2e/sandbox-ui" -v -timeout=10m -failfast -count=1`
 
 `make test-sandbox-ui-in-container SSO_USERNAME=<SSO_USERNAME> SSO_PASSWORD=<SSO_PASSWORD>`
 
 ### Deploy Developer Sandbox UI in E2E mode
+
 `make deploy-sandbox-ui HOST_NS=<HOST_NS>`
 
 Please note that OCP cluster does not have a valid CA, so when accessing the Developer Sandbox UI, you need to:
- 
+
 - accept to proceed unsafely
 
 ![private-connection](https://github.com/user-attachments/assets/5b35a65f-6703-42cf-a165-b7326fd4faab)
@@ -51,4 +58,6 @@ Please note that OCP cluster does not have a valid CA, so when accessing the Dev
 ![registration-service](https://github.com/user-attachments/assets/6c2f7446-1de2-4701-ace7-2d6796f49eeb)
 
 ### Clean Developer Sandbox UI
-`make clean-sandbox-ui HOST_NS=<HOST_NS> SSO_USERNAME=<SSO_USERNAME>`
+
+`make clean-sandbox-ui HOST_NS=<HOST_NS> SSO_USERNAME=<SSO_USERNAME>`
+