fix: have org-manager-user create space to retain SpaceDeveloper role

Root cause: org-manager-user lost SpaceDeveloper between the deploy step
and test execution (~50 min gap). Without it, cf curl POST to create tasks
returns an authorization error, failing all acceptance test suites.

Previously admin created the space and explicitly granted roles to
org-manager-user — but those roles could be lost if anything recreated
the space between deploy and test execution.

Fix:
- Have org-manager-user create the space (creator auto-gets
  SpaceManager+SpaceDeveloper, which persists with the space)
- Admin only needs roles for cleanup operations
- Test runner re-ensures SpaceDeveloper as a safety net before task launch
- Added diagnostic logging: app state, droplet, and full CF API response
  on task creation failure (no more silent errors)

Author: bonzofenix

.github/workflows/acceptance_tests_reusable.yaml

@@ -79,16 +79,28 @@ jobs:
         run: |
           make --directory="${AUTOSCALER_DIR}" deploy-cleanup || echo "Cleanup warnings expected if nothing exists"
 
-      - name: Grant space roles after space recreation
+      - name: Create space as org-manager-user (ensures inherent SpaceDeveloper role)
         shell: bash
         run: |
           source "${AUTOSCALER_DIR}/scripts/vars.source.sh"
           source "${AUTOSCALER_DIR}/scripts/common.sh"
           bbl_login
+
+          # Admin ensures org exists and grants OrgManager (needed for space creation)
+          cf_admin_login
+          find_or_create_org "${AUTOSCALER_ORG}"
+
+          # org-manager-user creates the space — creator auto-gets SpaceManager+SpaceDeveloper
+          cf api "https://api.${system_domain}" --skip-ssl-validation
+          cf auth "${AUTOSCALER_ORG_MANAGER_USER}" "${AUTOSCALER_ORG_MANAGER_PASSWORD}"
+          cf target -o "${AUTOSCALER_ORG}"
+          cf create-space "${AUTOSCALER_SPACE}" || echo "Space already exists"
+          cf target -s "${AUTOSCALER_SPACE}"
+
+          # Grant admin SpaceDeveloper too (needed for cleanup operations)
           cf_admin_login
-          cf_target "${AUTOSCALER_ORG}" "${AUTOSCALER_SPACE}"
-          cf set-space-role "${AUTOSCALER_ORG_MANAGER_USER}" "${AUTOSCALER_ORG}" "${AUTOSCALER_SPACE}" SpaceDeveloper
-          cf set-space-role "${AUTOSCALER_ORG_MANAGER_USER}" "${AUTOSCALER_ORG}" "${AUTOSCALER_SPACE}" SpaceManager
+          cf set-space-role admin "${AUTOSCALER_ORG}" "${AUTOSCALER_SPACE}" SpaceDeveloper
+          cf set-space-role admin "${AUTOSCALER_ORG}" "${AUTOSCALER_SPACE}" SpaceManager
 
       - name: Provision DB
         shell: bash

scripts/run-mta-acceptance-tests.sh

@@ -44,6 +44,19 @@ validate_app() {
 	[[ -n "${APP_GUID}" ]] || { echo "ERROR: acceptance-tests app not found. Run: make mta-deploy"; exit 1; }
 
 	echo "  ✓ App found (GUID: ${APP_GUID:0:8})"
+
+	# Diagnostic: check app state and droplet
+	local app_state
+	app_state=$(cf curl "/v3/apps/${APP_GUID}" 2>/dev/null | jq -r '.state // "UNKNOWN"')
+	echo "  App state: ${app_state}"
+
+	local droplet
+	droplet=$(cf curl "/v3/apps/${APP_GUID}/droplets/current" 2>/dev/null | jq -r '.guid // empty')
+	if [[ -n "${droplet}" ]]; then
+		echo "  ✓ Current droplet: ${droplet:0:8}"
+	else
+		echo "  ✗ WARNING: No current droplet assigned"
+	fi
 }
 
 launch_task() {
@@ -55,13 +68,18 @@ launch_task() {
 
 	local guid
 	local cf_output
-	cf_output=$(cf curl "/v3/apps/${APP_GUID}/tasks" -X POST -d "$(jq -n --arg n "${task_name}" --arg c "${cmd}" \
-		'{name:$n,command:$c,memory_in_mb:2048,disk_in_mb:2048}')" 2>&1)
-	guid=$(echo "$cf_output" | jq -r '.guid // empty')
+	local payload
+	payload=$(jq -n --arg n "${task_name}" --arg c "${cmd}" \
+		'{name:$n,command:$c,memory_in_mb:2048,disk_in_mb:2048}')
+	cf_output=$(cf curl "/v3/apps/${APP_GUID}/tasks" -X POST -d "${payload}" 2>&1)
+	guid=$(echo "$cf_output" | jq -r '.guid // empty' 2>/dev/null)
 	if [[ -n "${guid}" ]]; then
 		echo "${guid}"
 	else
 		echo "ERROR: Failed to launch task for ${suite}" >&2
+		echo "  API response: ${cf_output}" >&2
+		echo "  APP_GUID: ${APP_GUID}" >&2
+		echo "  Payload: ${payload}" >&2
 		echo "FAILED"
 	fi
 }
@@ -176,6 +194,18 @@ main() {
 	bbl_login
 	cf_deployment_login
 	cf_target "${autoscaler_org}" "${autoscaler_space}"
+
+	# On PR branches, ensure org-manager-user has SpaceDeveloper (may have been lost
+	# between deploy and test execution if another workflow recreated the space).
+	if [[ "${PR_NUMBER:-main}" != "main" ]]; then
+		step "Ensuring space roles for ${AUTOSCALER_ORG_MANAGER_USER}"
+		cf_admin_login
+		cf set-space-role "${AUTOSCALER_ORG_MANAGER_USER}" "${autoscaler_org}" "${autoscaler_space}" SpaceDeveloper 2>/dev/null || true
+		# Switch back to org-manager-user for task operations
+		cf_deployment_login
+		cf target -o "${autoscaler_org}" -s "${autoscaler_space}"
+	fi
+
 	validate_app
 
 	# Launch tasks