From d94c949a5df107a06acad8f52d63cb79e040609c Mon Sep 17 00:00:00 2001
From: Coatl <121911012+2-Coatl@users.noreply.github.com>
Date: Thu, 13 Nov 2025 03:02:30 -0600
Subject: [PATCH] fix(ci): improve run-all-check resilience

---
 scripts/ci/infrastructure/health-check.sh | 10 +++++--
 scripts/ci/run-all-checks.sh              | 24 ++++++++++++++---
 scripts/tests/test_ci_shell_scripts.py    | 32 +++++++++++++++++++++++
 3 files changed, 61 insertions(+), 5 deletions(-)
 create mode 100644 scripts/tests/test_ci_shell_scripts.py

diff --git a/scripts/ci/infrastructure/health-check.sh b/scripts/ci/infrastructure/health-check.sh
index b7a310f4..0d4bf4a2 100755
--- a/scripts/ci/infrastructure/health-check.sh
+++ b/scripts/ci/infrastructure/health-check.sh
@@ -53,11 +53,14 @@ log_info "Checking database connectivity..."
 cd "$PROJECT_ROOT/api/callcentersite"
 
 if [ -f "manage.py" ]; then
-    if python3 manage.py check --database default &> /dev/null; then
+    if DB_CHECK_OUTPUT=$(python3 manage.py check --database default 2>&1); then
         log_info "Database connectivity: OK"
         CHECKS_PASSED=$((CHECKS_PASSED + 1))
     else
         log_error "Database connectivity failed"
+        echo "$DB_CHECK_OUTPUT" | tail -n 20 | while IFS= read -r line; do
+            log_error "    $line"
+        done
         CHECKS_FAILED=$((CHECKS_FAILED + 1))
     fi
 else
@@ -67,11 +70,14 @@ fi
 
 # Check 4: Django configuration
 log_info "Checking Django configuration..."
-if python3 manage.py check &> /dev/null; then
+if DJANGO_CHECK_OUTPUT=$(python3 manage.py check 2>&1); then
     log_info "Django configuration: OK"
     CHECKS_PASSED=$((CHECKS_PASSED + 1))
 else
     log_error "Django configuration check failed"
+    echo "$DJANGO_CHECK_OUTPUT" | tail -n 20 | while IFS= read -r line; do
+        log_error "    $line"
+    done
     CHECKS_FAILED=$((CHECKS_FAILED + 1))
 fi
 
diff --git a/scripts/ci/run-all-checks.sh b/scripts/ci/run-all-checks.sh
index 1089e6ab..dd0e3ae5 100755
--- a/scripts/ci/run-all-checks.sh
+++ b/scripts/ci/run-all-checks.sh
@@ -11,7 +11,8 @@
 #   0 - All checks passed
 #   1 - One or more checks failed
 
-set -e
+set -u
+set -o pipefail
 
 RED='\033[0;31m'
 GREEN='\033[0;32m'
@@ -77,13 +78,30 @@ run_check() {
 
     chmod +x "$check_script"
 
+    local exit_code
+    local output=""
+    local previous_errexit=0
+
+    if [[ $- == *e* ]]; then
+        previous_errexit=1
+        set +e
+    fi
+
     if [ "$VERBOSE" = true ]; then
         "$check_script"
+        exit_code=$?
     else
-        "$check_script" > /dev/null 2>&1
+        output=$("$check_script" 2>&1)
+        exit_code=$?
     fi
 
-    local exit_code=$?
+    if [ $previous_errexit -eq 1 ]; then
+        set -e
+    fi
+
+    if [ "$VERBOSE" != true ] && [ $exit_code -ne 0 ] && [ -n "$output" ]; then
+        echo "$output"
+    fi
 
     if [ $exit_code -eq 0 ]; then
         log_info "$check_name: PASS"
diff --git a/scripts/tests/test_ci_shell_scripts.py b/scripts/tests/test_ci_shell_scripts.py
new file mode 100644
index 00000000..14d966d1
--- /dev/null
+++ b/scripts/tests/test_ci_shell_scripts.py
@@ -0,0 +1,32 @@
+import subprocess
+from pathlib import Path
+
+REPO_ROOT = Path(__file__).resolve().parents[2]
+RUN_ALL_CHECKS = REPO_ROOT / "scripts" / "ci" / "run-all-checks.sh"
+HEALTH_CHECK = REPO_ROOT / "scripts" / "ci" / "infrastructure" / "health-check.sh"
+
+
+def _run_script(script_path, *args):
+    return subprocess.run(
+        ["bash", str(script_path), *args],
+        cwd=REPO_ROOT,
+        stdout=subprocess.PIPE,
+        stderr=subprocess.PIPE,
+        text=True,
+    )
+
+
+def test_run_all_checks_reports_summary_even_on_failure():
+    result = _run_script(RUN_ALL_CHECKS)
+
+    assert result.returncode != 0, "Expected the aggregated checks to fail in the default dev environment"
+    combined_output = f"{result.stdout}\\n{result.stderr}"
+    assert "FINAL CI/CD REPORT" in combined_output
+
+
+def test_health_check_surfaces_underlying_error_details():
+    result = _run_script(HEALTH_CHECK)
+
+    assert result.returncode != 0, "The health check should fail when dependencies are missing"
+    combined_output = f"{result.stdout}\\n{result.stderr}"
+    assert "ModuleNotFoundError" in combined_output or "No module named" in combined_output