Fail smoke summary if any provider fails

Author: mneudert

.github/scripts/mcp-smoke/README.md

@@ -45,5 +45,6 @@ This harness powers `.github/workflows/mcp-ai-smoke.yml`.
 - Claude defaults to the generic `sonnet` alias, with `claude_model` still available in `workflow_dispatch` for pinned-model overrides.
 - Gemini, Codex, and Claude run in separate CI jobs with isolated Matomo environments and log files.
 - Provider jobs share a local composite action for the common Matomo/bootstrap sequence.
-- The workflow is intentionally report-first/non-blocking for internal prototype usage.
-- The workflow uploads one artifact per provider job and builds a combined summary from the downloaded per-provider `results.json` files.
+- The workflow is intentionally report-first: provider jobs continue far enough to upload artifacts and append per-provider summaries even when an individual smoke run fails.
+- The final workflow status is driven by explicit per-provider outcomes emitted by the workflow jobs, not by GitHub's masked `needs.<job>.result` values for `continue-on-error` jobs.
+- The workflow uploads one artifact per provider job, and detailed case-level diagnostics remain in each provider's `results.json` artifact bundle.

.github/workflows/mcp-ai-smoke.yml

@@ -112,6 +112,8 @@ jobs:
     needs: preflight
     if: needs.preflight.outputs.has_gemini == 'true'
     continue-on-error: true
+    outputs:
+      provider_outcome: ${{ steps.set_provider_outcome.outputs.provider_outcome }}
     timeout-minutes: 30
     services:
       mysql:
@@ -150,6 +152,8 @@ jobs:
         run: npm install -g @google/gemini-cli
 
       - name: Gemini MCP smoke suite
+        id: smoke_suite
+        continue-on-error: true
         run: ./.github/scripts/mcp-smoke/run-gemini-smoke.sh
         env:
           GEMINI_APIKEY: ${{ secrets.GEMINI_APIKEY }}
@@ -163,21 +167,37 @@ jobs:
           MATOMO_LOG_FILE: ${{ env.MATOMO_DIR }}/tmp/logs/matomo.log
           PHP_SERVER_LOG_FILE: ${{ env.MATOMO_DIR }}/tmp/logs/php-server.log
 
+      - name: Set Gemini provider outcome
+        id: set_provider_outcome
+        if: always()
+        run: |
+          provider_outcome="success"
+
+          if [ "${{ job.status }}" != "success" ] || [ "${{ steps.smoke_suite.outcome }}" = "failure" ]; then
+            provider_outcome="failure"
+          elif [ "${{ steps.smoke_suite.outcome }}" = "cancelled" ]; then
+            provider_outcome="cancelled"
+          fi
+
+          echo "provider_outcome=${provider_outcome}" >> "$GITHUB_OUTPUT"
+
       - name: Finalize Gemini smoke job
         if: always()
         uses: ./.github/actions/finalize-mcp-smoke-job
         with:
           provider_label: Gemini
           artifact_dir: ${{ github.workspace }}/.github/scripts/mcp-smoke/artifacts/gemini
           artifact_name: mcp-ai-smoke-gemini
-          job_status: ${{ job.status }}
+          job_status: ${{ steps.set_provider_outcome.outputs.provider_outcome }}
 
   smoke_codex:
     name: Codex smoke
     runs-on: ubuntu-24.04
     needs: preflight
     if: needs.preflight.outputs.has_codex == 'true'
     continue-on-error: true
+    outputs:
+      provider_outcome: ${{ steps.set_provider_outcome.outputs.provider_outcome }}
     timeout-minutes: 30
     services:
       mysql:
@@ -216,6 +236,8 @@ jobs:
         run: npm install -g @openai/codex
 
       - name: Codex MCP smoke suite
+        id: smoke_suite
+        continue-on-error: true
         run: ./.github/scripts/mcp-smoke/run-codex-smoke.sh
         env:
           OPENAI_APIKEY: ${{ secrets.OPENAI_APIKEY }}
@@ -228,21 +250,37 @@ jobs:
           MATOMO_LOG_FILE: ${{ env.MATOMO_DIR }}/tmp/logs/matomo.log
           PHP_SERVER_LOG_FILE: ${{ env.MATOMO_DIR }}/tmp/logs/php-server.log
 
+      - name: Set Codex provider outcome
+        id: set_provider_outcome
+        if: always()
+        run: |
+          provider_outcome="success"
+
+          if [ "${{ job.status }}" != "success" ] || [ "${{ steps.smoke_suite.outcome }}" = "failure" ]; then
+            provider_outcome="failure"
+          elif [ "${{ steps.smoke_suite.outcome }}" = "cancelled" ]; then
+            provider_outcome="cancelled"
+          fi
+
+          echo "provider_outcome=${provider_outcome}" >> "$GITHUB_OUTPUT"
+
       - name: Finalize Codex smoke job
         if: always()
         uses: ./.github/actions/finalize-mcp-smoke-job
         with:
           provider_label: Codex
           artifact_dir: ${{ github.workspace }}/.github/scripts/mcp-smoke/artifacts/codex
           artifact_name: mcp-ai-smoke-codex
-          job_status: ${{ job.status }}
+          job_status: ${{ steps.set_provider_outcome.outputs.provider_outcome }}
 
   smoke_claude:
     name: Claude smoke
     runs-on: ubuntu-24.04
     needs: preflight
     if: needs.preflight.outputs.has_claude == 'true'
     continue-on-error: true
+    outputs:
+      provider_outcome: ${{ steps.set_provider_outcome.outputs.provider_outcome }}
     timeout-minutes: 30
     services:
       mysql:
@@ -281,6 +319,8 @@ jobs:
         run: npm install -g @anthropic-ai/claude-code
 
       - name: Claude MCP smoke suite
+        id: smoke_suite
+        continue-on-error: true
         run: ./.github/scripts/mcp-smoke/run-claude-smoke.sh
         env:
           ANTHROPIC_API_KEY: ${{ secrets.CLAUDE_APIKEY }}
@@ -293,20 +333,34 @@ jobs:
           MATOMO_LOG_FILE: ${{ env.MATOMO_DIR }}/tmp/logs/matomo.log
           PHP_SERVER_LOG_FILE: ${{ env.MATOMO_DIR }}/tmp/logs/php-server.log
 
+      - name: Set Claude provider outcome
+        id: set_provider_outcome
+        if: always()
+        run: |
+          provider_outcome="success"
+
+          if [ "${{ job.status }}" != "success" ] || [ "${{ steps.smoke_suite.outcome }}" = "failure" ]; then
+            provider_outcome="failure"
+          elif [ "${{ steps.smoke_suite.outcome }}" = "cancelled" ]; then
+            provider_outcome="cancelled"
+          fi
+
+          echo "provider_outcome=${provider_outcome}" >> "$GITHUB_OUTPUT"
+
       - name: Finalize Claude smoke job
         if: always()
         uses: ./.github/actions/finalize-mcp-smoke-job
         with:
           provider_label: Claude
           artifact_dir: ${{ github.workspace }}/.github/scripts/mcp-smoke/artifacts/claude
           artifact_name: mcp-ai-smoke-claude
-          job_status: ${{ job.status }}
+          job_status: ${{ steps.set_provider_outcome.outputs.provider_outcome }}
 
   summary:
     name: Final status
     runs-on: ubuntu-24.04
     needs: [preflight, smoke_gemini, smoke_codex, smoke_claude]
-    if: always()
+    if: needs.preflight.outputs.should_run == 'true'
     steps:
       - name: Final status summary
         run: |
@@ -318,9 +372,42 @@ jobs:
             echo "- Codex available: ${{ needs.preflight.outputs.has_codex }}"
             echo "- Claude available: ${{ needs.preflight.outputs.has_claude }}"
             echo "- reason: ${{ needs.preflight.outputs.reason }}"
-            echo "- Gemini job result: ${{ needs.smoke_gemini.result || 'skipped' }}"
-            echo "- Codex job result: ${{ needs.smoke_codex.result || 'skipped' }}"
-            echo "- Claude job result: ${{ needs.smoke_claude.result || 'skipped' }}"
+            echo "- Gemini provider outcome: ${{ needs.preflight.outputs.has_gemini == 'true' && needs.smoke_gemini.outputs.provider_outcome || 'skipped' }}"
+            echo "- Codex provider outcome: ${{ needs.preflight.outputs.has_codex == 'true' && needs.smoke_codex.outputs.provider_outcome || 'skipped' }}"
+            echo "- Claude provider outcome: ${{ needs.preflight.outputs.has_claude == 'true' && needs.smoke_claude.outputs.provider_outcome || 'skipped' }}"
             echo
-            echo "Prototype smoke is intentionally non-blocking."
+            echo "Detailed per-case results remain in the individual provider jobs and uploaded artifacts."
           } >> "$GITHUB_STEP_SUMMARY"
+
+      - name: Evaluate overall result
+        run: |
+          set -euo pipefail
+
+          gemini_result="${{ needs.preflight.outputs.has_gemini == 'true' && needs.smoke_gemini.outputs.provider_outcome || 'skipped' }}"
+          codex_result="${{ needs.preflight.outputs.has_codex == 'true' && needs.smoke_codex.outputs.provider_outcome || 'skipped' }}"
+          claude_result="${{ needs.preflight.outputs.has_claude == 'true' && needs.smoke_claude.outputs.provider_outcome || 'skipped' }}"
+
+          ran_providers=0
+          failed_providers=0
+
+          for result in "$gemini_result" "$codex_result" "$claude_result"; do
+            if [ "$result" != "skipped" ]; then
+              ran_providers=$((ran_providers + 1))
+            fi
+
+            if [ "$result" != "success" ] && [ "$result" != "skipped" ]; then
+              failed_providers=$((failed_providers + 1))
+            fi
+          done
+
+          if [ "$ran_providers" -eq 0 ]; then
+            echo "No provider jobs ran."
+            exit 0
+          fi
+
+          if [ "$failed_providers" -gt 0 ]; then
+            echo "$failed_providers provider job(s) failed."
+            exit 1
+          fi
+
+          echo "All provider jobs succeeded."