ci: add check for failed deployments (#6549)

Author: matthewelwell

.github/actions/api-deploy-ecs/action.yml

@@ -153,3 +153,28 @@ runs:
         aws ecs wait services-stable --cluster ${{ inputs.aws_ecs_cluster_name }} --services ${{ inputs.aws_ecs_service_name }}
         aws ecs wait services-stable --cluster ${{ inputs.aws_ecs_cluster_name }} --services ${{ inputs.aws_ecs_sdk_service_name }}
       shell: bash
+
+    - name: Verify correct version is running
+      shell: bash
+      run: |
+        EXPECTED_TASK_DEF=${{ steps.deploy-api-task-def.outputs.task-definition-arn }}
+        RUNNING_ADMIN_API_TASK_DEF=$(aws ecs describe-services \
+          --cluster ${{ inputs.aws_ecs_cluster_name }} \
+          --services ${{ inputs.aws_ecs_service_name }} \
+          --query 'services[0].deployments[?status==`PRIMARY`].taskDefinition' \
+          --output text)
+        RUNNING_SDK_API_TASK_DEF=$(aws ecs describe-services \
+          --cluster ${{ inputs.aws_ecs_cluster_name }} \
+          --services ${{ inputs.aws_ecs_sdk_service_name }} \
+          --query 'services[0].deployments[?status==`PRIMARY`].taskDefinition' \
+          --output text)
+        if [[ "$RUNNING_ADMIN_API_TASK_DEF" == "$EXPECTED_TASK_DEF" && \
+              "$RUNNING_SDK_API_TASK_DEF" == "$EXPECTED_TASK_DEF" ]]; then
+          exit 0
+        else
+          echo "Task definition mismatch detected"
+          echo "Expected: $EXPECTED_TASK_DEF"
+          echo "Admin API running: $RUNNING_ADMIN_API_TASK_DEF"
+          echo "SDK API running: $RUNNING_SDK_API_TASK_DEF"
+          exit 1
+        fi

.github/actions/task-processor-deploy-ecs/action.yml

@@ -41,6 +41,7 @@ runs:
         image: ${{ inputs.api_ecr_image_url }}
 
     - name: Deploy Amazon ECS Task Processor task definition
+      id: deploy-task-processor-task-def
       uses: aws-actions/amazon-ecs-deploy-task-definition@v2
       with:
         cluster: ${{ inputs.aws_ecs_cluster_name }}
@@ -52,3 +53,21 @@ runs:
         aws ecs wait services-stable --cluster ${{ inputs.aws_ecs_cluster_name }} --services ${{
         inputs.aws_ecs_service_name }}
       shell: bash
+
+    - name: Verify correct version is running
+      shell: bash
+      run: |
+        EXPECTED_TASK_DEF=${{ steps.deploy-task-processor-task-def.outputs.task-definition-arn }}
+        RUNNING_TASK_DEF=$(aws ecs describe-services \
+          --cluster ${{ inputs.aws_ecs_cluster_name }} \
+          --services ${{ inputs.aws_ecs_service_name }} \
+          --query 'services[0].deployments[?status==`PRIMARY`].taskDefinition' \
+          --output text)
+        if [[ "$RUNNING_TASK_DEF" == "$EXPECTED_TASK_DEF" ]]; then
+          exit 0
+        else
+          echo "Task definition mismatch detected"
+          echo "Expected: $EXPECTED_TASK_DEF"
+          echo "Task Processor running: $RUNNING_TASK_DEF"
+          exit 1
+        fi

.github/workflows/api-deploy-production-ecs.yml

@@ -9,6 +9,10 @@ on:
       - .github/**
       - infrastructure/aws/production/**
 
+concurrency:
+  group: ${{ github.workflow }}
+  cancel-in-progress: true
+
 jobs:
   deploy-ecs:
     uses: ./.github/workflows/.reusable-deploy-ecs.yml

.github/workflows/api-deploy-staging-ecs.yml

@@ -10,9 +10,13 @@ on:
       - infrastructure/aws/staging/**
   workflow_dispatch:
 
+concurrency:
+  group: ${{ github.workflow }}
+  cancel-in-progress: true
+
 jobs:
   deploy-ecs:
     uses: ./.github/workflows/.reusable-deploy-ecs.yml
     with:
       environment: staging
-    secrets: inherit
+    secrets: inherit