chore(ci): reduce number of parallel jobs and improve container builds (RocketChat#37466)

Author: rodrigok

.github/actions/build-docker/action.yml

@@ -1,14 +1,10 @@
-name: 'Meteor Docker'
+name: 'Build Docker'
 
 inputs:
   CR_USER:
     required: true
   CR_PAT:
     required: true
-  node-version:
-    required: true
-    description: 'Node version'
-    type: string
   deno-version:
     required: true
     description: 'Deno version'
@@ -25,13 +21,10 @@ inputs:
     required: false
     description: 'Publish image'
     default: 'true'
-  setup:
-    required: false
-    description: 'Setup node.js'
-    default: 'true'
-  NPM_TOKEN:
+  setup-docker:
     required: false
-    description: 'NPM token'
+    description: 'Setup Docker'
+    default: true
   type:
     required: false
     description: 'production or coverage'
@@ -49,17 +42,6 @@ runs:
         username: ${{ inputs.CR_USER }}
         password: ${{ inputs.CR_PAT }}
 
-    - name: Restore packages build
-      uses: actions/download-artifact@v6
-      with:
-        name: packages-build
-        path: /tmp
-
-    - name: Unpack packages build
-      shell: bash
-      run: |
-        tar -xzf /tmp/RocketChat-packages-build.tar.gz -C .
-
     - name: Restore meteor build
       if: inputs.service == 'rocketchat'
       uses: actions/download-artifact@v6
@@ -76,20 +58,17 @@ runs:
         rm Rocket.Chat.tar.gz
 
     - name: Set up Docker
+      if: inputs.setup-docker == true
       uses: docker/setup-docker-action@v4
       with:
         daemon-config: |
           {
-            "debug": true,
+            "debug": false,
             "features": {
               "containerd-snapshotter": true
             }
           }
 
-    - uses: docker/setup-buildx-action@v3
-      with:
-        buildkitd-flags: --oci-worker-gc --oci-worker-gc-keepstorage=4000
-
     - name: Build Docker images
       shell: bash
       run: |
@@ -122,10 +101,12 @@ runs:
         echo "Contents of /tmp/meta.json:"
         cat /tmp/meta.json
 
-        mkdir -p /tmp/digests/${{ inputs.service }}${{ inputs.type == 'coverage' && '-cov' || '' }}/${{ inputs.arch }}
+        SERVICE_SUFFIX=${{ inputs.service == 'rocketchat' && inputs.type == 'coverage' && (github.event_name == 'release' || github.ref == 'refs/heads/develop') && '-cov' || '' }}
+
+        mkdir -p /tmp/digests/${{ inputs.service }}${SERVICE_SUFFIX}/${{ inputs.arch }}
         DIGEST=$(jq -r '.["${{ inputs.service }}"].["containerimage.digest"]' "/tmp/meta.json")
         IMAGE_NO_TAG=$(echo "$IMAGE"  | sed 's/:.*$//')
-        echo "${IMAGE_NO_TAG}@${DIGEST}" > "/tmp/digests/${{ inputs.service }}${{ inputs.type == 'coverage' && '-cov' || '' }}/${{ inputs.arch }}/digest.txt"
+        echo "${IMAGE_NO_TAG}@${DIGEST}" > "/tmp/digests/${{ inputs.service }}${SERVICE_SUFFIX}/${{ inputs.arch }}/digest.txt"
 
     - uses: actions/upload-artifact@v4
       if: inputs.publish-image == 'true'

.github/actions/meteor-build/action.yml

@@ -34,9 +34,7 @@ runs:
       id: cache-build
       with:
         path: /tmp/Rocket.Chat.tar.gz
-        key: ${{ runner.OS }}-rc-build-${{ inputs.source-hash }}
-        restore-keys: |
-          ${{ runner.os }}-rc-build-
+        key: ${{ runner.os }}-${{ runner.arch }}-${{ inputs.type }}-rc-build-${{ inputs.source-hash }}
 
     - name: Set Swap Space
       uses: pierotofy/set-swap-space@master
@@ -65,27 +63,27 @@ runs:
       if: steps.cache-build.outputs.cache-hit != 'true'
       with:
         path: ./node_modules/.vite
-        key: vite-local-cache-${{ runner.OS }}-${{ hashFiles('package.json') }}
+        key: vite-local-cache-${{ runner.os }}-${{ runner.arch }}-${{ hashFiles('package.json') }}
         restore-keys: |
-          vite-local-cache-${{ runner.os }}-
+          vite-local-cache-${{ runner.os }}-${{ runner.arch }}-
 
     - name: Cache meteor local
       uses: actions/cache@v3
       if: steps.cache-build.outputs.cache-hit != 'true'
       with:
         path: ./apps/meteor/.meteor/local
-        key: meteor-local-cache-${{ runner.OS }}-${{ hashFiles('apps/meteor/.meteor/versions') }}
+        key: meteor-local-cache-${{ runner.os }}-${{ runner.arch }}-${{ inputs.type }}-${{ hashFiles('apps/meteor/.meteor/versions') }}
         restore-keys: |
-          meteor-local-cache-${{ runner.os }}-
+          meteor-local-cache-${{ runner.os }}-${{ runner.arch }}-${{ inputs.type }}-
 
     - name: Cache meteor
       uses: actions/cache@v3
       if: steps.cache-build.outputs.cache-hit != 'true'
       with:
         path: ~/.meteor
-        key: meteor-cache-${{ runner.OS }}-${{ hashFiles('apps/meteor/.meteor/release') }}
+        key: meteor-cache-${{ runner.os }}-${{ runner.arch }}-${{ inputs.type }}-${{ hashFiles('apps/meteor/.meteor/release') }}
         restore-keys: |
-          meteor-cache-${{ runner.os }}-
+          meteor-cache-${{ runner.os }}-${{ runner.arch }}-${{ inputs.type }}-
 
     - name: Install Meteor
       shell: bash

.github/actions/setup-playwright/action.yml

@@ -11,11 +11,10 @@ runs:
         path: |
           ~/.cache/ms-playwright
         # This is the version of Playwright that we are using, if you are willing to upgrade, you should update this.
-        key: playwright-1.52.0
+        key: playwright-${{ runner.os }}-${{ runner.arch }}-1.52.0
 
     - name: Install Playwright
       shell: bash
       if: steps.cache-playwright.outputs.cache-hit != 'true'
       working-directory: .
       run: npx playwright install --with-deps
-      

.github/workflows/ci-test-e2e.yml

@@ -20,7 +20,7 @@ on:
       transporter:
         type: string
       mongodb-version:
-        default: "['5.0', '8.2']"
+        default: "['8.2']"
         required: false
         type: string
       release:
@@ -41,6 +41,9 @@ on:
       type:
         required: true
         type: string
+      coverage:
+        required: false
+        type: string
       db-watcher-disabled:
         default: 'true'
         required: false
@@ -66,12 +69,16 @@ env:
   TOOL_NODE_FLAGS: ${{ vars.TOOL_NODE_FLAGS }}
   LOWERCASE_REPOSITORY: ${{ inputs.lowercase-repo }}
   DOCKER_TAG: ${{ inputs.gh-docker-tag }}
-  DOCKER_TAG_SUFFIX_ROCKETCHAT: '-cov'
 
 jobs:
   test:
     runs-on: ubuntu-24.04
 
+    env:
+      # if building for production on develop branch or release, add suffix for coverage images
+      DOCKER_TAG_SUFFIX_ROCKETCHAT: ${{ inputs.coverage == matrix.mongodb-version && (github.event_name == 'release' || github.ref == 'refs/heads/develop') && '-cov' || '' }}
+      MONGODB_VERSION: ${{ matrix.mongodb-version }}
+
     strategy:
       fail-fast: false
       matrix:
@@ -105,13 +112,6 @@ jobs:
           username: ${{ secrets.CR_USER }}
           password: ${{ secrets.CR_PAT }}
 
-      - name: Launch MongoDB
-        uses: supercharge/mongodb-github-action@1.12.0
-        with:
-          mongodb-image: mongodb/mongodb-community-server
-          mongodb-version: ${{ matrix.mongodb-version }}-ubi8
-          mongodb-replica-set: rs0
-
       - uses: actions/checkout@v5
 
       - name: Setup NodeJS
@@ -142,12 +142,9 @@ jobs:
         with:
           CR_USER: ${{ secrets.CR_USER }}
           CR_PAT: ${{ secrets.CR_PAT }}
-          node-version: ${{ inputs.node-version }}
           # the same reason we need to rebuild the docker image at this point is the reason we dont want to publish it
           publish-image: false
-          setup: false
           service: 'rocketchat'
-          NPM_TOKEN: ${{ secrets.NPM_TOKEN }}
 
       - name: Set DEBUG_LOG_LEVEL (debug enabled)
         if: runner.debug == '1'
@@ -156,58 +153,34 @@ jobs:
       - name: Start httpbin container and wait for it to be ready
         if: inputs.type == 'api'
         run: |
-          docker run -d -p 10000:80 --name httpbin-container kennethreitz/httpbin
-          i=0
-          while [ $i -lt 10 ]; do
-            if curl -s -o /dev/null http://localhost:10000; then
-              echo "httpbin is running"
-              break
-            fi
-            i=$((i + 1))
-            sleep 5
-          done
-          if [ $i -eq 10 ]; then
-            echo "Failed to verify httpbin is running"
-            exit 1
-          fi
+          docker compose -f docker-compose-ci.yml up -d httpbin
 
       - name: Prepare code coverage directory
-        if: inputs.release == 'ee'
+        if: inputs.coverage == matrix.mongodb-version
         run: |
           mkdir -p /tmp/coverage
           chmod 777 /tmp/coverage
 
       - name: Start containers for CE
         if: inputs.release == 'ce'
-        env:
-          MONGO_URL: 'mongodb://host.docker.internal:27017/rocketchat?replicaSet=rs0&directConnection=true'
         run: |
           # when we are testing CE, we only need to start the rocketchat container
-          DEBUG_LOG_LEVEL=${DEBUG_LOG_LEVEL:-0} docker compose -f docker-compose-ci.yml up -d rocketchat
+          DEBUG_LOG_LEVEL=${DEBUG_LOG_LEVEL:-0} docker compose -f docker-compose-ci.yml up -d rocketchat --wait
 
       - name: Start containers for EE
         if: inputs.release == 'ee'
         env:
-          MONGO_URL: 'mongodb://host.docker.internal:27017/rocketchat?replicaSet=rs0&directConnection=true'
           ENTERPRISE_LICENSE: ${{ inputs.enterprise-license }}
           TRANSPORTER: ${{ inputs.transporter }}
           COVERAGE_DIR: '/tmp/coverage'
           COVERAGE_REPORTER: 'lcov'
           DISABLE_DB_WATCHERS: ${{ inputs.db-watcher-disabled }}
         run: |
-          DEBUG_LOG_LEVEL=${DEBUG_LOG_LEVEL:-0} docker compose -f docker-compose-ci.yml up -d
+          DEBUG_LOG_LEVEL=${DEBUG_LOG_LEVEL:-0} docker compose -f docker-compose-ci.yml up -d --wait
 
       - uses: ./.github/actions/setup-playwright
         if: inputs.type == 'ui'
 
-      - name: Wait for Rocket.Chat to start up
-        uses: cygnetdigital/wait_for_response@v2.0.0
-        with:
-          url: 'http://localhost:3000/health'
-          responseCode: '200'
-          timeout: 60000
-          interval: 1000
-
       - name: Wait services to start up
         if: inputs.release == 'ee'
         run: |
@@ -226,30 +199,15 @@ jobs:
         if: inputs.type == 'api'
         working-directory: ./apps/meteor
         env:
-          WEBHOOK_TEST_URL: 'http://host.docker.internal:10000'
+          WEBHOOK_TEST_URL: 'http://httpbin'
           IS_EE: ${{ inputs.release == 'ee' && 'true' || '' }}
           COVERAGE_DIR: '/tmp/coverage'
           COVERAGE_REPORTER: 'lcovonly'
         run: |
-          for i in $(seq 1 2); do
-            npm run testapi && s=0 && break || s=$?
-
-            docker compose -f ../../docker-compose-ci.yml logs --tail=100
-
-            docker compose -f ../../docker-compose-ci.yml stop
+          set -o xtrace
 
-            docker exec mongodb bash -c 'if command -v mongosh ; then mongosh --eval "use rocketchat" --eval "db.dropDatabase()" rocketchat; else mongo rocketchat --eval "db.dropDatabase()"; fi'
+          npm run testapi
 
-            NOW=$(date "+%Y-%m-%dT%H:%M:%S.000Z")
-
-            docker compose -f ../../docker-compose-ci.yml restart
-
-            until echo "$(docker compose -f ../../docker-compose-ci.yml logs rocketchat --since $NOW)" | grep -q "SERVER RUNNING"; do
-              echo "Waiting Rocket.Chat to start up"
-              ((c++)) && ((c==10)) && exit 1
-              sleep 10
-            done;
-          done;
           docker compose -f ../../docker-compose-ci.yml stop
 
           ls -l $COVERAGE_DIR
@@ -258,7 +216,7 @@ jobs:
       - name: E2E Test UI (${{ matrix.shard }}/${{ inputs.total-shard }})
         if: inputs.type == 'ui'
         env:
-          E2E_COVERAGE: ${{ inputs.release == 'ee' && 'true' || '' }}
+          E2E_COVERAGE: ${{ inputs.coverage == matrix.mongodb-version && 'true' || '' }}
           IS_EE: ${{ inputs.release == 'ee' && 'true' || '' }}
           REPORTER_ROCKETCHAT_API_KEY: ${{ secrets.REPORTER_ROCKETCHAT_API_KEY }}
           REPORTER_ROCKETCHAT_URL: ${{ secrets.REPORTER_ROCKETCHAT_URL }}
@@ -289,28 +247,28 @@ jobs:
           include-hidden-files: true
 
       - name: Show server logs if E2E test failed
-        if: failure() && inputs.release == 'ee'
-        run: docker compose -f docker-compose-ci.yml logs
+        if: failure()
+        run: docker compose -f docker-compose-ci.yml logs rocketchat authorization-service queue-worker-service ddp-streamer-service account-service presence-service stream-hub-service omnichannel-transcript-service
 
-      - name: Show server logs if E2E test failed
-        if: failure() && inputs.release != 'ee'
-        run: docker compose -f docker-compose-ci.yml logs rocketchat
+      - name: Show mongo logs if E2E test failed
+        if: failure()
+        run: docker compose -f docker-compose-ci.yml logs mongo
 
       - name: Extract e2e:ee:coverage
-        if: inputs.type == 'ui' && inputs.release == 'ee'
+        if: inputs.coverage == matrix.mongodb-version && inputs.type == 'ui'
         working-directory: ./apps/meteor
         run: yarn test:e2e:nyc
 
       - uses: codecov/codecov-action@v3
-        if: inputs.type == 'ui' && inputs.release == 'ee'
+        if: inputs.coverage == matrix.mongodb-version && inputs.type == 'ui'
         with:
           directory: ./apps/meteor
           flags: e2e
           verbose: true
           token: ${{ secrets.CODECOV_TOKEN }}
 
       - uses: codecov/codecov-action@v3
-        if: inputs.type == 'api' && inputs.release == 'ee'
+        if: inputs.coverage == matrix.mongodb-version && inputs.type == 'api'
         with:
           directory: /tmp/coverage
           working-directory: .
@@ -319,15 +277,15 @@ jobs:
           token: ${{ secrets.CODECOV_TOKEN }}
 
       - name: Store e2e-api-ee-coverage
-        if: inputs.type == 'api' && inputs.release == 'ee'
+        if: inputs.coverage == matrix.mongodb-version && inputs.type == 'api'
         uses: actions/upload-artifact@v4
         with:
           name: e2e-api-ee-coverage-${{ matrix.mongodb-version }}-${{ matrix.shard }}
           path: /tmp/coverage
           include-hidden-files: true
 
       - name: Store e2e-ee-coverage
-        if: inputs.type == 'ui' && inputs.release == 'ee'
+        if: inputs.coverage == matrix.mongodb-version && inputs.type == 'ui'
         uses: actions/upload-artifact@v4
         with:
           name: e2e-ee-coverage-${{ matrix.mongodb-version }}-${{ matrix.shard }}