Merged dspace-cris-2024_02_x into task/dspace-cris-2024_02_x/DSC-1956

Author: FrancescoMolinaro

.github/workflows/build.yml

@@ -24,6 +24,7 @@ jobs:
       # Spin up UI on 127.0.0.1 to avoid host resolution issues in e2e tests with Node 18+
       DSPACE_UI_HOST: 127.0.0.1
       DSPACE_UI_PORT: 4000
+      DSPACE_UI_BASEURL: http://127.0.0.1:4000
       # Ensure all SSR caching is disabled in test environment
       DSPACE_CACHE_SERVERSIDE_BOTCACHE_MAX: 0
       DSPACE_CACHE_SERVERSIDE_ANONYMOUSCACHE_MAX: 0
@@ -39,8 +40,11 @@ jobs:
       # Project name to use when running "docker compose" prior to e2e tests
       COMPOSE_PROJECT_NAME: 'ci'
       # Docker Registry to use for Docker compose scripts below.
-      # We use GitHub's Container Registry to avoid aggressive rate limits at DockerHub.
-      DOCKER_REGISTRY: ghcr.io
+      # On the upstream dspace/dspace-angular repository we use GitHub's Container Registry
+      # (ghcr.io) to avoid aggressive rate limits at DockerHub. Forks cannot authenticate
+      # against ghcr.io/dspace/* with their own GITHUB_TOKEN (and the images there require
+      # auth), so on forks we fall back to docker.io where the same images are public.
+      DOCKER_REGISTRY: ${{ github.repository == 'dspace/dspace-angular' && 'ghcr.io' || 'docker.io' }}
     strategy:
       # Create a matrix of Node versions to test against (in parallel)
       matrix:
@@ -51,11 +55,11 @@ jobs:
     steps:
       # https://github.com/actions/checkout
       - name: Checkout codebase
-        uses: actions/checkout@v4
+        uses: actions/checkout@v6
 
       # https://github.com/actions/setup-node
       - name: Install Node.js ${{ matrix.node-version }}
-        uses: actions/setup-node@v4
+        uses: actions/setup-node@v6
         with:
           node-version: ${{ matrix.node-version }}
 
@@ -80,7 +84,7 @@ jobs:
         id: yarn-cache-dir-path
         run: echo "dir=$(yarn cache dir)" >> $GITHUB_OUTPUT
       - name: Cache Yarn dependencies
-        uses: actions/cache@v4
+        uses: actions/cache@v5
         with:
           # Cache entire Yarn cache directory (see previous step)
           path: ${{ steps.yarn-cache-dir-path.outputs.dir }}
@@ -113,16 +117,21 @@ jobs:
       # so that it can be shared with the 'codecov' job (see below)
       # NOTE: Angular CLI only supports code coverage for specs. See https://github.com/angular/angular-cli/issues/6286
       - name: Upload code coverage report to Artifact
-        uses: actions/upload-artifact@v4
+        uses: actions/upload-artifact@v7
         if: matrix.node-version == '18.x'
         with:
           name: coverage-report-${{ matrix.node-version }}
           path: 'coverage/dspace-angular/lcov.info'
           retention-days: 14
 
-      # Login to our Docker registry, so that we can access private Docker images using "docker compose" below.
+      # Login to our Docker registry, so that we can access Docker images using "docker compose" below.
+      # This login is required on the upstream repository because DOCKER_REGISTRY is set to ghcr.io
+      # and pulling ghcr.io/dspace/* requires authentication. On forks, DOCKER_REGISTRY falls back to
+      # docker.io (see env block above) where the same images are publicly pullable without a login,
+      # and forks cannot authenticate against ghcr.io/dspace/* with their own GITHUB_TOKEN anyway.
       - name: Login to ${{ env.DOCKER_REGISTRY }}
-        uses: docker/login-action@v3
+        uses: docker/login-action@v4
+        if: github.repository == 'dspace/dspace-angular'
         with:
           registry: ${{ env.DOCKER_REGISTRY }}
           username: ${{ github.repository_owner }}
@@ -139,7 +148,7 @@ jobs:
       # https://github.com/cypress-io/github-action
       # (NOTE: to run these e2e tests locally, just use 'ng e2e')
       - name: Run e2e tests (integration tests)
-        uses: cypress-io/github-action@v6
+        uses: cypress-io/github-action@v7.1.9
         with:
           # Run tests in Chrome, headless mode (default)
           browser: chrome
@@ -154,7 +163,7 @@ jobs:
       # Cypress always creates a video of all e2e tests (whether they succeeded or failed)
       # Save those in an Artifact
       - name: Upload e2e test videos to Artifacts
-        uses: actions/upload-artifact@v4
+        uses: actions/upload-artifact@v7
         if: always()
         with:
           name: e2e-test-videos-${{ matrix.node-version }}
@@ -163,7 +172,7 @@ jobs:
       # If e2e tests fail, Cypress creates a screenshot of what happened
       # Save those in an Artifact
       - name: Upload e2e test failure screenshots to Artifacts
-        uses: actions/upload-artifact@v4
+        uses: actions/upload-artifact@v7
         if: failure()
         with:
           name: e2e-test-screenshots-${{ matrix.node-version }}
@@ -306,22 +315,25 @@ jobs:
   codecov:
     # Must run after 'tests' job above
     needs: tests
+    # Only run on the upstream repository: forks do not have the CODECOV_TOKEN secret,
+    # and Codecov refuses to create a commit on a protected branch without a token.
+    if: github.repository == 'dspace/dspace-angular'
     runs-on: ubuntu-latest
     steps:
       - name: Checkout
-        uses: actions/checkout@v4
+        uses: actions/checkout@v6
 
       # Download artifacts from previous 'tests' job
       - name: Download coverage artifacts
-        uses: actions/download-artifact@v4
+        uses: actions/download-artifact@v8
 
       # Now attempt upload to Codecov using its action.
       # NOTE: We use a retry action to retry the Codecov upload if it fails the first time.
       #
       # Retry action: https://github.com/marketplace/actions/retry-action
       # Codecov action: https://github.com/codecov/codecov-action
       - name: Upload coverage to Codecov.io
-        uses: Wandalen/wretry.action@v1.3.0
+        uses: Wandalen/wretry.action@v3.8.0
         with:
           action: codecov/codecov-action@v4
           # Ensure codecov-action throws an error when it fails to upload

.github/workflows/codescan.yml

@@ -35,19 +35,19 @@ jobs:
     steps:
       # https://github.com/actions/checkout
       - name: Checkout repository
-        uses: actions/checkout@v4
+        uses: actions/checkout@v6
 
       # Initializes the CodeQL tools for scanning.
       # https://github.com/github/codeql-action
       - name: Initialize CodeQL
-        uses: github/codeql-action/init@v2
+        uses: github/codeql-action/init@v4
         with:
           languages: javascript
 
       # Autobuild attempts to build any compiled languages
       - name: Autobuild
-        uses: github/codeql-action/autobuild@v2
+        uses: github/codeql-action/autobuild@v4
 
       # Perform GitHub Code Scanning.
       - name: Perform CodeQL Analysis
-        uses: github/codeql-action/analyze@v2
+        uses: github/codeql-action/analyze@v4

.github/workflows/docker.yml

@@ -80,3 +80,122 @@ jobs:
       # these sites as specified in reusable-docker-build.xml
       REDEPLOY_SANDBOX_URL:  ${{ secrets.REDEPLOY_SANDBOX_URL }}
       REDEPLOY_DEMO_URL:  ${{ secrets.REDEPLOY_DEMO_URL }}
+
+  #################################################################################
+  # Test Deployment via Docker to ensure newly built images are working properly
+  #################################################################################
+  docker-deploy:
+    # Ensure this job never runs on forked repos. It's only executed for 'dspace/dspace-angular'
+    if: github.repository == 'dspace/dspace-angular'
+    runs-on: ubuntu-latest
+        # Must run after all major images are built
+    needs: [dspace-angular, dspace-angular-dist]
+    env:
+      # Override default dspace.server.url & REST 'host' because backend starts at http://127.0.0.1:8080
+      dspace__P__server__P__url: http://127.0.0.1:8080/server
+      DSPACE_REST_HOST: 127.0.0.1
+      # Override default dspace.ui.url to also use 127.0.0.1.
+      dspace__P__ui__P__url: http://127.0.0.1:4000
+      # Override default ui.baseUrl to also use 127.0.0.1. This should match 'dspace.ui.url'.
+      DSPACE_UI_BASEURL: http://127.0.0.1:4000
+    steps:
+      # Checkout our codebase (to get access to Docker Compose scripts)
+      - name: Checkout codebase
+        uses: actions/checkout@v6
+      # Download Docker image artifacts (which were just built by reusable-docker-build.yml)
+      - name: Download Docker image artifacts
+        uses: actions/download-artifact@v8
+        with:
+          # Download all amd64 Docker images (TAR files) into the /tmp/docker directory
+          pattern: docker-image-*-linux-amd64
+          path: /tmp/docker
+          merge-multiple: true
+      # Load each of the images into Docker by calling "docker image load" for each.
+      # This ensures we are using the images just built & not any prior versions on DockerHub
+      - name: Load all downloaded Docker images
+        run: |
+          find /tmp/docker -type f -name "*.tar" -exec docker image load --input "{}" \;
+          docker image ls -a
+      # Start backend using our compose script in the codebase.
+      - name: Start backend in Docker
+        # MUST use docker.io as we don't have a copy of this backend image in our GitHub Action,
+        # and docker.io is the only public image. If we ever hit aggressive rate limits at DockerHub,
+        # we may need to consider making the 'ghcr.io' images public & switch this to 'ghcr.io'
+        env:
+          DOCKER_REGISTRY: docker.io
+        run: |
+          docker compose -f docker/docker-compose-rest.yml up -d
+          sleep 10
+          docker container ls
+      # Create a test admin account. Load test data from a simple set of AIPs as defined in cli.ingest.yml
+      # NOTE: Before creating test data, we wait for the backend to become responsive by requesting it every 10 sec.
+      # Timeout after 5 minutes. This is done to ensure the backend is fully initialized before we create test data.
+      - name: Load test data into Backend
+        run: |
+          timeout 5m wget --retry-connrefused -t 0 --waitretry=10 http://127.0.0.1:8080/server/api
+          docker compose -f docker/cli.yml run --rm dspace-cli create-administrator -e test@test.edu -f admin -l user -p admin -c en
+          docker compose -f docker/cli.yml -f docker/cli.ingest.yml run --rm dspace-cli
+      # Verify backend started successfully.
+      # 1. Make sure root endpoint is responding (check for dspace.name defined in docker-compose.yml)
+      # 2. Also check /collections endpoint to ensure the test data loaded properly (check for a collection name in AIPs)
+      - name: Verify backend is responding properly
+        run: |
+          result=$(wget -O- -q http://127.0.0.1:8080/server/api)
+          echo "$result"
+          echo "$result" |  grep -oE "\"DSpace Started with Docker Compose\""
+          result=$(wget -O- -q http://127.0.0.1:8080/server/api/core/collections)
+          echo "$result"
+          echo "$result" |  grep -oE "\"Dog in Yard\""
+      # Start production frontend using our compose script in the codebase.
+      - name: Start production frontend in Docker
+        # Specify the GHCR copy of the production frontend, so that we use the newly built image
+        env:
+          DOCKER_REGISTRY: ghcr.io
+        run: |
+          docker compose -f docker/docker-compose-dist.yml up -d
+          sleep 10
+          docker container ls
+      # Verify production frontend started successfully.
+      # 1. Make sure /home path has "DSpace software" (this is in the footer of the page)
+      # 2. Also check /community-list page lists one of the test Communities in the loaded test data
+      - name: Verify production frontend is responding properly
+        run: |
+          result=$(wget -O- -q http://127.0.0.1:4000/home)
+          echo "$result"
+          echo "$result" |  grep -oE "\"DSpace software\""
+      - name: Error logs of production frontend (if error in startup)
+        if: ${{ failure() }}
+        run: |
+          docker compose -f docker/docker-compose-dist.yml logs
+      # Now shutdown the production frontend image and startup the development frontend image
+      - name: Shutdown production frontend
+        run: |
+          docker compose -f docker/docker-compose-dist.yml down
+          sleep 10
+          docker container ls
+      - name: Startup development frontend
+        # Specify the GHCR copy of the development frontend, so that we use the newly built image
+        env:
+          DOCKER_REGISTRY: ghcr.io
+        run: |
+          docker compose -f docker/docker-compose.yml up -d
+          sleep 10
+          docker container ls
+      # Verify development frontend started successfully.
+      # 1. First, keep requesting the frontend every 10 seconds to wait until its up. Timeout after 10 minutes.
+      # 2. Once it's responding, check to see if the word "DSpace" appears.
+      #    We cannot check for anything more specific because development mode doesn't have SSR.
+      - name: Verify development frontend is responding properly
+        run: |
+          timeout 10m wget --retry-connrefused -t 0 --waitretry=10 http://127.0.0.1:4000
+          result=$(wget -O- -q http://127.0.0.1:4000)
+          echo "$result"
+          echo "$result" |  grep -oE "DSpace"
+      - name: Error logs of development frontend (if error in startup)
+        if: ${{ failure() }}
+        run: |
+          docker compose -f docker/docker-compose.yml logs
+      # Shutdown our containers
+      - name: Shutdown running Docker containers
+        run: |
+          docker compose -f docker/docker-compose.yml -f docker/docker-compose-rest.yml down

.github/workflows/issue_opened.yml

@@ -16,7 +16,7 @@ jobs:
       # Only add to project board if issue is flagged as "needs triage" or has no labels
       # NOTE: By default we flag new issues as "needs triage" in our issue template
       if: (contains(github.event.issue.labels.*.name, 'needs triage') || join(github.event.issue.labels.*.name) == '')
-      uses: actions/add-to-project@v1.0.0
+      uses: actions/add-to-project@v1.0.2
       # Note, the authentication token below is an ORG level Secret. 
       # It must be created/recreated manually via a personal access token with admin:org, project, public_repo permissions
       # See: https://docs.github.com/en/actions/configuring-and-managing-workflows/authenticating-with-the-github_token#permissions-for-the-github_token

.github/workflows/port_merged_pull_request.yml

@@ -23,11 +23,11 @@ jobs:
     if: github.event.pull_request.merged
     steps:
       # Checkout code
-      - uses: actions/checkout@v4
+      - uses: actions/checkout@v6
       # Port PR to other branch (ONLY if labeled with "port to")
       # See https://github.com/korthout/backport-action
       - name: Create backport pull requests
-        uses: korthout/backport-action@v2
+        uses: korthout/backport-action@v4
         with:
           # Trigger based on a "port to [branch]" label on PR
           # (This label must specify the branch name to port to)

.github/workflows/pull_request_opened.yml

@@ -21,4 +21,4 @@ jobs:
     # Assign the PR to whomever created it. This is useful for visualizing assignments on project boards
     # See https://github.com/toshimaru/auto-author-assign
     - name: Assign PR to creator
-      uses: toshimaru/auto-author-assign@v2.1.0
+      uses: toshimaru/auto-author-assign@v3.0.1

Dockerfile

@@ -8,17 +8,29 @@ ARG DOCKER_REGISTRY=docker.io
 FROM ${DOCKER_REGISTRY:-docker.io}/4science/dspace-cris-angular-dependencies:${DSPACE_VERSION:-dspace-cris-2024_02_x} AS dev
 
 WORKDIR /app
-ADD . /app/
-EXPOSE 4000
+
+# Copy over package files first, so this layer will only be rebuilt if those files change.
+COPY package.json yarn.lock ./
+
+# We run yarn install with an increased network timeout (5min) to avoid "ESOCKETTIMEDOUT" errors from hub.docker.com
+# See, for example https://github.com/yarnpkg/yarn/issues/5540
+RUN yarn install --network-timeout 300000
+
+# Add the rest of the source code
+COPY . /app/
 
 # When running in dev mode, 4GB of memory is required to build & launch the app.
 # This default setting can be overridden as needed in your shell, via an env file or in docker-compose.
 # See Docker environment var precedence: https://docs.docker.com/compose/environment-variables/envvars-precedence/
 ENV NODE_OPTIONS="--max_old_space_size=4096"
 
 # On startup, run in DEVELOPMENT mode (this defaults to live reloading enabled, etc).
-# Listen / accept connections from all IP addresses.
-# NOTE: At this time it is only possible to run Docker container in Production mode
-# if you have a public URL. See https://github.com/DSpace/dspace-angular/issues/1485
 ENV NODE_ENV=development
-CMD ["yarn", "serve", "--host", "0.0.0.0"]
+
+EXPOSE 4000
+
+# On startup, run this command to start application in dev mode
+ENTRYPOINT [ "yarn", "serve" ]
+# By default set host to 0.0.0.0 to listen/accept connections from all IP addresses.
+# Poll for changes every 5 seconds (if any detected, app will rebuild/restart)
+CMD ["--host 0.0.0.0", "--poll 5000"]

Dockerfile.dist

@@ -17,16 +17,31 @@ COPY . /app/
 
 WORKDIR /app
 
-# Build Angular app
+# Copy over package files first, so this layer will only be rebuilt if those files change.
+COPY package.json yarn.lock ./
+RUN yarn install --network-timeout 300000
+
+# Around 4GB of memory is required to build the app for production.
+# This default setting can be overridden as needed in your shell, via an env file or in docker-compose.
+# See Docker environment var precedence: https://docs.docker.com/compose/environment-variables/envvars-precedence/
+ENV NODE_OPTIONS="--max_old_space_size=4096"
+
+
 RUN yarn build:prod
 RUN yarn build:mirador
 
 # ---- Production image ----
 FROM docker.io/node:${NODE_VERSION-22}-alpine AS prod
 
+# Copy pre-built code from build image
+COPY --chown=node:node --from=build /app/dist /app/dist
+# Copy configs and PM2 startup script from local machine
+COPY --chown=node:node config /app/config
+COPY --chown=node:node docker/dspace-ui.json /app/dspace-ui.json
 # Install pm2 globally and clean npm cache
 RUN npm install --global pm2 && npm cache clean --force
 
+# Start up UI in PM2 in production mode
 WORKDIR /app
 
 # Only copy built files and config
@@ -39,3 +54,8 @@ ENV NODE_ENV=production
 EXPOSE 4000
 
 CMD ["pm2-runtime", "start", "dspace-ui.json", "--json"]
+
+# On startup, run start the DSpace UI in PM2
+ENTRYPOINT [ "pm2-runtime", "start", "dspace-ui.json" ]
+# By default, pass param that specifies to use JSON format logs.
+CMD ["--json"]