feat: workers for export item as zip (import export refactor) (#1891)

* feat: initialize worker for export item as zip, import export refactor

* refactor: fix lint

* fix: set worker as class var and use logger in error

* fix: add color to worker output on watch

* chore: change name of server process

* chore: ensure that localfile is accessible from inside and outside the server

* fix: add export extension

* build: docker file first version

* fix: dockerfile for workers

* fix: redis connection

* chore: change building of core image

* fix: changes to the compose file

* fix: add workers build in build script

* fix: make small adjustments

* fix: update deps to fix issues and lint problems

* fix: tests

* fix: do not expose the bashboard when mode is not dev

* fix: add tests to the worker using vitest

* fix: move vitest to build

* fix: cleanup code

* fix: add more test cases

* fix: make changes required by review

* fix: build issue

---------

Co-authored-by: spaenleh <spaenleh@gmail.com>

Author: pyphilia

.devcontainer/docker-compose.yml

@@ -18,9 +18,9 @@ services:
       # temporary storage for files inside the container
       FILE_STORAGE_ROOT_PATH: /tmp/graasp-file-item-storage
       # the localfile config host points to the static file server defined below
-      FILE_STORAGE_HOST: http://localhost:1081
+      FILE_STORAGE_HOST: http://localfile:1081
       # H5P local storage host used in local only
-      H5P_FILE_STORAGE_HOST: http://localhost:1081
+      H5P_FILE_STORAGE_HOST: http://localfile:1081
       # endpoint of the nudenet model
       IMAGE_CLASSIFIER_API: http://nudenet:8080/sync
       # the DB config is set by the "db" service below
@@ -156,12 +156,12 @@ services:
     environment:
       # Note: those envs are customizable but also optional
       # - SERVER_HOST=127.0.0.1
-      - SERVER_PORT=80
+      - SERVER_PORT=1081
       - SERVER_ROOT=/tmp/graasp-file-item-storage
     volumes:
       - ../tmp:/tmp/graasp-file-item-storage
     ports:
-      - 1081:80
+      - 1081:1081
 
   # necessary for graasp-embedded-link-item
   iframely:

.eslintrc

@@ -27,7 +27,9 @@
       "error",
       {
         "devDependencies": [
+          "vitest.config.ts",
           "**/*.test.*",
+          "**/*.spec.*",
           "**/test/*",
           "**/test/**/*"
         ]

.github/workflows/deploy-prod.yml

@@ -97,7 +97,7 @@ jobs:
           environment-variables: |
             DB_CONNECTION_POOL_SIZE=${{ vars.DB_CONNECTION_POOL_SIZE }}
             APPS_JWT_SECRET=${{ secrets.APPS_JWT_SECRET }}
-            APPS_PUBLISHER_ID=${{ secrets.APPS_PUBLISHER_ID }} 
+            APPS_PUBLISHER_ID=${{ secrets.APPS_PUBLISHER_ID }}
             AUTH_TOKEN_JWT_SECRET=${{ secrets.AUTH_TOKEN_JWT_SECRET }}
             CLIENT_HOST=${{ vars.CLIENT_HOST }}
             COOKIE_DOMAIN=${{ vars.COOKIE_DOMAIN }}
@@ -118,22 +118,21 @@ jobs:
             H5P_PATH_PREFIX=${{ vars.H5P_PATH_PREFIX }}
             HOSTNAME=${{ vars.HOSTNAME }}
             IMAGE_CLASSIFIER_API=${{ vars.IMAGE_CLASSIFIER_API }}
-            JOB_SCHEDULING=${{ vars.JOB_SCHEDULING || false }}
             JWT_SECRET=${{ secrets.JWT_SECRET }}
             PASSWORD_RESET_JWT_SECRET=${{ secrets.PASSWORD_RESET_JWT_SECRET }}
             EMAIL_CHANGE_JWT_SECRET=${{ secrets.EMAIL_CHANGE_JWT_SECRET }}
             LIBRARY_CLIENT_HOST=${{ vars.LIBRARY_CLIENT_HOST }}
             LOG_LEVEL=${{ vars.LOG_LEVEL }}
             MAILER_CONFIG_FROM_EMAIL=${{ secrets.MAILER_CONFIG_FROM_EMAIL }}
-            MAILER_CONNECTION=${{ secrets.MAILER_CONNECTION }} 
+            MAILER_CONNECTION=${{ secrets.MAILER_CONNECTION }}
             MEILISEARCH_MASTER_KEY=${{ secrets.MEILISEARCH_MASTER_KEY }}
             MEILISEARCH_REBUILD_SECRET=${{ secrets.MEILISEARCH_REBUILD_SECRET }}
             MEILISEARCH_URL=${{ secrets.MEILISEARCH_URL }}
             NODE_ENV=${{ vars.NODE_ENV }}
             PORT=${{ vars.PORT }}
             PUBLIC_URL=${{ secrets.PUBLIC_URL }}
             RECAPTCHA_SECRET_ACCESS_KEY=${{ secrets.RECAPTCHA_SECRET_ACCESS_KEY }}
-            REDIS_CONNECTION=${{ secrets.REDIS_CONNECTION }} 
+            REDIS_CONNECTION=${{ secrets.REDIS_CONNECTION }}
             REFRESH_TOKEN_JWT_SECRET=${{ secrets.REFRESH_TOKEN_JWT_SECRET }}
             S3_FILE_ITEM_ACCESS_KEY_ID=${{ secrets.S3_FILE_ITEM_ACCESS_KEY_ID }}
             S3_FILE_ITEM_BUCKET=${{ vars.S3_FILE_ITEM_BUCKET }}

.github/workflows/deploy-stage.yml

@@ -97,7 +97,7 @@ jobs:
           environment-variables: |
             DB_CONNECTION_POOL_SIZE=${{ vars.DB_CONNECTION_POOL_SIZE }}
             APPS_JWT_SECRET=${{ secrets.APPS_JWT_SECRET }}
-            APPS_PUBLISHER_ID=${{ secrets.APPS_PUBLISHER_ID }} 
+            APPS_PUBLISHER_ID=${{ secrets.APPS_PUBLISHER_ID }}
             AUTH_TOKEN_JWT_SECRET=${{ secrets.AUTH_TOKEN_JWT_SECRET }}
             CLIENT_HOST=${{ vars.CLIENT_HOST }}
             COOKIE_DOMAIN=${{ vars.COOKIE_DOMAIN }}
@@ -119,22 +119,21 @@ jobs:
             H5P_PATH_PREFIX=${{ vars.H5P_PATH_PREFIX }}
             HOSTNAME=${{ vars.HOSTNAME }}
             IMAGE_CLASSIFIER_API=${{ vars.IMAGE_CLASSIFIER_API }}
-            JOB_SCHEDULING=${{ vars.JOB_SCHEDULING || false }}
             JWT_SECRET=${{ secrets.JWT_SECRET }}
             PASSWORD_RESET_JWT_SECRET=${{ secrets.PASSWORD_RESET_JWT_SECRET }}
             EMAIL_CHANGE_JWT_SECRET=${{ secrets.EMAIL_CHANGE_JWT_SECRET }}
             LIBRARY_CLIENT_HOST=${{ vars.LIBRARY_CLIENT_HOST }}
             LOG_LEVEL=${{ vars.LOG_LEVEL }}
             MAILER_CONFIG_FROM_EMAIL=${{ secrets.MAILER_CONFIG_FROM_EMAIL }}
-            MAILER_CONNECTION=${{ secrets.MAILER_CONNECTION }} 
+            MAILER_CONNECTION=${{ secrets.MAILER_CONNECTION }}
             MEILISEARCH_MASTER_KEY=${{ secrets.MEILISEARCH_MASTER_KEY }}
             MEILISEARCH_REBUILD_SECRET=${{ secrets.MEILISEARCH_REBUILD_SECRET }}
             MEILISEARCH_URL=${{ secrets.MEILISEARCH_URL }}
             NODE_ENV=${{ vars.NODE_ENV }}
             PORT=${{ vars.PORT }}
             PUBLIC_URL=${{ secrets.PUBLIC_URL }}
             RECAPTCHA_SECRET_ACCESS_KEY=${{ secrets.RECAPTCHA_SECRET_ACCESS_KEY }}
-            REDIS_CONNECTION=${{ secrets.REDIS_CONNECTION }} 
+            REDIS_CONNECTION=${{ secrets.REDIS_CONNECTION }}
             REFRESH_TOKEN_JWT_SECRET=${{ secrets.REFRESH_TOKEN_JWT_SECRET }}
             S3_FILE_ITEM_ACCESS_KEY_ID=${{ secrets.S3_FILE_ITEM_ACCESS_KEY_ID }}
             S3_FILE_ITEM_BUCKET=${{ vars.S3_FILE_ITEM_BUCKET }}

.github/workflows/test.yml

@@ -51,7 +51,6 @@ env:
   SECURE_SESSION_SECRET_KEY: 0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef
   MEILISEARCH_URL: fake
   MEILISEARCH_MASTER_KEY: fake
-  JOB_SCHEDULING: false
   GEOLOCATION_API_KEY: geolocation-key
   GEOLOCATION_API_HOST: http://localhost:12345
 
@@ -79,6 +78,20 @@ jobs:
           --health-timeout 5s
           --health-retries 5
 
+      # Label used to access the service container
+      graasp-redis:
+        # Docker Hub image
+        image: redis
+        ports:
+          # Opens tcp port 6379 on the host and service container
+          - 6379:6379
+        # Set health checks to wait until redis has started
+        options: >-
+          --health-cmd "redis-cli ping"
+          --health-interval 10s
+          --health-timeout 5s
+          --health-retries 5
+
     steps:
       - uses: actions/checkout@v4
 
@@ -113,6 +126,9 @@ jobs:
       - name: check migrations are up to date
         run: yarn migration:check
 
+      - name: vitest tests
+        run: yarn vitest
+
   test-node:
     needs: build-node
     strategy:

README.md

@@ -216,9 +216,6 @@ MEILISEARCH_URL=http://meilisearch:7700
 MEILISEARCH_MASTER_KEY=masterKey
 MEILISEARCH_REBUILD_SECRET=secret
 
-# Enable job scheduling (for cron based tasks)
-JOB_SCHEDULING=true
-
 # OPEN AI
 # OPENAI_GPT_VERSION=<DEFAULT_GPT_VERSION> # valid values are gpt-4 or gpt-3.5-turbo
 OPENAI_API_KEY=<openai-api-key>

docker/Dockerfile

@@ -1,34 +1,43 @@
-# Production Docker backend image definition
-# Used in CI for creating images to push on registry
+FROM node:22.15-bookworm AS base
 
-# Update the NODE_VERSION arg in docker-compose.yml to pick a Node version of Docker Image
-ARG NODE_VERSION=22.14.0
 
 ##############################
 ### Build Application Step ###
 ##############################
 # This step will build the application
-FROM node:${NODE_VERSION} AS build
+FROM base AS tools
 
 # update packages and install the minimal init system "dumb-init"
 RUN apt-get update && apt-get install -y --no-install-recommends dumb-init
 
+
+# -------------------------------------------------------
+FROM base AS build
+
 WORKDIR /app
-COPY . .
-RUN yarn set version berry && yarn install
+COPY package.json yarn.lock .yarnrc.yml ./
+# We need a specific command because we need to copy the folder with it, not just the content.
+COPY .yarn/releases ./.yarn/releases/
+RUN yarn set version berry && yarn install --immutable
 
+ENV NODE_ENV=production
+COPY . .
 RUN yarn build-ts
 
-ENV NODE_ENV=production
+# -------------------------------------------------------
+FROM base AS deps
 
-# Delete old node_modules and re-install only production dependencies
-RUN rm -rf node_modules && yarn workspaces focus --all --production
+WORKDIR /app
+COPY package.json yarn.lock .yarnrc.yml ./
+# We need a specific command because we need to copy the folder with it, not just the content.
+COPY .yarn/releases ./.yarn/releases/
+RUN yarn set version berry && yarn workspaces focus --all --production
 
 ############################
 ### Run Application Step ###
 ############################
 # Final step that will run the application
-FROM node:${NODE_VERSION}-bookworm-slim AS runner
+FROM node:22.15-bookworm-slim AS runner
 
 # Variable passed as a build arg. Represents the tag or git sha used for the build
 ARG APP_VERSION
@@ -45,10 +54,10 @@ EXPOSE 3000
 WORKDIR /app
 
 # Copy the installed dumb-init system from build image
-COPY --from=build /usr/bin/dumb-init /usr/bin/dumb-init
+COPY --from=tools /usr/bin/dumb-init /usr/bin/dumb-init
 
 # Copy the dependencies and compiled server code
-COPY --chown=node:node --from=build ./app/node_modules ./node_modules
+COPY --chown=node:node --from=deps ./app/node_modules ./node_modules
 COPY --chown=node:node --from=build ./app/dist ./dist
 
 # Set user to be non-root node

docker/build.sh

@@ -31,11 +31,17 @@ tag_version=$2
 core_tag_short="graasp:core-$tag_version"
 core_tag_full="$aws_ecr_uri/$core_tag_short"
 
+workers_tag_short="graasp:workers-$tag_version"
+workers_tag_full="$aws_ecr_uri/$workers_tag_short"
+
 migrate_tag_short="graasp:migrate-$tag_version"
 migrate_tag_full="$aws_ecr_uri/$migrate_tag_short"
 
 docker build -t $core_tag_full -f docker/Dockerfile --platform linux/amd64 --build-arg APP_VERSION=$tag_version --build-arg BUILD_TIMESTAMP=$(date -u +%Y-%m-%dT%H:%M:%S) .
 docker push $core_tag_full
 
+docker build -t $workers_tag_full -f docker/workers.Dockerfile --platform linux/amd64 --build-arg APP_VERSION=$tag_version --build-arg BUILD_TIMESTAMP=$(date -u +%Y-%m-%dT%H:%M:%S) .
+docker push $workers_tag_full
+
 docker build -t $migrate_tag_full -f docker/migrate.Dockerfile --platform linux/amd64 .
 docker push $migrate_tag_full