Add docker layer caching (intelowlproject#3318) (intelowlproject#3358)

* feat: optimize Dockerfile and add build caching for CI

* Update .github/workflows/docker-build-cache.yml

Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com>

* Update .github/workflows/pull_request_automation.yml

Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com>

* Apply suggestion from @Copilot

Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com>

* Remove cache-to settings from Docker build steps

Removed cache-to configuration for Docker builds.

* Add django-server-requirements.txt to Dockerfile

* Add cache mounts for npm, apt, and pip installations

* Use registry cache

* Remove bake

* Discard changes to start

---------

Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com>

Author: gqvz

.github/workflows/docker-build-cache.yml

@@ -0,0 +1,45 @@
+permissions:
+  contents: read
+  packages: write
+
+name: Docker Build Cache
+
+on:
+  push:
+    branches: [develop]
+    paths-ignore:
+      - "**.md"
+      - "docs/**"
+      - "integrations/**"
+
+jobs:
+  build-main-image:
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Checkout IntelOwl
+        uses: actions/checkout@v6.0.2
+
+      - name: Set image repo
+        run: echo "IMAGE_REPO=ghcr.io/${GITHUB_REPOSITORY,,}" >> "$GITHUB_ENV"
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+
+      - name: Login to GHCR
+        uses: docker/login-action@v3
+        with:
+          registry: ghcr.io
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Build main image and push cache
+        uses: docker/build-push-action@v6
+        with:
+          context: .
+          file: docker/Dockerfile
+          push: false
+          build-args: |
+            REPO_DOWNLOADER_ENABLED=false
+          cache-from: type=registry,ref=${{ env.IMAGE_REPO }}:cache-main
+          cache-to: type=registry,ref=${{ env.IMAGE_REPO }}:cache-main,mode=max

.github/workflows/pull_request_automation.yml

@@ -76,6 +76,16 @@ jobs:
           cp docker/env_file_app_template docker/env_file_app
           cp docker/env_file_postgres_template docker/env_file_postgres
 
+      - name: Set image repo
+        run: echo "IMAGE_REPO=ghcr.io/${GITHUB_REPOSITORY,,}" >> "$GITHUB_ENV"
+
+      - name: Login to GHCR
+        uses: docker/login-action@v3
+        with:
+          registry: ghcr.io
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
       - name: Startup script launch (Slow)
         if: contains(github.base_ref, 'master')
         run: |
@@ -96,7 +106,7 @@ jobs:
           BUILDKIT_PROGRESS: "plain"
           STAGE: "ci"
           REPO_DOWNLOADER_ENABLED: false
-
+          
       - name: Docker debug
         if: always()
         run: |
@@ -154,4 +164,4 @@ jobs:
       - name: Test with Jest
         run: |
           npm run test -- --silent --coverage
-        working-directory: ./frontend
+        working-directory: ./frontend

docker/Dockerfile

@@ -3,14 +3,15 @@
 FROM node:lts-alpine3.21 AS frontend-build
 
 WORKDIR /
+
+COPY frontend/package.json frontend/package-lock.json ./
+RUN npm install npm@11.11.0 --location=global && npm install
+
 # copy react source code
 COPY frontend/ .
 # copy version file as an env file
 COPY docker/.env .env.local
-# install and build
-RUN npm install npm@latest --location=global \
-    && npm install \
-    && PUBLIC_URL=/static/reactapp/ npm run build
+RUN PUBLIC_URL=/static/reactapp/ npm run build
 
 # Stage 2: Backend
 FROM python:3.11.7 AS backend-build
@@ -49,13 +50,15 @@ RUN apt-get update \
 
 COPY requirements/project-requirements.txt $PYTHONPATH/project-requirements.txt
 COPY requirements/certego-requirements.txt $PYTHONPATH/certego-requirements.txt
+COPY requirements/django-server-requirements.txt $PYTHONPATH/requirements/django-server-requirements.txt
 WORKDIR $PYTHONPATH
 
 RUN pip3 install --no-cache-dir --use-pep517 --compile -r project-requirements.txt \
     && pip3 install --no-cache-dir pycti==${PYCTI_VERSION} \
     && pip3 install --no-cache-dir --compile -r certego-requirements.txt
 
-COPY . $PYTHONPATH
+COPY api_app/analyzers_manager/repo_downloader.sh ${PYTHONPATH}/api_app/analyzers_manager/repo_downloader.sh
+COPY docker/scripts/watchman_install.sh ${PYTHONPATH}/docker/scripts/watchman_install.sh
 
 RUN touch ${LOG_PATH}/django/api_app.log ${LOG_PATH}/django/api_app_errors.log \
     && touch ${LOG_PATH}/django/intel_owl.log ${LOG_PATH}/django/intel_owl_errors.log \
@@ -65,10 +68,12 @@ RUN touch ${LOG_PATH}/django/api_app.log ${LOG_PATH}/django/api_app_errors.log \
     && touch ${LOG_PATH}/django/authentication.log ${LOG_PATH}/django/authentication_errors.log \
     && touch ${LOG_PATH}/asgi/daphne.log \
     && mkdir -p -m 755 ${PYTHONPATH}/.cache \
-    && chown -R www-data:www-data ${LOG_PATH} /opt/deploy/ ${PYTHONPATH}/.cache \
-    && ${PYTHONPATH}/docker/scripts/watchman_install.sh \
-    # download github stuff
-    && ${PYTHONPATH}/api_app/analyzers_manager/repo_downloader.sh
+    && ${PYTHONPATH}/api_app/analyzers_manager/repo_downloader.sh \
+    && ${PYTHONPATH}/docker/scripts/watchman_install.sh
+
+COPY . $PYTHONPATH
+
+RUN chown -R www-data:www-data ${LOG_PATH} /opt/deploy/ ${PYTHONPATH}/.cache
 
 FROM backend-build
 

docker/ci.override.yml

@@ -14,6 +14,8 @@ services:
       dockerfile: docker/Dockerfile
       args:
         REPO_DOWNLOADER_ENABLED: ${REPO_DOWNLOADER_ENABLED}
+      cache_from:
+        - type=registry,ref=${IMAGE_REPO:-ghcr.io/intelowlproject/intelowl}:cache-main
     image: intelowlproject/intelowl:ci
     env_file:
       - env_file_app_ci