From aedef902f32f10c9ff312d73740aeb102a3db8cf Mon Sep 17 00:00:00 2001 From: Valentijn Scholten Date: Sat, 28 Jun 2025 15:13:16 +0200 Subject: [PATCH 1/3] hot reloading improvements celery/html/tpl --- Dockerfile.django-alpine | 1 + Dockerfile.django-debian | 1 + docker-compose.override.dev.yml | 1 + docker-compose.override.integration_tests.yml | 1 + docker/entrypoint-uwsgi-dev.sh | 10 +++++++++- readme-docs/DOCKER.md | 3 ++- requirements.txt | 1 + 7 files changed, 16 insertions(+), 2 deletions(-) diff --git a/Dockerfile.django-alpine b/Dockerfile.django-alpine index d018effb748..939662de865 100644 --- a/Dockerfile.django-alpine +++ b/Dockerfile.django-alpine @@ -67,6 +67,7 @@ RUN export PYCURL_SSL_LIBRARY=openssl && \ COPY \ docker/entrypoint-celery-beat.sh \ docker/entrypoint-celery-worker.sh \ + docker/entrypoint-celery-worker-dev.sh \ docker/entrypoint-initializer.sh \ docker/entrypoint-first-boot.sh \ docker/entrypoint-uwsgi.sh \ diff --git a/Dockerfile.django-debian b/Dockerfile.django-debian index f8b9dccdcfd..da5b4b3a22a 100644 --- a/Dockerfile.django-debian +++ b/Dockerfile.django-debian @@ -70,6 +70,7 @@ RUN export PYCURL_SSL_LIBRARY=openssl && \ COPY \ docker/entrypoint-celery-beat.sh \ docker/entrypoint-celery-worker.sh \ + docker/entrypoint-celery-worker-dev.sh \ docker/entrypoint-initializer.sh \ docker/entrypoint-first-boot.sh \ docker/entrypoint-uwsgi.sh \ diff --git a/docker-compose.override.dev.yml b/docker-compose.override.dev.yml index b58b80a9523..dc1f915dbee 100644 --- a/docker-compose.override.dev.yml +++ b/docker-compose.override.dev.yml @@ -11,6 +11,7 @@ services: DD_ADMIN_PASSWORD: "${DD_ADMIN_PASSWORD:-admin}" DD_EMAIL_URL: "smtp://mailhog:1025" celeryworker: + entrypoint: ['/wait-for-it.sh', '${DD_DATABASE_HOST:-postgres}:${DD_DATABASE_PORT:-5432}', '-t', '30', '--', '/entrypoint-celery-worker-dev.sh'] volumes: - '.:/app:z' environment: diff --git a/docker-compose.override.integration_tests.yml b/docker-compose.override.integration_tests.yml index 7836c00ccd2..215529180df 100644 --- a/docker-compose.override.integration_tests.yml +++ b/docker-compose.override.integration_tests.yml @@ -38,6 +38,7 @@ services: environment: DD_DATABASE_URL: ${DD_TEST_DATABASE_URL:-postgresql://defectdojo:defectdojo@postgres:5432/test_defectdojo} celeryworker: + entrypoint: ['/wait-for-it.sh', '${DD_DATABASE_HOST:-postgres}:${DD_DATABASE_PORT:-5432}', '-t', '30', '--', '/entrypoint-celery-worker-dev.sh'] environment: DD_DATABASE_URL: ${DD_TEST_DATABASE_URL:-postgresql://defectdojo:defectdojo@postgres:5432/test_defectdojo} initializer: diff --git a/docker/entrypoint-uwsgi-dev.sh b/docker/entrypoint-uwsgi-dev.sh index 68116173162..200e3ec2107 100755 --- a/docker/entrypoint-uwsgi-dev.sh +++ b/docker/entrypoint-uwsgi-dev.sh @@ -21,6 +21,14 @@ if [ "${DD_DEBUG}" = "True" ]; then DD_UWSGI_NUM_OF_THREADS=1 fi +# hot reload also on html/template changes +watchmedo shell-command \ + --patterns="*.html;*.tpl" \ + --recursive \ + --command='touch /app/dojo/settings/settings.py' \ + /app/dojo & + + exec uwsgi \ "--${DD_UWSGI_MODE}" "${DD_UWSGI_ENDPOINT}" \ --protocol uwsgi \ @@ -33,5 +41,5 @@ exec uwsgi \ --py-autoreload 1 \ --buffer-size="${DD_UWSGI_BUFFER_SIZE:-8192}" \ --lazy-apps \ - --touch-reload="/app/dojo/setting/settings.py" \ + --touch-reload="/app/dojo/settings/settings.py" \ --logformat "${DD_UWSGI_LOGFORMAT:-$DD_UWSGI_LOGFORMAT_DEFAULT}" diff --git a/readme-docs/DOCKER.md b/readme-docs/DOCKER.md index 66245eba3a1..a2188b3cda6 100644 --- a/readme-docs/DOCKER.md +++ b/readme-docs/DOCKER.md @@ -109,7 +109,8 @@ This will run the application based on merged configurations from docker-compose * python code (uwsgi and celeryworker containers). * The `--py-autoreload 1` parameter in entrypoint-uwsgi-dev.sh will make uwsgi handle python hot-reloading for the **uwsgi** container. -* Hot-reloading for the **celeryworker** container is not yet implemented. When working on deduplication for example, restart the celeryworker container with: +* Hot-reloading for the **celeryworker** container is implemented via `wathmedo` from the `watchdog` package. +* Changes in `.html` and `.tpl` files will also trigger a roload. ``` docker compose restart celeryworker diff --git a/requirements.txt b/requirements.txt index ffbacc09c56..bad69586f6b 100644 --- a/requirements.txt +++ b/requirements.txt @@ -76,3 +76,4 @@ fontawesomefree==6.6.0 PyYAML==6.0.2 pyopenssl==25.1.0 parameterized==0.9.0 +watchdog==6.0.0 # only needed for development, but would require some docker refactoring if we want to exclude it for production images \ No newline at end of file From 193253f16c55e5553557e3a516fc3686e87821eb Mon Sep 17 00:00:00 2001 From: Valentijn Scholten Date: Sat, 28 Jun 2025 15:16:05 +0200 Subject: [PATCH 2/3] typo --- readme-docs/DOCKER.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/readme-docs/DOCKER.md b/readme-docs/DOCKER.md index a2188b3cda6..2510bcd5984 100644 --- a/readme-docs/DOCKER.md +++ b/readme-docs/DOCKER.md @@ -109,7 +109,7 @@ This will run the application based on merged configurations from docker-compose * python code (uwsgi and celeryworker containers). * The `--py-autoreload 1` parameter in entrypoint-uwsgi-dev.sh will make uwsgi handle python hot-reloading for the **uwsgi** container. -* Hot-reloading for the **celeryworker** container is implemented via `wathmedo` from the `watchdog` package. +* Hot-reloading for the **celeryworker** container is implemented via `watchmedo` from the `watchdog` package. * Changes in `.html` and `.tpl` files will also trigger a roload. ``` From fdb714fae59e1d48b7379dec438592b1095d39f5 Mon Sep 17 00:00:00 2001 From: Valentijn Scholten Date: Sat, 28 Jun 2025 21:40:49 +0200 Subject: [PATCH 3/3] add new entrypoint script --- docker/entrypoint-celery-worker-dev.sh | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) create mode 100644 docker/entrypoint-celery-worker-dev.sh diff --git a/docker/entrypoint-celery-worker-dev.sh b/docker/entrypoint-celery-worker-dev.sh new file mode 100644 index 00000000000..bd38ed028b8 --- /dev/null +++ b/docker/entrypoint-celery-worker-dev.sh @@ -0,0 +1,26 @@ +#!/bin/bash +umask 0002 + +id + +set -e # needed to handle "exit" correctly + +. /secret-file-loader.sh +. /reach_database.sh + +wait_for_database_to_be_reachable +echo + +if [ "${DD_CELERY_WORKER_POOL_TYPE}" = "prefork" ]; then + EXTRA_PARAMS=("--autoscale=${DD_CELERY_WORKER_AUTOSCALE_MAX},${DD_CELERY_WORKER_AUTOSCALE_MIN}" + "--prefetch-multiplier=${DD_CELERY_WORKER_PREFETCH_MULTIPLIER}") +else + EXTRA_PARAMS=() +fi + +# do the check with Django stack +python3 manage.py check + +# hot reload using watmedo as we don't want to install celery[dev] and have that end up in our production images +watchmedo auto-restart --directory=./ --pattern="*.py;*.tpl" --recursive -- \ + celery --app=dojo worker --loglevel="${DD_CELERY_LOG_LEVEL}" --pool="${DD_CELERY_WORKER_POOL_TYPE}" --concurrency="${DD_CELERY_WORKER_CONCURRENCY:-1}" "${EXTRA_PARAMS[@]}"