diff --git a/.github/workflows/integration-tests.yml b/.github/workflows/integration-tests.yml index bf74a50643b..140c4f2befd 100644 --- a/.github/workflows/integration-tests.yml +++ b/.github/workflows/integration-tests.yml @@ -76,7 +76,7 @@ jobs: run: ln -s docker-compose.override.integration_tests.yml docker-compose.override.yml - name: Start Dojo - run: docker compose up --no-deps -d postgres nginx celerybeat celeryworker mailhog uwsgi redis + run: docker compose up --no-deps -d postgres nginx celerybeat celeryworker mailhog uwsgi valkey env: DJANGO_VERSION: ${{ matrix.os }} NGINX_VERSION: alpine diff --git a/README.md b/README.md index e239a7f6baf..f9d2511b07c 100644 --- a/README.md +++ b/README.md @@ -56,8 +56,8 @@ cd django-DefectDojo # Building Docker images docker compose build -# Run the application (for other profiles besides postgres-redis see -# https://github.com/DefectDojo/django-DefectDojo/blob/dev/readme-docs/DOCKER.md) +# Run the application +# (see https://github.com/DefectDojo/django-DefectDojo/blob/dev/readme-docs/DOCKER.md for more info) docker compose up -d # Obtain admin credentials. The initializer can take up to 3 minutes to run. @@ -67,7 +67,7 @@ docker compose logs initializer | grep "Admin password:" ## For Docker Compose V1 -You can run Compose V1 by calling `docker-compose` (by adding the hyphen (-) between `docker compose`). +You can run Compose V1 by calling `docker-compose` (by adding the hyphen (-) between `docker compose`). Following commands are using original version so you might need to adjust them: ```sh @@ -132,8 +132,8 @@ Moderators can help you with pull requests or feedback on dev ideas: * Blake Owens ([@blakeaowens](https://github.com/blakeaowens)) ## Hall of Fame -* Jannik Jürgens ([@alles-klar](https://github.com/alles-klar)) - Jannik was a long time contributor and moderator for - DefectDojo and made significant contributions to many areas of the platform. Jannik was instrumental in pioneering +* Jannik Jürgens ([@alles-klar](https://github.com/alles-klar)) - Jannik was a long time contributor and moderator for + DefectDojo and made significant contributions to many areas of the platform. Jannik was instrumental in pioneering and optimizing deployment methods. * Valentijn Scholten ([@valentijnscholten](https://github.com/valentijnscholten) | [Sponsor](https://github.com/sponsors/valentijnscholten) | diff --git a/docker-compose.override.unit_tests.yml b/docker-compose.override.unit_tests.yml index d1b90f57fdd..439abea2d3f 100644 --- a/docker-compose.override.unit_tests.yml +++ b/docker-compose.override.unit_tests.yml @@ -42,7 +42,7 @@ services: POSTGRES_DB: ${DD_TEST_DATABASE_NAME:-test_defectdojo} volumes: - defectdojo_postgres_unit_tests:/var/lib/postgresql/data - redis: !reset + valkey: !reset "webhook.endpoint": image: mccutchen/go-httpbin:2.18.3@sha256:3992f3763e9ce5a4307eae0a869a78b4df3931dc8feba74ab823dd2444af6a6b volumes: diff --git a/docker-compose.override.unit_tests_cicd.yml b/docker-compose.override.unit_tests_cicd.yml index 8d6eec1701c..0acd340ce4c 100644 --- a/docker-compose.override.unit_tests_cicd.yml +++ b/docker-compose.override.unit_tests_cicd.yml @@ -41,7 +41,7 @@ services: POSTGRES_DB: ${DD_TEST_DATABASE_NAME:-test_defectdojo} volumes: - defectdojo_postgres_unit_tests:/var/lib/postgresql/data - redis: !reset + valkey: !reset "webhook.endpoint": image: mccutchen/go-httpbin:2.18.3@sha256:3992f3763e9ce5a4307eae0a869a78b4df3931dc8feba74ab823dd2444af6a6b volumes: diff --git a/docker-compose.yml b/docker-compose.yml index be6bf4468cb..24832c74e3e 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -41,7 +41,7 @@ services: condition: service_completed_successfully postgres: condition: service_started - redis: + valkey: condition: service_started entrypoint: ['/wait-for-it.sh', '${DD_DATABASE_HOST:-postgres}:${DD_DATABASE_PORT:-5432}', '-t', '30', '--', '/entrypoint-uwsgi.sh'] environment: @@ -49,7 +49,7 @@ services: DD_DJANGO_METRICS_ENABLED: "${DD_DJANGO_METRICS_ENABLED:-False}" DD_ALLOWED_HOSTS: "${DD_ALLOWED_HOSTS:-*}" DD_DATABASE_URL: ${DD_DATABASE_URL:-postgresql://defectdojo:defectdojo@postgres:5432/defectdojo} - DD_CELERY_BROKER_URL: ${DD_CELERY_BROKER_URL:-redis://redis:6379/0} + DD_CELERY_BROKER_URL: ${DD_CELERY_BROKER_URL:-redis://valkey:6379/0} DD_SECRET_KEY: "${DD_SECRET_KEY:-hhZCp@D28z!n@NED*yB!ROMt+WzsY*iq}" DD_CREDENTIAL_AES_256_KEY: "${DD_CREDENTIAL_AES_256_KEY:-&91a*agLqesc*0DJ+2*bAbsUZfR*4nLw}" DD_DATABASE_READINESS_TIMEOUT: "${DD_DATABASE_READINESS_TIMEOUT:-30}" @@ -65,12 +65,12 @@ services: condition: service_completed_successfully postgres: condition: service_started - redis: + valkey: condition: service_started entrypoint: ['/wait-for-it.sh', '${DD_DATABASE_HOST:-postgres}:${DD_DATABASE_PORT:-5432}', '-t', '30', '--', '/entrypoint-celery-beat.sh'] environment: DD_DATABASE_URL: ${DD_DATABASE_URL:-postgresql://defectdojo:defectdojo@postgres:5432/defectdojo} - DD_CELERY_BROKER_URL: ${DD_CELERY_BROKER_URL:-redis://redis:6379/0} + DD_CELERY_BROKER_URL: ${DD_CELERY_BROKER_URL:-redis://valkey:6379/0} DD_SECRET_KEY: "${DD_SECRET_KEY:-hhZCp@D28z!n@NED*yB!ROMt+WzsY*iq}" DD_CREDENTIAL_AES_256_KEY: "${DD_CREDENTIAL_AES_256_KEY:-&91a*agLqesc*0DJ+2*bAbsUZfR*4nLw}" DD_DATABASE_READINESS_TIMEOUT: "${DD_DATABASE_READINESS_TIMEOUT:-30}" @@ -85,12 +85,12 @@ services: condition: service_completed_successfully postgres: condition: service_started - redis: + valkey: condition: service_started entrypoint: ['/wait-for-it.sh', '${DD_DATABASE_HOST:-postgres}:${DD_DATABASE_PORT:-5432}', '-t', '30', '--', '/entrypoint-celery-worker.sh'] environment: DD_DATABASE_URL: ${DD_DATABASE_URL:-postgresql://defectdojo:defectdojo@postgres:5432/defectdojo} - DD_CELERY_BROKER_URL: ${DD_CELERY_BROKER_URL:-redis://redis:6379/0} + DD_CELERY_BROKER_URL: ${DD_CELERY_BROKER_URL:-redis://valkey:6379/0} DD_SECRET_KEY: "${DD_SECRET_KEY:-hhZCp@D28z!n@NED*yB!ROMt+WzsY*iq}" DD_CREDENTIAL_AES_256_KEY: "${DD_CREDENTIAL_AES_256_KEY:-&91a*agLqesc*0DJ+2*bAbsUZfR*4nLw}" DD_DATABASE_READINESS_TIMEOUT: "${DD_DATABASE_READINESS_TIMEOUT:-30}" @@ -127,12 +127,14 @@ services: POSTGRES_PASSWORD: ${DD_DATABASE_PASSWORD:-defectdojo} volumes: - defectdojo_postgres:/var/lib/postgresql/data - redis: - # Pinning to this version due to licensing constraints - image: redis:7.2.11-alpine@sha256:1a34bdba051ecd8a58ec8a3cc460acef697a1605e918149cc53d920673c1a0a7 + valkey: + image: valkey/valkey:7.2.11-alpine@sha256:7b2019b47ad58be661fa6eba5ea66106eadde03459387113aaed29a464a5876b volumes: + # we keep using the redis volume as renaming is not possible and copying data over + # would require steps during downtime or complex commands in the intializer - defectdojo_redis:/data volumes: defectdojo_postgres: {} defectdojo_media: {} defectdojo_redis: {} + diff --git a/docs/content/en/open_source/installation/architecture.md b/docs/content/en/open_source/installation/architecture.md index cd3d70710d6..d3085609844 100644 --- a/docs/content/en/open_source/installation/architecture.md +++ b/docs/content/en/open_source/installation/architecture.md @@ -21,7 +21,9 @@ dynamic content. ## Message Broker The application server sends tasks to a [Message Broker](https://docs.celeryq.dev/en/stable/getting-started/backends-and-brokers/index.html) -for asynchronous execution. Currently, only [Redis](https://github.com/redis/redis) is supported as a broker. +for asynchronous execution. Currently, only [Valkey](https://valkey.io/) is supported as a broker in the docker compose setup. +The Helm chart still uses [Redis](https://github.com/redis/redis) is supported as a broker, but will be migrated to Valkey shortly. + ## Celery Worker diff --git a/docs/content/en/open_source/upgrading/2.52.md b/docs/content/en/open_source/upgrading/2.52.md index 04a206c74ff..20eef3fb214 100644 --- a/docs/content/en/open_source/upgrading/2.52.md +++ b/docs/content/en/open_source/upgrading/2.52.md @@ -2,7 +2,7 @@ title: 'Upgrading to DefectDojo Version 2.52.x' toc_hide: true weight: -20251006 -description: MobSF parsers & Helm chart changes. +description: Replaced Redis with Valkey & Helm chart changes & MobSF parser merge --- ## Fix UI overwriting service field from parsers @@ -23,9 +23,34 @@ See [PR 13517](https://github.com/DefectDojo/django-DefectDojo/pull/13517) for m A bug was fixed in the `UNIQUE_ID_OR_HASH_CODE` algorithm where it stopped processing candidate findings with equal `unique_id_from_tool` or `hash_code` value. Strictly speaking this is not a breaking change, but we wanted to make you aware that you can see more (better) more deduplicatation for parsers using this algorithm. -## Merge of MobSF parsers +## Valkey in `docker compose` -Mobsfscan Scan" has been merged into the "MobSF Scan" parser. The "Mobsfscan Scan" scan_type has been retained to keep deduplication working for existing Tests, but users are encouraged to move to the "MobSF Scan" scan_type. +Since the license change at Redis the fork ValKey has become widely popular and is backed by industry giants such as AWS. AWS is advising to use ValKey over Redis and is using lower prices for ValKey compared to Redis. + +Defect Dojo 2.52 now uses ValKey as a message broker. Teh existing redit volume can be used by Valkey, so this is just a drop in replacement. + +If you want to know more or have a setup where you cannot just re-use the existing volume, please visit https://valkey.io/topics/migration/. + +When you shutdown Defect Dojo to perform the upgrade, the celery tasks that are in the queue are stored to disk. After the upgrade, the celery workers will process these tasks as normal. + +If you want to be 110% sure no tasks will be lost you could perform the upgrade in two steps: + +1) Stop nginx, uwsgi, celerybeat to prevent new tasks from being created: + +`docker compose down nginx, uwsgi, celerybeat` + +2) Observe the Redis queue and/or the logs of the celeryworker(s) and wait until all tasks are finished: + +`docker compose exec redis redis-cli llen celery` -- should output 0 +`docker compose logs celeryworker` -- should stop outputting new task logs + +3) Stop the remaining services: + +`docker compose down` + +4) Continue the upgrade as normal per the [upgrade guide](upgrading_guide) +`docker compose pull` +`docker compose up -d` ## Helm Chart Changes @@ -62,3 +87,10 @@ The following Helm chart values have been modified in this release: - **Extra annotations**: Now we can add common annotations to all resources. There are other instructions for upgrading to 2.52.x. Check the [Release Notes](https://github.com/DefectDojo/django-DefectDojo/releases/tag/2.52.0) for the contents of the release. + +## Merge of MobSF parsers + +Mobsfscan Scan" has been merged into the "MobSF Scan" parser. The "Mobsfscan Scan" scan_type has been retained to keep deduplication working for existing Tests, but users are encouraged to move to the "MobSF Scan" scan_type. + +## Release notes +Check the [Release Notes](https://github.com/DefectDojo/django-DefectDojo/releases/tag/2.52.0) for the contents of the release. diff --git a/run-integration-tests.sh b/run-integration-tests.sh index 953fbbab31f..a07fe15e629 100755 --- a/run-integration-tests.sh +++ b/run-integration-tests.sh @@ -41,7 +41,7 @@ while [[ $# -gt 0 ]]; do esac done -echo "Running docker compose unit tests with profile postgres-redis and test case $TEST_CASE ..." +echo "Running docker compose unit tests and test case $TEST_CASE ..." # Compose V2 integrates compose functions into the Docker platform, # continuing to support most of the previous docker-compose features @@ -50,8 +50,8 @@ echo "Running docker compose unit tests with profile postgres-redis and test cas echo "Building images..." ./docker/setEnv.sh integration_tests docker compose build -echo "Setting up DefectDojo with Postgres and Redis..." -DD_INTEGRATION_TEST_FILENAME="$TEST_CASE" docker compose -d postgres nginx celerybeat celeryworker mailhog uwsgi redis +echo "Setting up DefectDojo" +DD_INTEGRATION_TEST_FILENAME="$TEST_CASE" docker compose -d postgres nginx celerybeat celeryworker mailhog uwsgi valkey echo "Initializing DefectDojo..." DD_INTEGRATION_TEST_FILENAME="$TEST_CASE" docker compose --exit-code-from initializer initializer echo "Running the integration tests..."