feat(broker): Add start-up checker (#13931)

Signed-off-by: kiblik <5609770+kiblik@users.noreply.github.com>

Author: kiblik

Dockerfile.django-alpine

@@ -78,6 +78,7 @@ COPY \
   docker/wait-for-it.sh \
   docker/secret-file-loader.sh \
   docker/reach_database.sh \
+  docker/reach_broker.sh \
   docker/certs/* \
   /
 COPY wsgi.py manage.py docker/unit-tests.sh ./

Dockerfile.django-debian

@@ -81,6 +81,7 @@ COPY \
   docker/wait-for-it.sh \
   docker/secret-file-loader.sh \
   docker/reach_database.sh \
+  docker/reach_broker.sh \
   docker/certs/* \
   /
 COPY wsgi.py manage.py docker/unit-tests.sh ./

Dockerfile.integration-tests-debian

@@ -73,6 +73,7 @@ COPY --from=openapitools /opt/openapi-generator/modules/openapi-generator-cli/ta
 COPY docker/wait-for-it.sh \
   docker/secret-file-loader.sh \
   docker/reach_database.sh \
+  docker/reach_broker.sh \
   docker/entrypoint-integration-tests.sh \
   /
 

docker/entrypoint-celery-beat.sh

@@ -4,6 +4,7 @@ set -e  # needed to handle "exit" correctly
 
 . /secret-file-loader.sh
 . /reach_database.sh
+. /reach_broker.sh
 
 umask 0002
 
@@ -23,6 +24,7 @@ if [ "$NUM_FILES" -gt 0 ]; then
 fi
 
 wait_for_database_to_be_reachable
+wait_for_broker_to_be_reachable
 echo
 
 # do the check with Django stack

docker/entrypoint-celery-worker-dev.sh

@@ -7,8 +7,10 @@ set -e  # needed to handle "exit" correctly
 
 . /secret-file-loader.sh
 . /reach_database.sh
+. /reach_broker.sh
 
 wait_for_database_to_be_reachable
+wait_for_broker_to_be_reachable
 echo
 
 if [ "${DD_CELERY_WORKER_POOL_TYPE}" = "prefork" ]; then

docker/entrypoint-celery-worker.sh

@@ -7,6 +7,7 @@ set -e  # needed to handle "exit" correctly
 
 . /secret-file-loader.sh
 . /reach_database.sh
+. /reach_broker.sh
 
 # Allow for bind-mount multiple settings.py overrides
 FILES=$(ls /app/docker/extra_settings/* 2>/dev/null || true)
@@ -22,6 +23,7 @@ if [ "$NUM_FILES" -gt 0 ]; then
 fi
 
 wait_for_database_to_be_reachable
+wait_for_broker_to_be_reachable
 echo
 
 if [ "${DD_CELERY_WORKER_POOL_TYPE}" = "prefork" ]; then

docker/entrypoint-uwsgi-dev.sh

@@ -4,8 +4,10 @@ set -e  # needed to handle "exit" correctly
 
 . /secret-file-loader.sh
 . /reach_database.sh
+. /reach_broker.sh
 
 wait_for_database_to_be_reachable
+wait_for_broker_to_be_reachable
 echo
 
 cd /app || exit

docker/entrypoint-uwsgi.sh

@@ -3,6 +3,7 @@ set -e  # needed to handle "exit" correctly
 
 . /secret-file-loader.sh
 . /reach_database.sh
+. /reach_broker.sh
 
 # Allow for bind-mount multiple settings.py overrides
 FILES=$(ls /app/docker/extra_settings/* 2>/dev/null || true)
@@ -18,6 +19,7 @@ if [ "$NUM_FILES" -gt 0 ]; then
 fi
 
 wait_for_database_to_be_reachable
+wait_for_broker_to_be_reachable
 echo
 
 umask 0002

docker/reach_broker.sh

@@ -0,0 +1,30 @@
+#!/bin/bash
+
+wait_for_broker_to_be_reachable() {
+    echo -n "Waiting for broker to be reachable "
+    failure_count=0
+    DD_BROKER_READINESS_TIMEOUT=${DD_BROKER_READINESS_TIMEOUT:-10}
+    while true;
+    do 
+        set +e
+        celery --app=dojo status 2>/dev/null >/dev/null
+        BROKER_TEST=$?
+        set -e
+        if [[ "$BROKER_TEST" == "0" ]]; then
+            echo "Broker test was successful. Broker and at least one worker is connected."
+            break
+        fi
+        if [[ "$BROKER_TEST" == "69" ]]; then
+            echo "Broker test was successful. Broker is up. No worker is connected (but we are not testing that here)."
+            break
+        fi
+        echo -n "."
+        failure_count=$((failure_count + 1)) 
+        if [ $DD_BROKER_READINESS_TIMEOUT = $failure_count ]; then
+            echo "Broker test was failed:"
+            # One more time with output
+            celery --app=dojo status
+            exit 1
+        fi
+    done
+}