diff --git a/etc/systemd/system/patchman-celery-beat.service b/etc/systemd/system/patchman-celery-beat.service index c9bce722..79a5bc5c 100644 --- a/etc/systemd/system/patchman-celery-beat.service +++ b/etc/systemd/system/patchman-celery-beat.service @@ -5,6 +5,8 @@ After=network-online.target [Service] Type=simple +Restart=on-failure +RestartSec=10 User=patchman Group=patchman Environment="REDIS_HOST=127.0.0.1" diff --git a/etc/systemd/system/patchman-celery-worker@.service b/etc/systemd/system/patchman-celery-worker@.service index b2d6f6b7..7e10bd85 100644 --- a/etc/systemd/system/patchman-celery-worker@.service +++ b/etc/systemd/system/patchman-celery-worker@.service @@ -5,6 +5,8 @@ After=network-online.target [Service] Type=simple +Restart=on-failure +RestartSec=10 User=patchman Group=patchman Environment="REDIS_HOST=127.0.0.1" @@ -19,6 +21,7 @@ ExecStart=/usr/bin/celery \ --task-events \ --pool ${CELERY_POOL_TYPE} \ --concurrency ${CELERY_CONCURRENCY} \ + --loglevel info \ --hostname patchman-celery-worker%i@%%h [Install] diff --git a/patchman/celery.py b/patchman/celery.py index c47f994d..a1f3188e 100644 --- a/patchman/celery.py +++ b/patchman/celery.py @@ -17,6 +17,7 @@ import os from celery import Celery +from celery.signals import task_prerun os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'patchman.settings') # noqa from django.conf import settings # noqa @@ -24,3 +25,16 @@ app = Celery('patchman') app.config_from_object('django.conf:settings', namespace='CELERY') app.autodiscover_tasks() + + +@task_prerun.connect +def close_stale_connections(**kwargs): + """Close stale DB connections before each task. + + Django does this automatically for HTTP requests but not for Celery + tasks. Without this, long-lived workers hit 'server has gone away' + (MySQL) or 'server closed the connection unexpectedly' (PostgreSQL) + when the DB server drops idle connections. + """ + from django import db + db.close_old_connections()