add celery worker resilience for database connection timeouts

- close stale db connections before each task via task_prerun signal
  (mirrors what django does for http requests)
- add Restart=on-failure to worker and beat systemd services
- add --loglevel info to worker service for diagnostics

Author: furlongm

etc/systemd/system/patchman-celery-beat.service

@@ -5,6 +5,8 @@ After=network-online.target
 
 [Service]
 Type=simple
+Restart=on-failure
+RestartSec=10
 User=patchman
 Group=patchman
 Environment="REDIS_HOST=127.0.0.1"

etc/systemd/system/patchman-celery-worker@.service

@@ -5,6 +5,8 @@ After=network-online.target
 
 [Service]
 Type=simple
+Restart=on-failure
+RestartSec=10
 User=patchman
 Group=patchman
 Environment="REDIS_HOST=127.0.0.1"
@@ -19,6 +21,7 @@ ExecStart=/usr/bin/celery \
     --task-events \
     --pool ${CELERY_POOL_TYPE} \
     --concurrency ${CELERY_CONCURRENCY} \
+    --loglevel info \
     --hostname patchman-celery-worker%i@%%h
 
 [Install]

patchman/celery.py

@@ -17,10 +17,24 @@
 import os
 
 from celery import Celery
+from celery.signals import task_prerun
 
 os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'patchman.settings')  # noqa
 from django.conf import settings  # noqa
 
 app = Celery('patchman')
 app.config_from_object('django.conf:settings', namespace='CELERY')
 app.autodiscover_tasks()
+
+
+@task_prerun.connect
+def close_stale_connections(**kwargs):
+    """Close stale DB connections before each task.
+
+    Django does this automatically for HTTP requests but not for Celery
+    tasks. Without this, long-lived workers hit 'server has gone away'
+    (MySQL) or 'server closed the connection unexpectedly' (PostgreSQL)
+    when the DB server drops idle connections.
+    """
+    from django import db
+    db.close_old_connections()