feat(admin): enhance celery queue status UI

- Split status display into separate Redis Broker and Celery Worker badges
- Add loading spinners on page load and refresh for all status indicators
- Add per-task queue breakdown table behind a 'View Details' button (O(N)
  scan with warning); shows task name, count, oldest/newest queue position,
  and expiry timestamps with human-readable time-left
- Add per-row purge button to remove all queued tasks by task name
- Add global queue purge and per-task purge API endpoints
- Move Celery settings table into its own panel section

Author: valentijnscholten

dojo/api_v2/serializers.py

@@ -3104,12 +3104,24 @@ def validate(self, data):
 
 class CeleryStatusSerializer(serializers.Serializer):
     worker_status = serializers.BooleanField(read_only=True)
+    broker_status = serializers.BooleanField(read_only=True)
     queue_length = serializers.IntegerField(allow_null=True, read_only=True)
     task_time_limit = serializers.IntegerField(allow_null=True, read_only=True)
     task_soft_time_limit = serializers.IntegerField(allow_null=True, read_only=True)
     task_default_expires = serializers.IntegerField(allow_null=True, read_only=True)
 
 
+class CeleryQueueTaskDetailSerializer(serializers.Serializer):
+    task_name = serializers.CharField(read_only=True)
+    count = serializers.IntegerField(read_only=True)
+    oldest_position = serializers.IntegerField(read_only=True)
+    newest_position = serializers.IntegerField(read_only=True)
+    oldest_eta = serializers.CharField(allow_null=True, read_only=True)
+    newest_eta = serializers.CharField(allow_null=True, read_only=True)
+    earliest_expires = serializers.CharField(allow_null=True, read_only=True)
+    latest_expires = serializers.CharField(allow_null=True, read_only=True)
+
+
 class FindingNoteSerializer(serializers.Serializer):
     note_id = serializers.IntegerField()
 

dojo/api_v2/views.py

@@ -184,12 +184,14 @@
 from dojo.utils import (
     async_delete,
     generate_file_response,
+    get_celery_queue_details,
     get_celery_queue_length,
     get_celery_worker_status,
     get_setting,
     get_system_setting,
     process_tag_notifications,
     purge_celery_queue,
+    purge_celery_queue_by_task_name,
 )
 
 logger = logging.getLogger(__name__)
@@ -3278,9 +3280,11 @@ class CeleryStatusView(APIView):
     queryset = System_Settings.objects.none()
 
     def get(self, request):
+        queue_length = get_celery_queue_length()
         data = {
             "worker_status": get_celery_worker_status(),
-            "queue_length": get_celery_queue_length(),
+            "broker_status": queue_length is not None,
+            "queue_length": queue_length,
             "task_time_limit": getattr(settings, "CELERY_TASK_TIME_LIMIT", None),
             "task_soft_time_limit": getattr(settings, "CELERY_TASK_SOFT_TIME_LIMIT", None),
             "task_default_expires": getattr(settings, "CELERY_TASK_DEFAULT_EXPIRES", None),
@@ -3307,6 +3311,45 @@ def post(self, request):
         return Response({"purged": purged})
 
 
+@extend_schema(
+    responses=serializers.CeleryQueueTaskDetailSerializer(many=True),
+    summary="Get per-task breakdown of the Celery queue",
+    description=(
+        "Scans every message in the queue (O(N)) and returns task name, count, and "
+        "oldest/newest queue positions. May be slow for large queues."
+    ),
+)
+class CeleryQueueDetailsView(APIView):
+    permission_classes = (permissions.IsSuperUser, DjangoModelPermissions)
+    queryset = System_Settings.objects.none()
+
+    def get(self, request):
+        details = get_celery_queue_details()
+        if details is None:
+            return Response({"error": "Unable to read queue details."}, status=503)
+        return Response(serializers.CeleryQueueTaskDetailSerializer(details, many=True).data)
+
+
+@extend_schema(
+    request={"application/json": {"type": "object", "properties": {"task_name": {"type": "string"}}, "required": ["task_name"]}},
+    responses={200: {"type": "object", "properties": {"purged": {"type": "integer"}}}},
+    summary="Purge all queued tasks with a given task name",
+    description="Removes all pending tasks matching the given task name from the default Celery queue.",
+)
+class CeleryQueueTaskPurgeView(APIView):
+    permission_classes = (permissions.IsSuperUser, DjangoModelPermissions)
+    queryset = System_Settings.objects.none()
+
+    def post(self, request):
+        task_name = request.data.get("task_name", "").strip()
+        if not task_name:
+            return Response({"error": "task_name is required."}, status=400)
+        purged = purge_celery_queue_by_task_name(task_name)
+        if purged is None:
+            return Response({"error": "Unable to purge tasks."}, status=503)
+        return Response({"purged": purged})
+
+
 # Authorization: superuser
 @extend_schema_view(**schema_with_prefetch())
 class NotificationsViewSet(

dojo/templates/dojo/system_status.html

@@ -9,18 +9,43 @@ <h3>System Status</h3>
         <div id="celery-status-panel" class="col-md-5">
             <div class="panel panel-default">
                 <div class="panel-heading">
-                    <h4>Celery <span id="celery-status-badge" class="label label-default">Loading...</span></h4>
+                    <h4>Celery Status</h4>
                 </div>
-                <div class="panel-body text-left" id="celery-status-msg">—</div>
                 <div class="panel-body text-left">
-                    <span id="celery-queue-msg">—</span>
+                    <table class="table table-condensed" style="margin-bottom: 0">
+                        <tbody>
+                            <tr>
+                                <td>Redis Broker</td>
+                                <td>
+                                    <i id="broker-loading" class="fa-solid fa-spinner fa-spin text-muted"></i>
+                                    <span id="broker-status-badge" class="label label-default" style="display:none"></span>
+                                </td>
+                            </tr>
+                            <tr>
+                                <td>Celery Worker</td>
+                                <td>
+                                    <i id="worker-loading" class="fa-solid fa-spinner fa-spin text-muted"></i>
+                                    <span id="worker-status-badge" class="label label-default" style="display:none"></span>
+                                </td>
+                            </tr>
+                        </tbody>
+                    </table>
+                </div>
+                <div class="panel-body text-left">
+                    <span id="celery-queue-msg"><i class="fa-solid fa-spinner fa-spin text-muted"></i></span>
                     <div style="margin-top: 8px;">
                         <button id="purge-queue-btn" class="btn btn-danger btn-xs" disabled title="Loading...">
                             Purge queue
                         </button>
                         <button id="refresh-status-btn" class="btn btn-default btn-xs" style="margin-left: 6px;">
                             <span class="glyphicon glyphicon-refresh"></span> Refresh
                         </button>
+                        <button id="view-details-btn" class="btn btn-default btn-xs" disabled title="Loading..." style="margin-left: 6px;">
+                            View Details
+                        </button>
+                        <span class="text-muted" style="font-size: 0.85em; margin-left: 6px;">
+                            Inspects every message in the queue &mdash; may be slow for large queues.
+                        </span>
                     </div>
                     <p class="text-muted" style="margin-top: 8px; font-size: 0.9em">
                         <strong>Note:</strong> Purging the queue removes pending tasks that have not yet been executed,
@@ -29,6 +54,32 @@ <h4>Celery <span id="celery-status-badge" class="label label-default">Loading...
                         <code>python manage.py dedupe</code>
                     </p>
                 </div>
+                <div class="panel-body text-left">
+                    <div id="queue-details-container" style="display: none;">
+                        <table class="table table-condensed table-bordered table-hover" style="margin-bottom: 0; font-size: 0.9em;">
+                            <thead>
+                                <tr>
+                                    <th>#</th>
+                                    <th>Task name</th>
+                                    <th>Count</th>
+                                    <th>Oldest (queue pos.)</th>
+                                    <th>Newest (queue pos.)</th>
+                                    <th>Expires</th>
+                                    <th></th>
+                                </tr>
+                            </thead>
+                            <tbody id="queue-details-tbody"></tbody>
+                        </table>
+                        <p class="text-muted" style="font-size: 0.8em; margin-top: 4px;">
+                            Position 1 = oldest task in queue. ETA shown in parentheses when set.
+                        </p>
+                    </div>
+                </div>
+            </div>
+            <div class="panel panel-default">
+                <div class="panel-heading">
+                    <h4>Celery Settings</h4>
+                </div>
                 <div class="panel-body text-left">
                     <table class="table table-condensed" style="margin-bottom: 0">
                         <thead><tr><th>Setting</th><th>Value</th></tr></thead>
@@ -56,24 +107,32 @@ <h4>Celery <span id="celery-status-badge" class="label label-default">Loading...
         (function() {
             var purgeUrl = "{% url 'celery_queue_purge_api' %}";
             var statusUrl = "{% url 'celery_status_api' %}";
+            var detailsUrl = "{% url 'celery_queue_details_api' %}";
+            var taskPurgeUrl = "{% url 'celery_queue_task_purge_api' %}";
 
-            function renderCeleryStatus(data) {
-                var badge = $('#celery-status-badge');
-                if (data.worker_status) {
-                    badge.text('Running').removeClass('label-default label-danger').addClass('label-success');
-                    $('#celery-status-msg').text('Celery is processing tasks.');
+            function setStatusBadge(loadingId, badgeId, isOk, okText, failText) {
+                $('#' + loadingId).hide();
+                var badge = $('#' + badgeId).show();
+                if (isOk) {
+                    badge.text(okText).removeClass('label-default label-danger').addClass('label-success');
                 } else {
-                    badge.text('Not Running').removeClass('label-default label-success').addClass('label-danger');
-                    $('#celery-status-msg').text('Celery does not appear to be running.');
+                    badge.text(failText).removeClass('label-default label-success').addClass('label-danger');
                 }
+            }
+
+            function renderCeleryStatus(data) {
+                setStatusBadge('broker-loading', 'broker-status-badge', data.broker_status, 'Reachable', 'Unreachable');
+                setStatusBadge('worker-loading', 'worker-status-badge', data.worker_status, 'Running', 'Not Running');
 
                 var qLen = data.queue_length;
                 if (qLen === null) {
                     $('#celery-queue-msg').text('It is not possible to identify the number of waiting tasks.');
                     $('#purge-queue-btn').prop('disabled', true).attr('title', 'Broker unreachable');
+                    $('#view-details-btn').prop('disabled', true).attr('title', 'Broker unreachable');
                 } else {
                     $('#celery-queue-msg').text(qLen + ' task(s) waiting to be processed.');
                     $('#purge-queue-btn').prop('disabled', false).removeAttr('title');
+                    $('#view-details-btn').prop('disabled', false).removeAttr('title');
                 }
 
                 function humanDuration(seconds) {
@@ -101,13 +160,78 @@ <h4>Celery <span id="celery-status-badge" class="label label-default">Loading...
                 });
             }
 
+            function resetToLoadingState() {
+                ['broker-loading', 'worker-loading'].forEach(function(id) { $('#' + id).show(); });
+                ['broker-status-badge', 'worker-status-badge'].forEach(function(id) { $('#' + id).hide(); });
+                $('#celery-queue-msg').html('<i class="fa-solid fa-spinner fa-spin text-muted"></i>');
+                $('#purge-queue-btn').prop('disabled', true).attr('title', 'Loading...');
+                $('#view-details-btn').prop('disabled', true).attr('title', 'Loading...');
+            }
+
             function loadCeleryStatus() {
+                resetToLoadingState();
                 $.get(statusUrl).done(renderCeleryStatus).fail(function() {
-                    $('#celery-status-badge').text('Error').removeClass('label-default label-success').addClass('label-danger');
-                    $('#celery-status-msg').text('Failed to load Celery status.');
+                    ['broker-loading', 'worker-loading'].forEach(function(id) { $('#' + id).hide(); });
+                    setStatusBadge('broker-loading', 'broker-status-badge', false, '', 'Error');
+                    setStatusBadge('worker-loading', 'worker-status-badge', false, '', 'Error');
+                    $('#celery-queue-msg').text('Failed to load Celery status.');
                 });
             }
 
+            var MONTHS = ['Jan','Feb','Mar','Apr','May','Jun','Jul','Aug','Sep','Oct','Nov','Dec'];
+            function formatExpiry(isoStr) {
+                if (!isoStr) return '—';
+                var d = new Date(isoStr);
+                var hh = ('0' + d.getHours()).slice(-2);
+                var mm = ('0' + d.getMinutes()).slice(-2);
+                var ss = ('0' + d.getSeconds()).slice(-2);
+                return d.getDate() + ' ' + MONTHS[d.getMonth()] + ' ' + hh + ':' + mm + ':' + ss;
+            }
+
+            function timeLeft(isoStr) {
+                if (!isoStr) return '';
+                var ms = new Date(isoStr) - Date.now();
+                if (ms <= 0) return ' <span class="label label-danger">expired</span>';
+                var s = Math.floor(ms / 1000);
+                var parts = [];
+                var d = Math.floor(s / 86400); if (d) { parts.push(d + 'd'); s -= d * 86400; }
+                var h = Math.floor(s / 3600);  if (h) { parts.push(h + 'h'); s -= h * 3600; }
+                var m = Math.floor(s / 60);    if (m) { parts.push(('0' + m).slice(-2) + 'm'); s -= m * 60; }
+                if (!parts.length) parts.push(s + 's');
+                return ' <span class="text-muted">(' + parts.join(' ') + ' left)</span>';
+            }
+
+            function renderQueueDetails(tasks) {
+                var tbody = $('#queue-details-tbody').empty();
+                if (tasks.length === 0) {
+                    tbody.append('<tr><td colspan="7" class="text-muted text-center">Queue is empty.</td></tr>');
+                } else {
+                    tasks.forEach(function(t, idx) {
+                        var oldest = '#' + t.oldest_position + (t.oldest_eta ? ' <span class="text-muted">(' + t.oldest_eta + ')</span>' : '');
+                        var newest = '#' + t.newest_position + (t.newest_eta ? ' <span class="text-muted">(' + t.newest_eta + ')</span>' : '');
+                        var expires = '—';
+                        if (t.earliest_expires || t.latest_expires) {
+                            expires = '<span class="text-muted">Oldest:</span> ' + formatExpiry(t.earliest_expires) + timeLeft(t.earliest_expires) +
+                                      '<br><span class="text-muted">Newest:</span> ' + formatExpiry(t.latest_expires) + timeLeft(t.latest_expires);
+                        }
+                        var safeTaskName = $('<span>').text(t.task_name).html();
+                        var purgeBtn = '<button class="btn btn-danger btn-xs task-purge-btn" data-task-name="' + safeTaskName + '">Purge</button>';
+                        tbody.append(
+                            '<tr>' +
+                            '<td>' + (idx + 1) + '</td>' +
+                            '<td><code>' + safeTaskName + '</code></td>' +
+                            '<td>' + t.count + '</td>' +
+                            '<td>' + oldest + '</td>' +
+                            '<td>' + newest + '</td>' +
+                            '<td>' + expires + '</td>' +
+                            '<td>' + purgeBtn + '</td>' +
+                            '</tr>'
+                        );
+                    });
+                }
+                $('#queue-details-container').show();
+            }
+
             $(function() {
                 loadCeleryStatus();
 
@@ -126,6 +250,41 @@ <h4>Celery <span id="celery-status-badge" class="label label-default">Loading...
                         loadCeleryStatus();
                     });
                 });
+
+                $('#queue-details-tbody').on('click', '.task-purge-btn', function() {
+                    var btn = $(this);
+                    var taskName = btn.data('task-name');
+                    if (!confirm('Purge all pending "' + taskName + '" tasks from the queue? This cannot be undone.')) return;
+                    btn.prop('disabled', true).html('<i class="fa-solid fa-spinner fa-spin"></i>');
+                    $.ajax({
+                        url: taskPurgeUrl,
+                        method: 'POST',
+                        contentType: 'application/json',
+                        data: JSON.stringify({task_name: taskName}),
+                        headers: {'X-CSRFToken': '{{ csrf_token }}'},
+                    }).done(function(data) {
+                        btn.closest('tr').fadeOut(300, function() { $(this).remove(); });
+                        var remaining = parseInt($('#celery-queue-msg').text()) - data.purged;
+                        if (!isNaN(remaining)) {
+                            $('#celery-queue-msg').text(remaining + ' task(s) waiting to be processed.');
+                        }
+                    }).fail(function() {
+                        alert('Purge failed. Check server logs.');
+                        btn.prop('disabled', false).text('Purge');
+                    });
+                });
+
+                $('#view-details-btn').on('click', function() {
+                    var btn = $(this);
+                    btn.prop('disabled', true).html('<i class="fa-solid fa-spinner fa-spin"></i> Loading...');
+                    $.get(detailsUrl).done(function(data) {
+                        renderQueueDetails(data);
+                        btn.prop('disabled', false).text('View Details');
+                    }).fail(function() {
+                        alert('Failed to load queue details. Check server logs.');
+                        btn.prop('disabled', false).text('View Details');
+                    });
+                });
             });
         })();
     </script>

dojo/urls.py

@@ -15,7 +15,9 @@
     AnnouncementViewSet,
     AppAnalysisViewSet,
     BurpRawRequestResponseViewSet,
+    CeleryQueueDetailsView,
     CeleryQueuePurgeView,
+    CeleryQueueTaskPurgeView,
     CeleryStatusView,
     ConfigurationPermissionViewSet,
     CredentialsMappingViewSet,
@@ -246,6 +248,8 @@
     re_path(r"^{}api/v2/user_profile/".format(get_system_setting("url_prefix")), UserProfileView.as_view(), name="user_profile"),
     re_path(r"^{}api/v2/celery/status/$".format(get_system_setting("url_prefix")), CeleryStatusView.as_view(), name="celery_status_api"),
     re_path(r"^{}api/v2/celery/queue/purge/$".format(get_system_setting("url_prefix")), CeleryQueuePurgeView.as_view(), name="celery_queue_purge_api"),
+    re_path(r"^{}api/v2/celery/queue/details/$".format(get_system_setting("url_prefix")), CeleryQueueDetailsView.as_view(), name="celery_queue_details_api"),
+    re_path(r"^{}api/v2/celery/queue/task/purge/$".format(get_system_setting("url_prefix")), CeleryQueueTaskPurgeView.as_view(), name="celery_queue_task_purge_api"),
 ]
 
 if hasattr(settings, "API_TOKENS_ENABLED") and hasattr(settings, "API_TOKEN_AUTH_ENDPOINT_ENABLED"):