Async exporting of applications (#4471)

This PR makes the generation of submission exports async as (especially
in production) the generation of these files can take a long time and
cause a request timeout. This allows the user to do other tasks while
the file generates, and will notify them when the file is ready for
download via a new task on their dashboard or a green indicator on the
submission export button in the "All Submissions" view.

**Changes TLDR**:
- Added tasks to `hypha.apply.funds` module to handle the generation of
the submissions CSV
- Added a `SubmissionExportManager` model that keeps track of the status
of the generation & the finalized data
     - Each user can only have one active model/download
- Added a check in the nprogress event to allow for the disabling of the
progress bar while polling
- Preserved all sync functionality for users that don't need the
background worker, which is the default behavior

This is a WIP that should be completed early next week, but wanted to
get eyes on it early for any critiques. Steps left:
- [x] Add unit tests
- [x] Make a management command to remove `SubmissionExportManager`s
older than a day
- [x] Error handling for a failed generation
- [x] Triggering updating of  `My Tasks` when downloading from dashboard

**Running locally**
1. Get a Redis instance setup locally (I used docker, ie. `docker run -d
-p 6379:6379 redis`)
2. Install requirements
3. In a separate terminal from the running `make serve` (but with same
environment vars), run `celery -A hypha.celery worker -E
--loglevel=info` to start celery

## Demo

*Generating and downloading the exported CSV*


https://github.com/user-attachments/assets/71758bf1-2a0c-461f-9a4a-6942bf77b694

*Failed generation of a CSV*


https://github.com/user-attachments/assets/bb3029b2-0eb2-471a-930e-d8c85ff4ed6f

Author: wes-otf

Procfile

@@ -1,3 +1,3 @@
 release: python manage.py migrate --noinput && python manage.py clear_cache --cache=default && python manage.py sync_roles
 web: gunicorn hypha.wsgi:application --log-file -
-worker: celery --app=hypha.celery worker --autoscale=6,2 --events
+worker: celery --app=hypha.celery worker --autoscale=6,2 --events

hypha/__init__.py

@@ -0,0 +1,3 @@
+from .celery import app as celery_app
+
+__all__ = ("celery_app",)

hypha/apply/funds/management/commands/export_manager_cleanup.py

@@ -0,0 +1,86 @@
+import argparse
+from datetime import timedelta
+
+from django.core.management.base import BaseCommand
+from django.db import transaction
+from django.utils import timezone
+
+from hypha.apply.funds.models.utils import SubmissionExportManager
+
+
+def check_not_negative(value) -> int:
+    """Used to validate `older_than_days` argument
+
+    Args:
+        value: Argument to be validated
+
+    Returns:
+        int: Valid non-negative value
+
+    Raises:
+        argparse.ArgumentTypeError: if not non-negative integer
+    """
+    try:
+        ivalue = int(value)
+    except ValueError:
+        ivalue = -1
+
+    if ivalue < 0:
+        raise argparse.ArgumentTypeError(
+            f'"{value}" is an invalid non-negative integer value'
+        )
+    return ivalue
+
+
+class Command(BaseCommand):
+    help = "Delete all generated exports older than the specified time (in days)"
+
+    def add_arguments(self, parser):
+        parser.add_argument(
+            "older_than_days",
+            action="store",
+            type=check_not_negative,
+            help="Time in days to delete exports older than",
+        )
+        parser.add_argument(
+            "--noinput",
+            "--no-input",
+            action="store_false",
+            dest="interactive",
+            help="Do not prompt the user for confirmation",
+            required=False,
+        )
+
+    @transaction.atomic
+    def handle(self, *args, **options):
+        interactive = options["interactive"]
+        older_than = options["older_than_days"]
+
+        older_than_date = timezone.now() - timedelta(days=older_than)
+
+        old_exports = SubmissionExportManager.objects.filter(
+            created_time__lte=older_than_date
+        )
+
+        export_count = old_exports.count()
+
+        if not export_count:
+            self.stdout.write(
+                f"No exports older than {older_than} day{'s' if older_than > 1 else ''} exist."
+            )
+            return
+
+        if interactive:
+            confirm = input(
+                f"This action will permanently delete {export_count} generated export{'s' if export_count != 1 else ''}.\nAre you sure you want to do this?\n\nType 'yes' to continue, or 'no' to cancel: "
+            )
+        else:
+            confirm = "yes"
+
+        if confirm == "yes":
+            old_exports.delete()
+            self.stdout.write(
+                f"{export_count} generated export{'s' if export_count != 1 else ''} deleted."
+            )
+        else:
+            self.stdout.write("Deletion cancelled.")

hypha/apply/funds/migrations/0124_submissionexportmanager.py

@@ -0,0 +1,52 @@
+# Generated by Django 4.2.20 on 2025-03-24 21:29
+
+from django.conf import settings
+from django.db import migrations, models
+import django.db.models.deletion
+
+
+class Migration(migrations.Migration):
+    dependencies = [
+        migrations.swappable_dependency(settings.AUTH_USER_MODEL),
+        ("funds", "0123_help_text_rich_text"),
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name="SubmissionExportManager",
+            fields=[
+                (
+                    "id",
+                    models.AutoField(
+                        auto_created=True,
+                        primary_key=True,
+                        serialize=False,
+                        verbose_name="ID",
+                    ),
+                ),
+                ("export_data", models.TextField()),
+                ("created_time", models.DateTimeField(auto_now_add=True)),
+                ("completed_time", models.DateTimeField(null=True)),
+                (
+                    "status",
+                    models.CharField(
+                        choices=[
+                            ("error", "Failed"),
+                            ("success", "Success"),
+                            ("generating", "In Progress"),
+                        ],
+                        default="generating",
+                    ),
+                ),
+                ("total_export", models.IntegerField(null=True)),
+                (
+                    "user",
+                    models.ForeignKey(
+                        limit_choices_to={"groups__name": "Staff", "is_active": True},
+                        on_delete=django.db.models.deletion.CASCADE,
+                        to=settings.AUTH_USER_MODEL,
+                    ),
+                ),
+            ],
+        ),
+    ]

hypha/apply/funds/models/utils.py

@@ -2,6 +2,7 @@
 from django.db import models
 from django.shortcuts import redirect
 from django.urls import reverse
+from django.utils import timezone
 from django.utils.translation import gettext_lazy as _
 from wagtail.admin.panels import (
     FieldPanel,
@@ -200,3 +201,52 @@ def send_mail(self, submission):
     ]
 
     email_tab = ObjectList(email_confirmation_panels, heading=_("Confirmation email"))
+
+
+# Managing async submission exports
+
+STATUS_ERROR = "error"
+STATUS_SUCCESS = "success"
+STATUS_GENERATING = "generating"
+
+STATUS_CHOICES = [
+    (STATUS_ERROR, _("Failed")),
+    (STATUS_SUCCESS, _("Success")),
+    (STATUS_GENERATING, _("In Progress")),
+]
+
+
+class SubmissionExportManager(models.Model):
+    user = models.ForeignKey(
+        settings.AUTH_USER_MODEL,
+        limit_choices_to=LIMIT_TO_STAFF,
+        on_delete=models.CASCADE,
+    )
+
+    export_data = models.TextField()
+
+    created_time = models.DateTimeField(auto_now_add=True)
+
+    completed_time = models.DateTimeField(null=True)
+
+    status = models.CharField(choices=STATUS_CHOICES, default=STATUS_GENERATING)
+
+    total_export = models.IntegerField(null=True)
+
+    def set_completed_and_save(self) -> None:
+        """Sets the status to completed and saves the object"""
+        self.status = "success"
+        self.completed_time = timezone.now()
+        self.save()
+
+    def set_failed_and_save(self) -> None:
+        """Sets the status to error and saves the object"""
+        self.status = "error"
+        self.save()
+
+    def get_absolute_url(self) -> str:
+        """Returns the submissions all page where the user can download the file
+
+        Primarily used for tasks
+        """
+        return reverse("apply:submissions:list")

hypha/apply/funds/tasks.py

@@ -0,0 +1,70 @@
+from typing import List
+
+from celery import shared_task
+from django.conf import settings
+
+from hypha.apply.funds.models.submissions import ApplicationSubmission
+from hypha.apply.funds.models.utils import SubmissionExportManager
+from hypha.apply.funds.utils import export_submissions_to_csv
+from hypha.apply.todo.options import (
+    DOWNLOAD_SUBMISSIONS_EXPORT,
+    FAILED_SUBMISSIONS_EXPORT,
+)
+from hypha.apply.todo.views import add_task_to_user
+from hypha.apply.users.models import User
+
+
+@shared_task
+def generate_submission_csv(
+    qs_ids: List[int], request_user_id: int, base_uri: str
+) -> None:
+    """Celery task to generate a CSV file containing the given submission IDs
+
+    Integer IDs have to be used as QuerySets are not simple data types & can't be
+    passed to workers.
+
+    Updates the user's SubmissionExportManager object with status/final data, then
+    adds a download task to the user's `My Tasks` when completed.
+
+    Args:
+        qs_ids: A list of application IDs to generate the CSV export for
+        request_user_id: The ID of the user issuing the export request
+    """
+    try:
+        qs = ApplicationSubmission.objects.filter(id__in=qs_ids)
+        request_user = User.objects.get(pk=request_user_id)
+
+        # If the user already has an existing export, delete it to begin the new one
+        if current := SubmissionExportManager.objects.filter(user=request_user):
+            current.delete()
+
+        export_manager = SubmissionExportManager.objects.create(
+            user=request_user, total_export=len(qs_ids)
+        )
+        csv_string = export_submissions_to_csv(qs, base_uri)
+        export_manager.export_data = "".join(csv_string.readlines())
+        export_manager.set_completed_and_save()
+
+        user_task = DOWNLOAD_SUBMISSIONS_EXPORT
+
+    except Exception as exc:
+        # Update the status to failed
+        export_manager.set_failed_and_save()
+        user_task = FAILED_SUBMISSIONS_EXPORT
+
+        if settings.SENTRY_DSN:
+            # If sentry is enabled, pass the exception to sentry
+            from sentry_sdk import capture_exception
+
+            capture_exception(exc)
+        else:
+            # Otherwise re-raise it
+            raise exc
+    finally:
+        # When the generation is complete or failed, add a task to the user's dashboard (only if async)
+        if not settings.CELERY_TASK_ALWAYS_EAGER:
+            add_task_to_user(
+                code=user_task,
+                user=request_user,
+                related_obj=export_manager,
+            )

hypha/apply/funds/templates/submissions/all.html

@@ -174,15 +174,16 @@
         {% endif %}
 
         {% if can_export_submissions %}
-            <a
+            <button
                 class="py-1.5 px-2 rounded-sm border transition-colors hover:bg-gray-100 shadow-xs"
-                aria-label="{% trans 'Submissions: Download as CSV' %}"
-                href="{% modify_query "page" format="csv" %}"
-                onclick="return confirm('{% blocktrans %}Are you sure you want to download the submissions as a csv file? This file may contain sensitive information, so please handle it carefully.{% endblocktrans %}');"
-                data-tippy-content="{% trans 'Export as CSV' %}"
+                hx-get="{% url 'apply:submissions:submission-export-status' %}"
+                hx-swap="outerHTML"
+                hx-target="this"
+                hx-push-url="false"
+                hx-trigger="load"
             >
                 {% heroicon_mini "arrow-down-tray" %}
-            </a>
+            </button>
         {% endif %}
     </form>
 

hypha/apply/funds/templates/submissions/partials/export-submission-button.html

@@ -0,0 +1,80 @@
+{% load i18n querystrings heroicons %}
+
+
+{% if not_async %}
+    {% comment %} For sync uses: no polling, just a download after pressing the button {% endcomment %}
+    <a
+        class="relative py-1.5 px-2 rounded-sm border transition-colors hover:bg-gray-100 shadow-xs"
+        aria-label="{% trans 'Submissions: Export as CSV' %}"
+        href="{{ start_export_url }}"
+        data-tippy-content="{% trans 'Export as CSV' %}"
+        onclick="return confirm('{% blocktrans %}Are you sure you want to export the submissions as a CSV file? This file may contain sensitive information, so please handle it carefully.{% endblocktrans %}')"
+    >
+        {% heroicon_mini "arrow-down-tray" %}
+    </a>
+{% else %}
+    {% if generating %}
+        {% comment %} Disabled button used to indicate generation of the CSV is in progress {% endcomment %}
+        <button
+            class="py-1.5 px-2 rounded-sm border transition-colors animate-pulse shadow-xs"
+            aria-label="{% trans 'Submissions: Generating downloadable CSV' %}"
+            title="{% trans 'Generating downloadable CSV...' %}"
+            disabled
+            hx-get="{% url 'apply:submissions:submission-export-status' %}"
+            hx-swap="outerHTML"
+            hx-target="this"
+            hx-trigger="every {{ poll_time }}s"
+            hx-push-url="false"
+            hx-noprog
+        >
+            {% heroicon_mini "clock" %}
+        </button>
+    {% elif success %}
+        {% comment %} The final download link for the generated CSV {% endcomment %}
+        <a
+            class="relative py-1.5 px-2 rounded-sm border transition-colors hover:bg-gray-100 shadow-xs"
+            aria-label="{% trans 'Submissions: Download generated CSV' %}"
+            href="{% url 'apply:submissions:submission-export-download' %}"
+            data-tippy-content="{% trans 'Download generated CSV' %}"
+            hx-get="{% url 'apply:submissions:submission-export-status' %}"
+            hx-swap="outerHTML"
+            hx-target="this"
+            hx-trigger="every 2s"
+            hx-push-url="false"
+            hx-noprog
+        >
+            <span class="flex absolute top-0 right-0 -mt-1 -mr-1 size-3">
+                <span class="inline-flex absolute w-full h-full bg-green-400 rounded-full opacity-75 animate-ping"></span>
+                <span class="inline-flex relative bg-green-500 rounded-full size-3"></span>
+            </span>
+            {% heroicon_mini "arrow-down-tray" %}
+        </a>
+
+    {% else %}
+        {% comment %} Button that will begin the generation of the CSV, used to start a generation or retry a failed one {% endcomment %}
+        <button
+            class="relative py-1.5 px-2 rounded-sm border transition-colors cursor-pointer hover:bg-gray-100 shadow-xs"
+            aria-label="{% trans 'Submissions: Generate downloadable CSV' %}"
+            {% if failed %}
+                data-tippy-content="{% trans 'Generation failed, click to retry generating downloadable CSV' %}"
+            {% else %}
+                data-tippy-content="{% trans 'Generate downloadable CSV' %}"
+            {% endif %}
+            hx-get="{{ start_export_url }}"
+            hx-swap="outerHTML"
+            hx-target="this"
+            hx-push-url="false"
+            hx-confirm="{% trans 'Are you sure you want to export the submissions as a CSV file? This file may contain sensitive information, so please handle it carefully.' %}"
+        >
+            {% if not failed %}
+                {% heroicon_mini "arrow-down-tray" %}
+            {% else %}
+                <span class="flex absolute top-0 right-0 -mt-1 -mr-1 size-3">
+                    <span class="inline-flex absolute w-full h-full bg-red-400 rounded-full opacity-75 animate-ping"></span>
+                    <span class="inline-flex relative bg-red-500 rounded-full size-3"></span>
+                </span>
+                {% heroicon_mini "exclamation-circle" %}
+            {% endif %}
+        </button>
+    {% endif %}
+{% endif %}