FC-73 Xqu add submission file (Feature: Submission File System - Enhanced File Management for Graded Submissions)

.gitignore

@@ -63,3 +63,4 @@ test.txt.tmp
 
 # VSCode
 .vscode
+test_queue/*

docs/source/decisions/0002-add-submission-file-model.rst

@@ -0,0 +1,77 @@
+2. File Handling Implementation for Submission System
+#####################################################
+
+Status
+******
+
+**Provisional** *2025-02-14*
+
+Implemented by https://github.com/openedx/edx-submissions/pull/286
+
+Context
+*******
+
+As part of the XQueue migration effort detailed in ADR 0001, we need to implement a file handling system within edx-submissions. Currently, XQueue manages file submissions through a tightly coupled approach.
+
+### Current Limitations
+
+1. **Inadequate File Management**: XQueue's approach relies on JSON strings in character fields, with size constraints and manual URL manipulation for file handling.
+
+2. **Process Inefficiencies**: The current system uses synchronous HTTP for file retrieval, lacks proper validation, and tightly couples submission processing with file handling.
+
+3. **Integration Challenges**: External graders depend on specific URL formats with HTTP-based retrieval, embedding file information directly in submission payloads.
+
+Decision
+********
+
+We will implement a dedicated file management system for the assessment submission process, focusing on workflow and educational needs:
+
+1. **Centralized Storage**: Create a unified repository for student-submitted files, ensuring they are properly associated with their assessments and accessible throughout the grading process.
+
+2. **Streamlined Workflow**: Design a clear process where files are automatically processed during submission creation, securely stored, and efficiently delivered to grading systems.
+
+3. **Consistent Experience**: Maintain compatibility with existing systems to ensure a smooth transition, allowing instructors and external graders to access files without changes to their established workflows.
+
+Consequences
+************
+
+Positive:
+---------
+
+1. **Architecture**: Clean separation of concerns, improved maintainability, better error handling, optimized database access
+
+2. **Integration**: Seamless xqueue-watcher compatibility, support for existing workflows, minimal client code changes
+
+3. **Operations**: Robust file validation, improved tracking, better error visibility, simplified lifecycle management
+
+Negative:
+---------
+
+1. **Technical**: Additional database structures
+
+2. **Migration**: Temporary system complexity, additional testing needs
+
+3. **Performance**: File processing overhead
+
+References
+**********
+
+Current System Documentation:
+   * XQueue Repository: https://github.com/openedx/xqueue
+   * XQueue Watcher Repository: https://github.com/openedx/xqueue-watcher
+
+Related Repositories:
+   * edx-submissions: https://github.com/openedx/edx-submissions
+   * edx-platform: https://github.com/openedx/edx-platform
+   * XQueue Repository: https://github.com/openedx/xqueue
+
+Related Documentation:
+   * ADR 0001: Creation of ExternalGraderDetail Model for XQueue Migration
+
+Future Event Integration:
+   * Open edX Events Framework: https://github.com/openedx/openedx-events
+   * Event Bus Documentation: https://openedx.atlassian.net/wiki/spaces/AC/pages/124125264/Event+Bus
+
+Related Architecture Documents:
+   * Open edX Architecture Guidelines: https://openedx.atlassian.net/wiki/spaces/AC/pages/124125264/Architecture+Guidelines
+

settings.py

@@ -82,3 +82,11 @@
 )
 
 TEST_APPS = ('submissions',)
+
+EDX_SUBMISSIONS = {
+    'MEDIA': {
+        'BACKEND': 'django.core.files.storage.InMemoryStorage',
+        'OPTIONS': {
+        }
+    }
+}

submissions/__init__.py

@@ -1,2 +1,2 @@
 """ API for creating submissions and scores. """
-__version__ = '3.10.1'
+__version__ = '3.11.0'

submissions/api.py

@@ -15,7 +15,6 @@
 # SubmissionError imported so that code importing this api has access
 from submissions.errors import (  # pylint: disable=unused-import
     ExternalGraderQueueEmptyError,
-    SubmissionError,
     SubmissionInternalError,
     SubmissionNotFoundError,
     SubmissionRequestError
@@ -28,6 +27,7 @@
     ScoreSummary,
     StudentItem,
     Submission,
+    SubmissionFile,
     score_reset,
     score_set
 )
@@ -49,7 +49,6 @@
 TOP_SUBMISSIONS_CACHE_TIMEOUT = 300
 
 
-# pylint: disable=unused-argument
 def create_external_grader_detail(student_item_dict,
                                   answer,
                                   queue_name: str,
@@ -58,7 +57,7 @@ def create_external_grader_detail(student_item_dict,
                                   **external_grader_additional_data
                                   ):
     """
-    Creates a submission and an associated ExternalGraderDetail record.
+    Creates a submission and an associated ExternalGraderDetail record with optional file attachments.
 
     Args:
         student_item_dict (dict): The student_item this submission is associated with.
@@ -73,13 +72,28 @@ def create_external_grader_detail(student_item_dict,
         points_possible (int, optional): The maximum possible points for this submission. Defaults to 1.
 
         external_grader_additional_data: Additional keyword arguments that may be used for the external grader.
+            If a 'files' dictionary is included, files will be processed and stored as SubmissionFile objects.
+            The 'files' dictionary should map filenames to file objects (typically FileObjForWebobFiles).
 
     Returns:
         ExternalGraderDetail: The created external grader detail record that references the submission.
+            Any processed files are accessible through the `files` related_name on this object.
 
     Raises:
         ExternalGraderQueueEmptyError: If queue_name is empty.
         SubmissionInternalError: If there's an error creating the submission or external grader detail.
+
+    Example:
+        >>> files = {'assignment.py': file_obj}
+        >>> external_grader = create_external_grader_detail(
+        ...     student_item_dict,
+        ...     answer,
+        ...     'python_grader',
+        ...     files=files
+        ... )
+        >>> # Access files through external_grader.files.all()
+        >>> # Get URLs for external systems:
+        >>> urls = {f.original_filename: f.xqueue_url for f in external_grader.files.all()}
     """
 
     submission = create_submission(student_item_dict, answer)
@@ -93,9 +107,20 @@ def create_external_grader_detail(student_item_dict,
             submission_uuid=submission_uuid,
             queue_name=queue_name,
             grader_file_name=grader_file_name,
-            points_possible=points_possible,
+            points_possible=points_possible)
+
+        files_dict = external_grader_additional_data.get('files')
+        if files_dict:
+
+            files_urls = {}
+            for filename, file_obj in files_dict.items():
+                submission_file = SubmissionFile.objects.create(
+                    external_grader=instance,
+                    file=file_obj.file,
+                    original_filename=filename
+                )
+                files_urls[filename] = submission_file.xqueue_url
 
-        )
         return instance
 
     except DatabaseError as error:
@@ -106,6 +131,16 @@ def create_external_grader_detail(student_item_dict,
         raise SubmissionInternalError(error_message) from error
 
 
+def get_files_for_grader(external_grader):
+    """
+    Returns files in format expected by xqueue-watcher when is calling by get_submission endpoint.
+    """
+    return {
+        file.original_filename: file.file.url
+        for file in external_grader.files.all()
+    }
+
+
 def create_submission(
     student_item_dict,
     answer,

submissions/migrations/0005_submissionfile.py

@@ -0,0 +1,33 @@
+# Generated by Django 4.2.19 on 2025-05-15 14:14
+
+from django.db import migrations, models
+import django.db.models.deletion
+import django.utils.timezone
+import submissions.models
+import uuid
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('submissions', '0004_externalgraderdetail'),
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name='SubmissionFile',
+            fields=[
+                ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+                ('uuid', models.UUIDField(default=uuid.uuid4, editable=False)),
+                ('file', models.FileField(max_length=512, storage=submissions.models.get_storage,
+                                          upload_to=submissions.models.submission_file_path)),
+                ('original_filename', models.CharField(max_length=255)),
+                ('created_at', models.DateTimeField(default=django.utils.timezone.now)),
+                ('external_grader', models.ForeignKey(null=True, on_delete=django.db.models.deletion.SET_NULL,
+                                                      related_name='files', to='submissions.externalgraderdetail')),
+            ],
+            options={
+                'indexes': [models.Index(fields=['external_grader', 'uuid'], name='submissions_externa_ff8089_idx')],
+            },
+        ),
+    ]

submissions/models.py

@@ -9,15 +9,19 @@
     ./manage.py makemigrations submissions
 """
 
+import functools
 import logging
+import os
 from datetime import timedelta
 from uuid import uuid4
 
 from django.conf import settings
 from django.contrib import auth
+from django.core.files.storage import default_storage
 from django.db import DatabaseError, models, transaction
 from django.db.models.signals import post_save, pre_save
 from django.dispatch import Signal, receiver
+from django.utils.module_loading import import_string
 from django.utils.timezone import now
 from jsonfield import JSONField
 from model_utils.models import TimeStampedModel
@@ -693,3 +697,113 @@ def update_status(self, new_status):
     def create_from_uuid(cls, submission_uuid, **kwargs):
         submission = Submission.objects.get(uuid=submission_uuid)
         return cls.objects.create(submission=submission, **kwargs)
+
+
+def submission_file_path(instance, _):
+    """
+    Generate file path for submission files.
+    Format: queue_name/uuid
+    The filename is replaced with the UUID to ensure uniqueness without preserving extension.
+    """
+    return os.path.join(
+        instance.external_grader.queue_name,
+        f"{instance.uuid}"
+    )
+
+
+@functools.cache
+def _get_storage_cached():
+    """
+    Cached implementation to get the configured storage backend.
+
+    This private function loads storage configuration from settings and
+    dynamically instantiates the specified storage backend. It expects
+    EDX_SUBMISSIONS['MEDIA'] to be a dict with 'BACKEND' (string path to
+    storage class) and optional 'OPTIONS' (dict of parameters).
+
+    This function is for internal use only and is cached to improve performance.
+    """
+    edx_submissions_config = getattr(settings, 'EDX_SUBMISSIONS', {})
+    storage_config = edx_submissions_config.get('MEDIA')
+
+    if storage_config:
+        storage_cls = import_string(storage_config['BACKEND'])
+        options = storage_config.get('OPTIONS', {})
+        return storage_cls(**options)
+
+    return default_storage
+
+
+def get_storage():
+    """
+    Get the configured storage backend or fallback to default storage.
+
+    This function checks for a storage configuration in the Django settings.
+    It first looks for 'MEDIA' in the 'EDX_SUBMISSIONS' configuration dictionary.
+
+    The function uses an internal cached implementation while remaining
+    serializable for Django migrations, avoiding "Cannot serialize" errors.
+
+    Returns:
+        Storage instance: Returns the configured storage if found in EDX_SUBMISSIONS['MEDIA'],
+                         otherwise returns Django's default_storage.
+
+    Example:
+        # In settings.py
+        EDX_SUBMISSIONS = {
+            'MEDIA': {
+                'BACKEND': 'storages.backends.s3boto3.S3Boto3Storage',
+                'OPTIONS': {
+                    'bucket_name': 'my-bucket'
+                }
+            }
+        }
+
+        # Then get_storage() will return an S3Boto3Storage instance
+    """
+    return _get_storage_cached()  # For performance while keeping this function serializable for migrations
+
+
+class SubmissionFile(models.Model):
+    """
+    Model to handle files associated with submissions
+    """
+    uuid = models.UUIDField(default=uuid4, editable=False)  # legacy S3 key
+    external_grader = models.ForeignKey(
+        'submissions.ExternalGraderDetail',
+        on_delete=models.SET_NULL,
+        related_name='files',
+        null=True,
+    )
+    file = models.FileField(
+        upload_to=submission_file_path,
+        max_length=512,
+        storage=get_storage
+    )
+    original_filename = models.CharField(max_length=255)  # This is necessary to send file name to xqueue-watcher
+    created_at = models.DateTimeField(default=now)
+
+    class Meta:
+        indexes = [
+            models.Index(fields=['external_grader', 'uuid']),
+        ]
+
+    @property
+    def xqueue_url(self):
+        """
+        Returns a URL in the XQueue-compatible format: /queue_name/uuid
+
+        This format is used for file references in both the legacy XQueue system
+        and the new integrated standard. It maintains backward compatibility
+        while supporting the migration from the external XQueue API to the
+        integrated Open edX solution.
+
+        The URL follows the pattern: /{queue_name}/{submission_uuid}
+        where:
+        - queue_name: identifies the external grader queue
+        - uuid: uniquely identifies this submission (legacy S3 key)
+
+        Returns:
+            str: Formatted URL path following XQueue conventions
+        """
+        return f"/{self.external_grader.queue_name}/{self.uuid}"