|
10 | 10 | """ |
11 | 11 |
|
12 | 12 | import logging |
| 13 | +import os |
13 | 14 | from datetime import timedelta |
14 | 15 | from uuid import uuid4 |
15 | 16 |
|
16 | 17 | from django.conf import settings |
17 | 18 | from django.contrib import auth |
| 19 | +from django.core.files.base import ContentFile |
| 20 | +from django.core.files.uploadedfile import SimpleUploadedFile |
18 | 21 | from django.db import DatabaseError, models, transaction |
19 | 22 | from django.db.models.signals import post_save, pre_save |
20 | 23 | from django.dispatch import Signal, receiver |
21 | 24 | from django.utils.timezone import now |
22 | 25 | from jsonfield import JSONField |
23 | 26 | from model_utils.models import TimeStampedModel |
24 | 27 |
|
25 | | -from submissions.errors import DuplicateTeamSubmissionsError, TeamSubmissionInternalError, TeamSubmissionNotFoundError |
| 28 | +from submissions.errors import ( |
| 29 | + DuplicateTeamSubmissionsError, |
| 30 | + FileProcessingError, |
| 31 | + InvalidFileTypeError, |
| 32 | + TeamSubmissionInternalError, |
| 33 | + TeamSubmissionNotFoundError |
| 34 | +) |
26 | 35 |
|
27 | 36 | logger = logging.getLogger(__name__) |
28 | 37 | User = auth.get_user_model() |
@@ -693,3 +702,136 @@ def update_status(self, new_status): |
693 | 702 | def create_from_uuid(cls, submission_uuid, **kwargs): |
694 | 703 | submission = Submission.objects.get(uuid=submission_uuid) |
695 | 704 | return cls.objects.create(submission=submission, **kwargs) |
| 705 | + |
| 706 | + |
| 707 | +def submission_file_path(instance, _): |
| 708 | + """ |
| 709 | + Generate file path for submission files. |
| 710 | + Format: queue_name/uuid |
| 711 | + The filename is replaced with the UUID to ensure uniqueness without preserving extension. |
| 712 | + """ |
| 713 | + return os.path.join( |
| 714 | + instance.external_grader.queue_name, |
| 715 | + f"{instance.uid}" |
| 716 | + ) |
| 717 | + |
| 718 | + |
| 719 | +class SubmissionFile(models.Model): |
| 720 | + """ |
| 721 | + Model to handle files associated with submissions |
| 722 | + """ |
| 723 | + uid = models.UUIDField(default=uuid4, editable=False) # legacy S3 key |
| 724 | + external_grader = models.ForeignKey( |
| 725 | + 'submissions.ExternalGraderDetail', |
| 726 | + on_delete=models.SET_NULL, |
| 727 | + related_name='files', |
| 728 | + null=True, |
| 729 | + ) |
| 730 | + file = models.FileField( |
| 731 | + upload_to=submission_file_path, |
| 732 | + max_length=512 |
| 733 | + ) |
| 734 | + original_filename = models.CharField(max_length=255) # This is necessary to send file name to xqueue-watcher |
| 735 | + created_at = models.DateTimeField(default=now) |
| 736 | + |
| 737 | + class Meta: |
| 738 | + indexes = [ |
| 739 | + models.Index(fields=['external_grader', 'uid']), |
| 740 | + ] |
| 741 | + |
| 742 | + @property |
| 743 | + def xqueue_url(self): |
| 744 | + """ |
| 745 | + Returns URL in xqueue format: /queue_name/uid |
| 746 | + """ |
| 747 | + return f"/{self.external_grader.queue_name}/{self.uid}" |
| 748 | + |
| 749 | + |
| 750 | +class SubmissionFileManager: |
| 751 | + """ |
| 752 | + Manages file operations for submissions |
| 753 | + """ |
| 754 | + |
| 755 | + def __init__(self, external_grader): |
| 756 | + self.external_grader = external_grader |
| 757 | + |
| 758 | + def process_files(self, files_dict): |
| 759 | + """ |
| 760 | + Process uploaded files from an Open edX environment and store them as SubmissionFile objects. |
| 761 | +
|
| 762 | + This method handles various file object types that might be received from Open edX, including: |
| 763 | + - Native Open edX FileObjForWebobFiles objects |
| 764 | + - Bytes objects |
| 765 | + - ContentFile objects |
| 766 | + - SimpleUploadedFile objects |
| 767 | + - Any object with a 'read' method |
| 768 | +
|
| 769 | + The method performs the following operations: |
| 770 | + 1. Validates each file object type |
| 771 | + 2. Reads content from file-like objects |
| 772 | + 3. Converts byte content to ContentFile objects |
| 773 | + 4. Creates SubmissionFile records in the database |
| 774 | + 5. Returns URLs in xqueue-compatible format |
| 775 | +
|
| 776 | + Args: |
| 777 | + files_dict (dict): Dictionary mapping filenames to file objects. |
| 778 | + Format: {filename: file_object, ...} |
| 779 | +
|
| 780 | + Returns: |
| 781 | + dict: Dictionary mapping original filenames to xqueue URLs. |
| 782 | + Format: {filename: "/queue_name/uuid", ...} |
| 783 | +
|
| 784 | + Raises: |
| 785 | + InvalidFileTypeError: If a file object has an unsupported type. |
| 786 | + FileProcessingError: If there's an error reading from a file object, |
| 787 | + including I/O errors or Unicode decoding errors. |
| 788 | +
|
| 789 | + Example: |
| 790 | + >>> file_manager = SubmissionFileManager(external_grader) |
| 791 | + >>> files = {'assignment.py': file_obj} |
| 792 | + >>> urls = file_manager.process_files(files) |
| 793 | + >>> print(urls) |
| 794 | + {'assignment.py': '/my_queue/550e8400-e29b-41d4-a716-446655440000'} |
| 795 | + """ |
| 796 | + files_urls = {} |
| 797 | + for filename, file_obj in files_dict.items(): |
| 798 | + # Validate file object type |
| 799 | + if not (isinstance(file_obj, (bytes, ContentFile, SimpleUploadedFile)) or hasattr(file_obj, 'read')): |
| 800 | + logger.error(f"Invalid file object type for {filename}") |
| 801 | + raise InvalidFileTypeError(f"Invalid file object type for {filename}") |
| 802 | + |
| 803 | + # Handle file-like objects by reading their content |
| 804 | + # This returns a bytes object that will be converted to ContentFile later |
| 805 | + if hasattr(file_obj, 'read'): |
| 806 | + try: |
| 807 | + file_obj = file_obj.read() # read() returns a bytes object |
| 808 | + except (IOError, OSError) as e: |
| 809 | + logger.error(f"Error reading file {filename}: {e}") |
| 810 | + raise FileProcessingError(f"Error reading file {filename}: {e}") from e |
| 811 | + except UnicodeDecodeError as e: |
| 812 | + logger.error(f"Error decoding file {filename}: {e}") |
| 813 | + raise FileProcessingError(f"Error decoding file {filename}: {e}") from e |
| 814 | + |
| 815 | + # Convert bytes to ContentFile |
| 816 | + # The read() method from file-like objects returns bytes, which we handle here |
| 817 | + if isinstance(file_obj, bytes): |
| 818 | + file_obj = ContentFile(file_obj, name=filename) |
| 819 | + |
| 820 | + # Create a SubmissionFile record for storage and retrieval |
| 821 | + submission_file = SubmissionFile.objects.create( |
| 822 | + external_grader=self.external_grader, |
| 823 | + file=file_obj, |
| 824 | + original_filename=filename |
| 825 | + ) |
| 826 | + files_urls[filename] = submission_file.xqueue_url |
| 827 | + |
| 828 | + return files_urls |
| 829 | + |
| 830 | + def get_files_for_grader(self): |
| 831 | + """ |
| 832 | + Returns files in format expected by xqueue-watcher |
| 833 | + """ |
| 834 | + return { |
| 835 | + file.original_filename: file.file.url |
| 836 | + for file in self.external_grader.files.all() |
| 837 | + } |
0 commit comments