Merge pull request learningequality#6041 from rtibblesbot/issue-5987-e45197

feat: export 64-bit file_size_bigint at publish

Author: rtibbles

contentcuration/contentcuration/tests/test_exportchannel.py

@@ -16,6 +16,7 @@
 from kolibri_content.router import cleanup_content_database_connection
 from kolibri_content.router import get_active_content_database
 from kolibri_content.router import set_active_content_database
+from le_utils.constants import content_kinds
 from le_utils.constants import exercises
 from le_utils.constants import format_presets
 from le_utils.constants import modalities
@@ -57,6 +58,10 @@
 pytestmark = pytest.mark.django_db
 
 
+# Larger than the signed 32-bit maximum (2_147_483_647); ~3 GB.
+LARGE_FILE_SIZE = 3 * 1024 ** 3
+
+
 def description():
     return "".join(random.sample(string.printable, 20))
 
@@ -387,6 +392,36 @@ def setUp(self):
         lesson_topic.extra_fields = {"options": {"modality": modalities.LESSON}}
         lesson_topic.save()
 
+        document_kind, _ = cc.ContentKind.objects.get_or_create(
+            kind=content_kinds.DOCUMENT
+        )
+        large_file_node = cc.ContentNode(
+            kind=document_kind,
+            parent=self.content_channel.main_tree,
+            title="Large file node",
+            node_id=uuid.uuid4(),
+            content_id=uuid.uuid4(),
+            sort_order=1,
+            complete=True,
+        )
+        large_file_node.save()
+
+        large_db_file = create_studio_file(
+            b"large file body", preset="document", ext="pdf"
+        )["db_file"]
+        # A >2.1 GB file cannot fit the legacy 32-bit File.file_size column; its
+        # true size lives in the studio#5974 file_size_bigint shadow, with the
+        # legacy file_size left NULL.
+        large_db_file.file_size = None
+        large_db_file.contentnode = large_file_node
+        large_db_file.save()
+        # Set the shadow directly; the mirror trigger leaves it alone because
+        # file_size is unchanged (NULL).
+        cc.File.objects.filter(pk=large_db_file.pk).update(
+            file_size_bigint=LARGE_FILE_SIZE
+        )
+        self.large_file_checksum = large_db_file.checksum
+
         set_channel_icon_encoding(self.content_channel)
         self.tempdb = create_content_database(
             self.content_channel, True, self.admin_user.id, True
@@ -506,6 +541,22 @@ def test_contentnode_file_size_data(self):
         for file in files.prefetch_related("local_file"):
             self.assertEqual(file.file_size, file.local_file.file_size)
 
+    def test_localfile_file_size_bigint_matches_small_files(self):
+        # Files that fit in 32 bits write the same value to both columns.
+        local_files = kolibri_models.LocalFile.objects.exclude(
+            pk=self.large_file_checksum
+        )
+        assert local_files.count() > 0
+        for local_file in local_files:
+            self.assertEqual(local_file.file_size_bigint, local_file.file_size)
+
+    def test_localfile_large_file_size_bigint(self):
+        # A >2.1 GB file keeps its real size in file_size_bigint and NULLs the
+        # legacy 32-bit file_size.
+        local_file = kolibri_models.LocalFile.objects.get(pk=self.large_file_checksum)
+        self.assertEqual(local_file.file_size_bigint, LARGE_FILE_SIZE)
+        self.assertIsNone(local_file.file_size)
+
     def test_file_included_presets_renderable(self):
         # Every non-supplementary (renderable) exported file carries its own preset bit.
         files = kolibri_models.File.objects.filter(supplementary=False)

contentcuration/contentcuration/utils/publish.py

@@ -65,6 +65,8 @@
 PERSEUS_IMG_DIR = exercises.IMG_PLACEHOLDER + "/images"
 THUMBNAIL_DIMENSION = 128
 MIN_SCHEMA_VERSION = "1"
+# Largest value the legacy 32-bit LocalFile.file_size / File.file_size columns hold.
+INT_32BIT_MAX = 2 ** 31 - 1
 PUBLISHING_UPDATE_THRESHOLD = 3600
 
 
@@ -659,11 +661,23 @@ def create_associated_file_objects(kolibrinode, ccnode):
                 create_associated_thumbnail(ccnode, ccfilemodel) or ccfilemodel
             )
 
+        # The true size lives in the studio#5974 file_size_bigint shadow (the
+        # legacy 32-bit file_size cannot hold >2.1 GB); fall back to file_size
+        # for rows the shadow has not been backfilled onto yet.
+        real_size = ccfilemodel.file_size_bigint
+        if real_size is None:
+            real_size = ccfilemodel.file_size
+        if real_size is not None and real_size > INT_32BIT_MAX:
+            legacy_size = None
+        else:
+            legacy_size = real_size
+
         kolibrilocalfilemodel, new = kolibrimodels.LocalFile.objects.get_or_create(
             pk=ccfilemodel.checksum,
             defaults={
                 "extension": fformat.extension,
-                "file_size": ccfilemodel.file_size,
+                "file_size": legacy_size,
+                "file_size_bigint": real_size,
             },
         )
 
@@ -687,7 +701,7 @@ def create_associated_file_objects(kolibrinode, ccnode):
             checksum=ccfilemodel.checksum,
             extension=fformat.extension,
             available=True,  # TODO: Set this to False, once we have availability stamping implemented in Kolibri
-            file_size=ccfilemodel.file_size,
+            file_size=legacy_size,
             contentnode=kolibrinode,
             preset=preset.pk,
             supplementary=preset.supplementary,

contentcuration/kolibri_content/base_models.py

@@ -152,6 +152,7 @@ class LocalFile(models.Model):
     )
     available = models.BooleanField(default=False)
     file_size = models.IntegerField(blank=True, null=True)
+    file_size_bigint = models.BigIntegerField(blank=True, null=True)
 
     class Meta:
         abstract = True

contentcuration/kolibri_content/migrations/0025_localfile_file_size_bigint.py

@@ -0,0 +1,17 @@
+from django.db import migrations
+from django.db import models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("content", "0024_file_included_presets"),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name="localfile",
+            name="file_size_bigint",
+            field=models.BigIntegerField(blank=True, null=True),
+        ),
+    ]

contentcuration/kolibri_public/migrations/0010_localfile_file_size_bigint.py

@@ -0,0 +1,17 @@
+from django.db import migrations
+from django.db import models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("kolibri_public", "0009_file_included_presets"),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name="localfile",
+            name="file_size_bigint",
+            field=models.BigIntegerField(blank=True, null=True),
+        ),
+    ]