Merge pull request #380 from toolforge/fix/large-file-error

Improve error message for files that exceed 5 GiB

Author: don-vip

video2commons/backend/encode/globals.py

@@ -27,8 +27,12 @@
 # The total amout of time a transcoding shell command can take:
 background_time_limit = 3600 * 24 * 2  # 2 days
 
-# Maximum file size transcoding processes can create, in KB.
-background_size_limit = 10 * 1024 * 1024  # 10GB
+# Maximum file size transcoding processes can create, passed to `ulimit -f`.
+#
+# /bin/sh on the workers is dash, which uses 512-byte blocks (per POSIX).
+# Matches the Commons 5 GiB upload limit. Anything larger would fail to upload
+# anyway, so there's no point in allowing anything higher than that.
+background_size_limit = 10 * 1024 * 1024
 
 # Number of hardware threads available to ffmpeg for transcoding.
 ffmpeg_threads = __import__("multiprocessing").cpu_count()

video2commons/frontend/errors.py

@@ -88,6 +88,13 @@
         "i18n_key": "segfault-error",
         "reportable": True,
     },
+    {
+        # This error happens when the ulimit restriction prevents a file larger
+        # than the Commons 5 GiB upload limit from being written to disk.
+        "pattern": r"Exitcode: 153$",
+        "i18n_key": "output-too-large-error",
+        "urls": ["https://commons.wikimedia.org/wiki/Commons:Maximum_file_size"],
+    },
 ]
 
 

video2commons/frontend/i18n/en.json

@@ -79,6 +79,7 @@
 	"title-forbidden-error": "The title of the file is not allowed. Please upload the file using a different title. Check the {{#a}}file naming policy{{/a}} for more information.",
 	"unsupported-format-error": "The file could not be encoded. This may be caused by unsupported formats or incompatible settings. Please report a bug so we can investigate.",
 	"video-not-found-error": "The file cannot be found at the provided URL. Please create a new task with a different URL.",
+	"output-too-large-error": "The output file is too large to be uploaded to Commons. Files larger than 5 GiB cannot be uploaded at this time. See the {{#a}}maximum file size{{/a}} page on Commons for more information.",
 	"youtube-bot-error": "The video cannot be downloaded due to YouTube's bot detection. Please try again in 1-3 hours.",
 	"youtube-private-error": "The video cannot be downloaded from YouTube as it is marked as private.",
 	"youtube-ratelimit-error": "The video cannot be downloaded from YouTube due to rate limiting. Please try again in 1-3 hours."

video2commons/frontend/i18n/qqq.json

@@ -86,6 +86,7 @@
 	"title-forbidden-error": "Semi-plain text shown as an error message in task results rows in the tasks table. There is a link in the text that links to the [[:commons:Commons:File_naming|file naming policy]] on [[:commons:|Commons]]. <code><nowiki>{{#a}}</nowiki></code> indicates the start of the link while <code><nowiki>{{/a}}</nowiki></code> indicates the end of the link.",
 	"unsupported-format-error": "Plain text shown as an error message in task results rows in the tasks table.",
 	"video-not-found-error": "Plain text shown as an error message in task results rows in the tasks table.",
+	"output-too-large-error": "Plain text shown as an error message in task results rows in the tasks table. There is a link in the text that links to the [[:commons:Commons:Maximum_file_size|maximum file size]] page on [[:commons:|Commons]]. <code><nowiki>{{#a}}</nowiki></code> indicates the start of the link while <code><nowiki>{{/a}}</nowiki></code> indicates the end of the link.",
 	"youtube-bot-error": "Plain text shown as an error message in task results rows in the tasks table.",
 	"youtube-private-error": "Plain text shown as an error message in task results rows in the tasks table.",
 	"youtube-ratelimit-error": "Plain text shown as an error message in task results rows in the tasks table."