Merge pull request #135 from ComputerScienceHouse/develop

feat: chunked uploads

Author: BigSpaceships

gallery/__init__.py

@@ -40,6 +40,7 @@
 import flask_migrate
 import requests
 from werkzeug.utils import secure_filename
+import shutil
 
 app = Flask(__name__)
 app.config.update({
@@ -1056,7 +1057,7 @@ def render_dir(dir_id: int, auth_dict: Optional[Dict[str, Any]] = None):
     dir_model = Directory.query.filter(Directory.id == dir_id).first()
     if dir_model is None:
         abort(404)
-    description = dir_model.description
+    description = dir_model.description or ""
     display_description = len(description) > 0
 
     display_parent = True
@@ -1104,7 +1105,7 @@ def render_file(file_id: int, auth_dict: Optional[Dict[str, Any]] = None):
     if gallery_lockdown and (not auth_dict['is_eboard'] and not auth_dict['is_rtp']):
         abort(405)
 
-    description = file_model.caption
+    description = file_model.caption or ""
     display_description = len(description) > 0
     display_parent = True
     if file_model is None or file_model.parent is None:
@@ -1139,6 +1140,106 @@ def render_file(file_id: int, auth_dict: Optional[Dict[str, Any]] = None):
                            lockdown=gallery_lockdown)
 
 
+@app.route('/upload/chunk', methods=['POST'])
+@auth.oidc_auth('default')
+@gallery_auth
+def upload_chunk(auth_dict: Optional[Dict[str, Any]] = None):
+    chunk = request.files.get('gallery-upload')
+    if chunk is None:
+        return jsonify({'error': 'no chunk'}), 400
+    
+    dz_uuid      = request.form.get('dzuuid')
+    chunk_index  = int(request.form.get('dzchunkindex', 0))
+    chunk_size   = int(request.form.get('dzchunksize', 0))
+    filename     = secure_filename(request.form.get('dzfilename', ''))
+
+    if not dz_uuid or not filename:
+        return jsonify({'error': 'missing chunk metadata'}), 400
+
+    chunk_dir = os.path.join(tempfile.gettempdir(), 'chonks', dz_uuid)
+    os.makedirs(chunk_dir, exist_ok=True)
+
+    out_path = os.path.join(chunk_dir, 'assembled')
+    with open(out_path, 'ab') as out:
+        out.write(chunk.read())
+
+    return jsonify({'status': 'ok', 'chunk': chunk_index}), 200
+
+
+@app.route('/upload/chunk/finalize', methods=['POST'])
+@auth.oidc_auth('default')
+@gallery_auth
+def finalize_upload(auth_dict: Optional[Dict[str, Any]] = None):
+    assert auth_dict is not None
+    owner     = auth_dict['uuid']
+    parent    = request.form.get('parent_id')
+    dz_uuid   = request.form.get('dzuuid')
+    filename  = secure_filename(request.form.get('filename', ''))
+    total_chunks = int(request.form.get('dztotalchunkcount', 0))
+
+    if not all([parent, dz_uuid, filename, total_chunks]):
+        return jsonify({'error': 'missing parameters'}), 400
+
+    chunk_dir = os.path.join(tempfile.gettempdir(), 'chonks', dz_uuid)
+
+    upload_status: Dict[str, Any] = {}
+    upload_status['redirect'] = '/view/dir/' + str(parent)
+    errors: List[str] = []
+    success: List[Dict[str, Any]] = []
+
+    file_model = File.query.filter(File.parent == parent) \
+                           .filter(File.name == filename).first()
+    if file_model is not None:
+        errors.append(filename)
+        upload_status['error'] = errors
+        upload_status['success'] = success
+        return jsonify(upload_status), 200
+
+    dir_path = tempfile.mkdtemp()
+    filepath = os.path.join(dir_path, filename)
+    try:
+        assembled_path = os.path.join(chunk_dir, 'assembled')
+        if not os.path.exists(assembled_path):
+            return jsonify({'error': 'assembled file missing'}), 400
+        shutil.move(assembled_path, filepath)
+        try:
+            mime, file_model = add_file(filename, dir_path, parent, '', owner)
+        except OSError as e:
+            if e.errno == 28:
+                return jsonify({'error': 'storage full'}), 507
+            raise
+        if file_model is None:
+            errors.append(filename)
+        else:
+            with open(filepath, 'rb') as f_hnd:
+                storage_interface.put(
+                    'files/{}'.format(file_model.s3_id),
+                    f_hnd,
+                    filename,
+                    mime
+                )
+            os.remove(filepath)
+
+            thumb_path = os.path.join(dir_path, file_model.thumbnail_uuid)
+            with open(thumb_path, 'rb') as f_hnd:
+                storage_interface.put(
+                    'thumbnails/' + file_model.s3_id,
+                    f_hnd,
+                    'thumb_' + filename + '.' + thumb_path.split('.')[-1],
+                    'image/gif' if thumb_path.endswith('.gif') else 'image/jpeg'
+                )
+            os.remove(thumb_path)
+            success.append({'name': file_model.name, 'id': file_model.id})
+    finally:
+        shutil.rmtree(chunk_dir, ignore_errors=True)
+        shutil.rmtree(dir_path, ignore_errors=True)
+
+    upload_status['error'] = errors
+    upload_status['success'] = success
+    refresh_default_thumbnails()
+    return jsonify(upload_status), 200
+
+
 @app.route("/view/random_file")
 @auth.oidc_auth('default')
 def get_random_file():

gallery/templates/view_dir.html

@@ -245,34 +245,60 @@ <h4 class="modal-title">Upload</h4>
 Dropzone.autoDiscover = false;
 
     $(function() {
-
+        
         var myDropzone = new Dropzone('div#galleryUpload', {
-            url: '/upload',
+            url: '/upload/chunk',
             paramName: "gallery-upload",
             uploadMultiple: false,
             autoProcessQueue: false,
             parallelUploads: 3,
-            success: afterUpload,
-            maxFilesize: 1024,
+            maxFilesize: 5 * 1024,
+            chunking: true,
+            forceChunking: true,
+            chunkSize: 10 * 1024 * 1024,
+            retryChunks: true,
+            retryChunksLimit: 3,
+            timeout: 0,
             init: function() {
                 this.on('addedfile', function(file) {
                     console.log(file.name + ": " + file.size);
-                    if (file.size > (1024 * 1024 * 1024)) {
-                        var warning = "<div class='alert alert-dismissible alert-danger' id='upload-alert'><button type='button' class='close' data-dismiss='alert'>&times;</button><span class='glyphicon glyphicon-exclamation-sign'></span> File \"<strong>" + file.name + "</strong>\" is too large, max filesize is 1GB.</div>";
-                        $('#upload').after(warning);
-                        this.removeFile(file);
-                    }
                 });
 
                 this.on('accept', function(file, done) {
                     done();
                 });
 
                 this.on('sending', function(file, xhr, formData) {
-                    formData.append('parent_id', "{{directory.id}}");
+                    formData.append('parent_id', '{{ directory.id }}');
+                    formData.append('dzfilename', file.name);
+                });
+                
+                this.on('success', function(file, response) {
+                    if (file.upload.chunked && file.upload.chunkIndex < file.upload.totalChunkCount - 1) {
+                        return;
+                    }
+                    var formData = new FormData();
+                    formData.append('parent_id', '{{ directory.id }}');
+                    formData.append('dzuuid', file.upload.uuid);
+                    formData.append('filename', file.name);
+                    formData.append('dztotalchunkcount', file.upload.totalChunkCount);
+                    fetch('/upload/chunk/finalize', {
+                        method: 'POST',
+                        body: formData,
+                    })
+                    .then(function(r) { return r.json(); })
+                    .then(function(data) {
+                        console.log('finalize response:', JSON.stringify(data));
+                        afterUpload(file, data);
+                    })
+                    .catch(function(err) {
+                        console.error('Finalize error:', err);
+                    });
                 });
 
-                this.on('complete', function(file) {
+                this.on('error', function(file, message) {
+                    var warning = "<div class='alert alert-dismissible alert-danger' id='upload-alert'><button type='button' class='close' data-dismiss='alert'>&times;</button><span class='glyphicon glyphicon-exclamation-sign'></span> Upload failed for <strong>" + file.name + "</strong>: " + (typeof message === 'object' ? (message.error || JSON.stringify(message)) : message) + "</div>";
+                    $('#upload').after(warning);
                     this.removeFile(file);
                 });
             }

gallery/templates/view_file.html

@@ -59,7 +59,7 @@
 
             {% elif file.mimetype.split('/')[0] == "video" %}
             <video id="file-content" controls>
-                <source src="/api/file/get/{{file.id}}">
+                <source src="/api/file/get/{{file.id}}" type="{{ file.mimetype }}">
             </video>
 
             {% elif file.mimetype == "application/pdf" or file.mimetype == "text/plain" %}