add support for gzipped input files

Author: gamcil

backend/foldmasonmsa.go

@@ -35,7 +35,10 @@ func (r FoldMasonMSAJob) WritePDB(path string) error {
 	os.Mkdir(pdbDir, os.ModePerm)
 	for idx, query := range r.Queries {
 		name := cleanPathComponent.ReplaceAllString(r.FileNames[idx], "")
-		ext := filepath.Ext(r.FileNames[idx])
+		if strings.EqualFold(filepath.Ext(name), ".gz") {
+			name = strings.TrimSuffix(name, filepath.Ext(name))
+		}
+		ext := filepath.Ext(name)
 		if ext == ".cif" || ext == ".mmcif" {
 			name = strings.TrimSuffix(name, ext) + ".cif"
 		}

backend/server.go

@@ -3,6 +3,7 @@ package main
 import (
 	"bufio"
 	"bytes"
+	"compress/gzip"
 	"encoding/json"
 	"io"
 	"log"
@@ -23,6 +24,34 @@ import (
 	"github.com/rs/cors"
 )
 
+func readUploadedText(r io.Reader) (string, error) {
+	data, err := io.ReadAll(r)
+	if err != nil {
+		return "", err
+	}
+
+	if !isGzipPayload(data) {
+		return string(data), nil
+	}
+
+	gr, err := gzip.NewReader(bytes.NewReader(data))
+	if err != nil {
+		return "", err
+	}
+	defer gr.Close()
+
+	decoded, err := io.ReadAll(gr)
+	if err != nil {
+		return "", err
+	}
+
+	return string(decoded), nil
+}
+
+func isGzipPayload(data []byte) bool {
+	return len(data) >= 2 && data[0] == 0x1f && data[1] == 0x8b
+}
+
 type DatabaseResponse struct {
 	Databases []Params `json:"databases"`
 }
@@ -148,10 +177,13 @@ func server(jobsystem JobSystem, config ConfigRoot) {
 					http.Error(w, err.Error(), http.StatusBadRequest)
 					return
 				}
+				defer f.Close()
 
-				buf := new(bytes.Buffer)
-				buf.ReadFrom(f)
-				data = buf.String()
+				data, err = readUploadedText(f)
+				if err != nil {
+					http.Error(w, err.Error(), http.StatusBadRequest)
+					return
+				}
 			} else {
 				err := req.ParseForm()
 				if err != nil {
@@ -292,10 +324,13 @@ func server(jobsystem JobSystem, config ConfigRoot) {
 				http.Error(w, err.Error(), http.StatusBadRequest)
 				return
 			}
+			defer f.Close()
 
-			buf := new(bytes.Buffer)
-			buf.ReadFrom(f)
-			query = buf.String()
+			query, err = readUploadedText(f)
+			if err != nil {
+				http.Error(w, err.Error(), http.StatusBadRequest)
+				return
+			}
 			dbs = req.Form["database[]"]
 			mode = req.FormValue("mode")
 			email = req.FormValue("email")
@@ -378,10 +413,13 @@ func server(jobsystem JobSystem, config ConfigRoot) {
 				http.Error(w, err.Error(), http.StatusBadRequest)
 				return
 			}
+			defer f.Close()
 
-			buf := new(bytes.Buffer)
-			buf.ReadFrom(f)
-			query = buf.String()
+			query, err = readUploadedText(f)
+			if err != nil {
+				http.Error(w, err.Error(), http.StatusBadRequest)
+				return
+			}
 			dbs = req.Form["database[]"]
 			mode = req.FormValue("mode")
 			email = req.FormValue("email")
@@ -440,10 +478,13 @@ func server(jobsystem JobSystem, config ConfigRoot) {
 				http.Error(w, err.Error(), http.StatusBadRequest)
 				return
 			}
+			defer f.Close()
 
-			buf := new(bytes.Buffer)
-			buf.ReadFrom(f)
-			query = buf.String()
+			query, err = readUploadedText(f)
+			if err != nil {
+				http.Error(w, err.Error(), http.StatusBadRequest)
+				return
+			}
 			mode = req.FormValue("mode")
 			email = req.FormValue("email")
 		} else {
@@ -502,14 +543,11 @@ func server(jobsystem JobSystem, config ConfigRoot) {
 				}
 				defer file.Close()
 
-				buf := new(bytes.Buffer)
-				_, err = io.Copy(buf, file)
+				query, err := readUploadedText(file)
 				if err != nil {
-					http.Error(w, err.Error(), http.StatusInternalServerError)
+					http.Error(w, err.Error(), http.StatusBadRequest)
 					return
 				}
-
-				query := buf.String()
 				queries = append(queries, query)
 			}
 		} else {
@@ -561,10 +599,13 @@ func server(jobsystem JobSystem, config ConfigRoot) {
 				http.Error(w, err.Error(), http.StatusBadRequest)
 				return
 			}
+			defer f.Close()
 
-			buf := new(bytes.Buffer)
-			buf.ReadFrom(f)
-			query = buf.String()
+			query, err = readUploadedText(f)
+			if err != nil {
+				http.Error(w, err.Error(), http.StatusBadRequest)
+				return
+			}
 			dbs = req.Form["database[]"]
 			//mode = req.FormValue("mode")
 			email = req.FormValue("email")

frontend/FoldDiscoSearch.vue

@@ -174,6 +174,7 @@ import Panel from "./Panel.vue";
 import FileButton from "./FileButton.vue";
 import LoadAcessionButton from './LoadAcessionButton.vue';
 import Reference from "./Reference.vue";
+import { readUploadedText } from './Utilities.js';
 import { AxiosCompressRequest } from './lib/AxiosCompressRequest.js';
 import { convertToQueryUrl } from './lib/convertToQueryUrl';
 import TaxonomyAutocomplete from './TaxonomyAutocomplete.vue';
@@ -413,12 +414,13 @@ export default {
                 this.inSearch = false;
             }
         },
-        upload(files) {
-            var reader = new FileReader();
-            reader.onload = async e => {
-                this.query = e.target.result;
-            };
-            reader.readAsText(files[0]);
+        async upload(files) {
+            try {
+                this.query = await readUploadedText(files[0]);
+            } catch (error) {
+                this.errorMessage = "Error reading uploaded file";
+                throw error;
+            }
         },
         async goToFoldseek() {
             await db.setItem('query', this.query);

frontend/FoldMasonSearch.vue

@@ -227,7 +227,7 @@ export default {
             const params = new FormData();
             this.queries.forEach((v) => {
                 params.append('fileNames[]', v.name);
-                params.append('queries[]', new Blob([v.text], { type: 'text/plain' }), v.name);
+                params.append('queries[]', v.file || new Blob([v.text], { type: 'text/plain' }), v.name);
             });
 
             try {
@@ -285,21 +285,7 @@ export default {
          * @param {*} files - FileList from upload event
          */
         async upload(files) {
-            const readFile = async (file) => {
-                return new Promise((resolve, reject) => {
-                    const reader = new FileReader();
-                    reader.onload = e => resolve({ name: file.name, text: e.target.result });
-                    reader.onerror = err => reject(err);
-                    reader.readAsText(file);
-                });
-            }
-            try {
-                const fileReadPromises = Array.from(files).map(readFile);
-                const fileContents = await Promise.all(fileReadPromises);
-                this.addFiles(fileContents);
-            } catch(error) {
-                console.log("Error reading files", error);
-            }
+            this.addFiles(Array.from(files).map(file => ({ name: file.name, file })));
         },
         uploadJSON(files) {
             let file = files[0];

frontend/MultimerSearch.vue

@@ -138,14 +138,13 @@ import LoadAcessionButton from './LoadAcessionButton.vue';
 import Reference from "./Reference.vue";
 import { convertToQueryUrl } from './lib/convertToQueryUrl';
 import TaxonomyAutocomplete from './TaxonomyAutocomplete.vue';
-import { djb2, parseResultsList, checkMultimer } from './Utilities.js';
+import { djb2, parseResultsList, checkMultimer, readUploadedText } from './Utilities.js';
 import { AxiosCompressRequest } from './lib/AxiosCompressRequest.js';
 import ApiDialog from './ApiDialog.vue';
 import { StorageWrapper, HistoryMixin } from './lib/HistoryMixin.js';
 import { BlobDatabase } from './lib/BlobDatabase.js';
 import Databases from './Databases.vue';
 import QueryTextarea from "./QueryTextarea.vue";
-
 const db = BlobDatabase();
 const storage = new StorageWrapper("complex");
 
@@ -286,12 +285,13 @@ export default {
                 this.inSearch = false;
             }
         },
-        upload(files) {
-            var reader = new FileReader();
-            reader.onload = e => {
-                this.query = e.target.result;
-            };
-            reader.readAsText(files[0]);
+        async upload(files) {
+            try {
+                this.query = await readUploadedText(files[0]);
+            } catch (error) {
+                this.errorMessage = "Error reading uploaded file";
+                throw error;
+            }
         },
         uploadJSON(files) {
             let file = files[0];

frontend/Search.vue

@@ -155,14 +155,13 @@ import LoadAcessionButton from './LoadAcessionButton.vue';
 import Reference from "./Reference.vue";
 import { convertToQueryUrl } from './lib/convertToQueryUrl';
 import TaxonomyAutocomplete from './TaxonomyAutocomplete.vue';
-import { djb2, parseResultsList, checkMultimer } from './Utilities.js';
+import { djb2, parseResultsList, checkMultimer, readUploadedText } from './Utilities.js';
 import { AxiosCompressRequest } from './lib/AxiosCompressRequest.js';
 import ApiDialog from './ApiDialog.vue';
 import { storage, HistoryMixin } from './lib/HistoryMixin.js';
 import { BlobDatabase } from './lib/BlobDatabase.js';
 import Databases from './Databases.vue';
 import QueryTextarea from "./QueryTextarea.vue";
-
 const db = BlobDatabase();
 
 export default {
@@ -325,12 +324,13 @@ export default {
                 this.inSearch = false;
             }
         },
-        upload(files) {
-            var reader = new FileReader();
-            reader.onload = e => {
-                this.query = e.target.result;
-            };
-            reader.readAsText(files[0]);
+        async upload(files) {
+            try {
+                this.query = await readUploadedText(files[0]);
+            } catch (error) {
+                this.errorMessage = "Error reading uploaded file";
+                throw error;
+            }
         },
         uploadJSON(files) {
             let file = files[0];

frontend/Utilities.js

@@ -1,4 +1,5 @@
 import { Selection, Matrix4, PdbWriter } from "ngl";
+import { ungzip } from "pako";
 
 function tryLinkTargetToDB(target, db) {
   try {
@@ -174,6 +175,15 @@ export function tryFixName(name) {
   return name.replaceAll(/\.(cif|pdb|gz)/g, "");
 }
 
+export async function readUploadedText(file) {
+  const bytes = new Uint8Array(await file.arrayBuffer());
+  const data =
+    bytes.length >= 2 && bytes[0] === 0x1f && bytes[1] === 0x8b
+      ? ungzip(bytes)
+      : bytes;
+  return new TextDecoder().decode(data);
+}
+
 export function parseResults(data) {
   let empty = 0;
   let total = 0;