pass through gzipped files as bytes in foldmason

Author: gamcil

backend/foldmasonmsa.go

@@ -3,21 +3,50 @@ package main
 import (
 	"crypto/sha256"
 	"encoding/base64"
+	"encoding/json"
 	"os"
 	"path/filepath"
-	"strings"
+	"regexp"
 )
 
 type FoldMasonMSAJob struct {
-	Queries   []string `json:"queries"`
+	Queries   [][]byte `json:"queries"`
 	FileNames []string `json:"fileNames"`
 }
 
+var foldMasonFileName = regexp.MustCompile(`[^a-zA-Z0-9_.-]+`)
+
+func (j *FoldMasonMSAJob) UnmarshalJSON(data []byte) error {
+	type current FoldMasonMSAJob
+	var cur current
+	if err := json.Unmarshal(data, &cur); err == nil {
+		*j = FoldMasonMSAJob(cur)
+		return nil
+	}
+
+	type legacy struct {
+		Queries   []string `json:"queries"`
+		FileNames []string `json:"fileNames"`
+	}
+	var old legacy
+	if err := json.Unmarshal(data, &old); err != nil {
+		return err
+	}
+
+	j.FileNames = old.FileNames
+	j.Queries = make([][]byte, len(old.Queries))
+	for i, query := range old.Queries {
+		j.Queries[i] = []byte(query)
+	}
+
+	return nil
+}
+
 func (r FoldMasonMSAJob) Hash() Id {
 	h := sha256.New224()
 	h.Write(([]byte)(JobFoldMasonMSA))
 	for _, query := range r.Queries {
-		h.Write([]byte(query))
+		h.Write(query)
 	}
 	bs := h.Sum(nil)
 	return Id(base64.URLEncoding.WithPadding(base64.NoPadding).EncodeToString(bs))
@@ -34,15 +63,8 @@ func (r FoldMasonMSAJob) WritePDB(path string) error {
 	var pdbDir = filepath.Join(path, "pdbs")
 	os.Mkdir(pdbDir, os.ModePerm)
 	for idx, query := range r.Queries {
-		name := cleanPathComponent.ReplaceAllString(r.FileNames[idx], "")
-		if strings.EqualFold(filepath.Ext(name), ".gz") {
-			name = strings.TrimSuffix(name, filepath.Ext(name))
-		}
-		ext := filepath.Ext(name)
-		if ext == ".cif" || ext == ".mmcif" {
-			name = strings.TrimSuffix(name, ext) + ".cif"
-		}
-		err := os.WriteFile(filepath.Join(pdbDir, name), []byte(query), 0644)
+		name := foldMasonFileName.ReplaceAllString(filepath.Base(r.FileNames[idx]), "")
+		err := os.WriteFile(filepath.Join(pdbDir, name), query, 0644)
 		if err != nil {
 			return err
 		}
@@ -51,7 +73,7 @@ func (r FoldMasonMSAJob) WritePDB(path string) error {
 }
 
 func NewFoldMasonMSAJobRequest(
-	queries []string,
+	queries [][]byte,
 	fileNames []string,
 	gapOpen int64,
 	gapExtend int64,

backend/server.go

@@ -3,7 +3,6 @@ package main
 import (
 	"bufio"
 	"bytes"
-	"compress/gzip"
 	"encoding/json"
 	"io"
 	"log"
@@ -24,34 +23,6 @@ import (
 	"github.com/rs/cors"
 )
 
-func readUploadedText(r io.Reader) (string, error) {
-	data, err := io.ReadAll(r)
-	if err != nil {
-		return "", err
-	}
-
-	if !isGzipPayload(data) {
-		return string(data), nil
-	}
-
-	gr, err := gzip.NewReader(bytes.NewReader(data))
-	if err != nil {
-		return "", err
-	}
-	defer gr.Close()
-
-	decoded, err := io.ReadAll(gr)
-	if err != nil {
-		return "", err
-	}
-
-	return string(decoded), nil
-}
-
-func isGzipPayload(data []byte) bool {
-	return len(data) >= 2 && data[0] == 0x1f && data[1] == 0x8b
-}
-
 type DatabaseResponse struct {
 	Databases []Params `json:"databases"`
 }
@@ -179,11 +150,9 @@ func server(jobsystem JobSystem, config ConfigRoot) {
 				}
 				defer f.Close()
 
-				data, err = readUploadedText(f)
-				if err != nil {
-					http.Error(w, err.Error(), http.StatusBadRequest)
-					return
-				}
+				buf := new(bytes.Buffer)
+				buf.ReadFrom(f)
+				data = buf.String()
 			} else {
 				err := req.ParseForm()
 				if err != nil {
@@ -326,11 +295,9 @@ func server(jobsystem JobSystem, config ConfigRoot) {
 			}
 			defer f.Close()
 
-			query, err = readUploadedText(f)
-			if err != nil {
-				http.Error(w, err.Error(), http.StatusBadRequest)
-				return
-			}
+			buf := new(bytes.Buffer)
+			buf.ReadFrom(f)
+			query = buf.String()
 			dbs = req.Form["database[]"]
 			mode = req.FormValue("mode")
 			email = req.FormValue("email")
@@ -415,11 +382,9 @@ func server(jobsystem JobSystem, config ConfigRoot) {
 			}
 			defer f.Close()
 
-			query, err = readUploadedText(f)
-			if err != nil {
-				http.Error(w, err.Error(), http.StatusBadRequest)
-				return
-			}
+			buf := new(bytes.Buffer)
+			buf.ReadFrom(f)
+			query = buf.String()
 			dbs = req.Form["database[]"]
 			mode = req.FormValue("mode")
 			email = req.FormValue("email")
@@ -480,11 +445,9 @@ func server(jobsystem JobSystem, config ConfigRoot) {
 			}
 			defer f.Close()
 
-			query, err = readUploadedText(f)
-			if err != nil {
-				http.Error(w, err.Error(), http.StatusBadRequest)
-				return
-			}
+			buf := new(bytes.Buffer)
+			buf.ReadFrom(f)
+			query = buf.String()
 			mode = req.FormValue("mode")
 			email = req.FormValue("email")
 		} else {
@@ -517,7 +480,7 @@ func server(jobsystem JobSystem, config ConfigRoot) {
 	}
 
 	ticketFoldMasonMSAHandlerFunc := func(w http.ResponseWriter, req *http.Request) {
-		var queries []string
+		var queries [][]byte
 		var fileNames []string
 		var gapOpen int64
 		var gapExtend int64
@@ -543,20 +506,19 @@ func server(jobsystem JobSystem, config ConfigRoot) {
 				}
 				defer file.Close()
 
-				query, err := readUploadedText(file)
-				if err != nil {
-					http.Error(w, err.Error(), http.StatusBadRequest)
-					return
-				}
-				queries = append(queries, query)
+				buf := new(bytes.Buffer)
+				buf.ReadFrom(file)
+				queries = append(queries, buf.Bytes())
 			}
 		} else {
 			err := req.ParseForm()
 			if err != nil {
 				http.Error(w, err.Error(), http.StatusBadRequest)
 				return
 			}
-			queries = req.Form["queries[]"]
+			for _, query := range req.Form["queries[]"] {
+				queries = append(queries, []byte(query))
+			}
 
 		}
 		fileNames = req.Form["fileNames[]"]
@@ -601,11 +563,9 @@ func server(jobsystem JobSystem, config ConfigRoot) {
 			}
 			defer f.Close()
 
-			query, err = readUploadedText(f)
-			if err != nil {
-				http.Error(w, err.Error(), http.StatusBadRequest)
-				return
-			}
+			buf := new(bytes.Buffer)
+			buf.ReadFrom(f)
+			query = buf.String()
 			dbs = req.Form["database[]"]
 			//mode = req.FormValue("mode")
 			email = req.FormValue("email")

frontend/FoldMasonSearch.vue

@@ -140,7 +140,6 @@ import { AxiosCompressRequest } from './lib/AxiosCompressRequest.js';
 import ApiDialog from './ApiDialog.vue';
 import { HistoryMixin } from './lib/HistoryMixin.js';
 import Databases from './Databases.vue';
-import QueryTextarea from "./QueryTextarea.vue";
 import DragUploadBox from "./DragUploadBox.vue";
 import { BlobDatabase } from "./lib/BlobDatabase.js";
 
@@ -159,7 +158,6 @@ export default {
         Reference,
         ApiDialog,
         Databases,
-        QueryTextarea,
         DragUploadBox
     },
     data() {

frontend/InterfaceSearch.vue

@@ -136,7 +136,7 @@ import LoadAcessionButton from './LoadAcessionButton.vue';
 import Reference from "./Reference.vue";
 import { convertToQueryUrl } from './lib/convertToQueryUrl';
 import TaxonomyAutocomplete from './TaxonomyAutocomplete.vue';
-import { djb2, parseResultsList, checkMultimer } from './Utilities.js';
+import { djb2, parseResultsList, checkMultimer, readUploadedText } from './Utilities.js';
 import { AxiosCompressRequest } from './lib/AxiosCompressRequest.js';
 import ApiDialog from './ApiDialog.vue';
 import { StorageWrapper, HistoryMixin } from './lib/HistoryMixin.js';
@@ -282,12 +282,13 @@ export default {
                 this.inSearch = false;
             }
         },
-        upload(files) {
-            var reader = new FileReader();
-            reader.onload = e => {
-                this.query = e.target.result;
-            };
-            reader.readAsText(files[0]);
+        async upload(files) {
+            try {
+                this.query = await readUploadedText(files[0]);
+            } catch (error) {
+                this.errorMessage = "Error reading uploaded file";
+                throw error;
+            }
         },
         uploadJSON(files) {
             let file = files[0];

frontend/QueryTextarea.vue

@@ -16,6 +16,8 @@
 </template>
 
 <script>
+import { readUploadedText } from "./Utilities.js";
+
 export default {
     name: 'QueryTextarea',
     props: {
@@ -28,11 +30,9 @@ export default {
             event.stopPropagation();
             var dataTransfer = event.dataTransfer || event.target;
             if (dataTransfer && dataTransfer.files && dataTransfer.files.length > 0) {
-                var reader = new FileReader();
-                reader.onload = e => {
-                    this.$emit('input', e.target.result);
-                };
-                reader.readAsText(dataTransfer.files[0]);
+                readUploadedText(dataTransfer.files[0]).then((text) => {
+                    this.$emit('input', text);
+                });
             }
         }
     }
@@ -59,4 +59,4 @@ export default {
     align-self: stretch;
 }
 
-</style>
+</style>