Add .pecan file support for data uploader

Author: haoruizhou

server/installer/file-uploader/app.py

@@ -7,6 +7,8 @@
     Response,
 )
 import uuid, time, threading, json, logging, requests, os, asyncio, io, zipfile
+from datetime import datetime, timezone
+from zoneinfo import ZoneInfo
 from typing import Optional, Tuple, List
 from urllib.parse import quote
 from helper import CANTimescaleStreamer
@@ -20,7 +22,7 @@
 
 error_logger = logging.getLogger(__name__)
 
-ALLOWED_EXTENSIONS = {"csv", "zip"}
+ALLOWED_EXTENSIONS = {"csv", "zip", "pecan"}
 UPLOAD_ZIP_MAX_ARCHIVE_BYTES = int(os.getenv("UPLOAD_ZIP_MAX_ARCHIVE_BYTES", str(2 * 1024**3)))
 UPLOAD_ZIP_MAX_MEMBER_BYTES = int(os.getenv("UPLOAD_ZIP_MAX_MEMBER_BYTES", str(4 * 1024**3)))
 UPLOAD_ZIP_MAX_TOTAL_UNCOMPRESSED_BYTES = int(
@@ -155,6 +157,16 @@ def _zip_entry_path_safe(arcname: str) -> bool:
     return ".." not in n.split("/")
 
 
+class _InMemoryFile:
+    """Minimal file-like object for passing in-memory bytes through expand_upload_files_to_csv_payloads."""
+    def __init__(self, filename: str, data: bytes):
+        self.filename = filename
+        self._data = data
+
+    def read(self) -> bytes:
+        return self._data
+
+
 def expand_upload_files_to_csv_payloads(files) -> Tuple[List[Tuple[str, bytes]], Optional[str]]:
     out: List[Tuple[str, bytes]] = []
     zip_idx = 0
@@ -178,36 +190,76 @@ def expand_upload_files_to_csv_payloads(files) -> Tuple[List[Tuple[str, bytes]],
                     infos = [
                         i for i in z.infolist()
                         if not i.is_dir()
-                        and i.filename.lower().endswith(".csv")
+                        and (i.filename.lower().endswith(".csv") or i.filename.lower().endswith(".pecan"))
                         and _zip_entry_path_safe(i.filename)
                         # exclude macOS resource forks (__MACOSX/ and ._filename)
                         and not i.filename.startswith("__MACOSX/")
                         and not os.path.basename(i.filename).startswith("._")
                     ]
                     if not infos:
-                        return [], f"No CSV files found in zip: {name}"
+                        return [], f"No CSV or .pecan files found in zip: {name}"
                     if len(infos) > UPLOAD_ZIP_MAX_CSV_IN_ZIP:
                         return [], f"Too many CSV entries in {name} (max {UPLOAD_ZIP_MAX_CSV_IN_ZIP})"
                     total_uc = sum(i.file_size for i in infos)
                     if total_uc > UPLOAD_ZIP_MAX_TOTAL_UNCOMPRESSED_BYTES:
                         return [], f"Zip {name} uncompressed total too large"
                     for i in infos:
                         if i.file_size > UPLOAD_ZIP_MAX_MEMBER_BYTES:
-                            return [], f"CSV inside zip too large: {i.filename} in {name}"
+                            return [], f"File inside zip too large: {i.filename} in {name}"
                         leaf = os.path.basename(i.filename) or "data.csv"
                         key = (zlabel, leaf.lower())
                         if key in seen_in_zip:
-                            return [], f'Duplicate CSV filename "{leaf}" inside zip {name}'
+                            return [], f'Duplicate filename "{leaf}" inside zip {name}'
                         seen_in_zip.add(key)
                         with z.open(i, "r") as fp:
                             body = fp.read()
-                        out.append((f"_z{zlabel}/{leaf}", body))
+                        if leaf.lower().endswith(".pecan"):
+                            # Convert .pecan to CSV in-place so the pipeline is uniform
+                            sub_out, err = expand_upload_files_to_csv_payloads(
+                                [_InMemoryFile(leaf, body)]
+                            )
+                            if err:
+                                return [], f"{err} (inside zip {name})"
+                            out.extend(sub_out)
+                        else:
+                            out.append((f"_z{zlabel}/{leaf}", body))
             except zipfile.BadZipFile:
                 return [], f"Invalid or corrupt zip: {name}"
             except RuntimeError as e:
                 return [], f"Could not read zip {name}: {e}"
+        elif ext == "pecan":
+            try:
+                payload = json.loads(data.decode("utf-8"))
+            except Exception:
+                return [], f"Invalid .pecan file (bad JSON): {name}"
+            if payload.get("format") != "pecan-session" or payload.get("version") != 2:
+                return [], f".pecan file must be pecan-session v2 format: {name}"
+            frames = payload.get("frames") or []
+            if not frames:
+                return [], f"No frames in .pecan file: {name}"
+            epoch_base_ms = payload.get("epochBaseMs")
+            if epoch_base_ms is None:
+                return [], f".pecan file missing epochBaseMs — cannot determine timestamps: {name}"
+            tz_toronto = ZoneInfo("America/Toronto")
+            start_dt = datetime.fromtimestamp(epoch_base_ms / 1000, tz=tz_toronto)
+            csv_filename = start_dt.strftime("%Y-%m-%d-%H-%M-%S") + ".csv"
+            lines = []
+            for frame in frames:
+                if not isinstance(frame, list) or len(frame) < 4:
+                    continue
+                try:
+                    t_rel_ms = int(frame[0])
+                    can_id = int(frame[1])
+                    data_bytes = bytes.fromhex(str(frame[3]))
+                    padded = (data_bytes + b"\x00" * 8)[:8]
+                except Exception:
+                    continue
+                lines.append(f"{t_rel_ms},CAN,{can_id}," + ",".join(str(b) for b in padded))
+            if not lines:
+                return [], f"No parseable frames in .pecan file: {name}"
+            out.append((csv_filename, "\n".join(lines).encode("utf-8")))
         else:
-            return [], f"Invalid file type (only .csv and .zip): {name}"
+            return [], f"Invalid file type (only .csv, .zip, and .pecan): {name}"
     if not out:
         return [], "No CSV data to process"
     return out, None

server/installer/file-uploader/static/index.js

@@ -143,7 +143,7 @@ const DROP_SVG = `<svg id="file-upload-img" aria-hidden="true"
 	d="M13 13h3a3 3 0 0 0 0-6h-.025A5.56 5.56 0 0 0 16 6.5 5.5 5.5 0 0 0 5.207 5.021C5.137
 	5.017 5.071 5 5 5a4 4 0 0 0 0 8h2.167M10 15V6m0 0L8 8m2-2 2 2"/>
 </svg>
-<h3>Click to upload CSV or ZIP, or drag and drop</h3>`;
+<h3>Click to upload CSV, ZIP, or .pecan, or drag and drop</h3>`;
 
 const SPINNER_HTML = `<svg class="spinner" viewBox="0 0 50 50">
 	<circle class="path" cx="25" cy="25" r="20" fill="none" stroke-width="5"></circle>
@@ -264,8 +264,9 @@ function submitCsvUpload(files) {
 		const n = file.name.toLowerCase();
 		const okCsv = file.type === "text/csv" || n.endsWith(".csv") || file.type === "application/csv";
 		const okZip = n.endsWith(".zip") || file.type === "application/zip" || file.type === "application/x-zip-compressed";
-		if (!okCsv && !okZip) {
-			alert(`${file.name} must be .csv or .zip`);
+		const okPecan = n.endsWith(".pecan");
+		if (!okCsv && !okZip && !okPecan) {
+			alert(`${file.name} must be .csv, .zip, or .pecan`);
 			return;
 		}
 	}

server/installer/file-uploader/templates/index.html

@@ -16,7 +16,7 @@
 				<div class="info">
 					<img src="{{ url_for('static', filename='logo-32_375x.webp' )}}" />
 					<h1>Upload DAQ Data</h1>
-					<h2>(multiple CSVs or .zip of CSVs)</h2>
+					<h2>(CSV, .zip of CSVs, or .pecan)</h2>
 					<span id="season-select_span">
 						<label for="season-select">Season:</label>
 						<select name="season" id="season-select">
@@ -65,11 +65,11 @@ <h2>Task: <span id="task-id-label">{{ task_id }}</span></h2>
 								d="M13 13h3a3 3 0 0 0 0-6h-.025A5.56 5.56 0 0 0 16 6.5 5.5 5.5 0 0 0 5.207 5.021C5.137 5.017 5.071 5 5 5a4 4 0 0 0 0 8h2.167M10 15V6m0 0L8 8m2-2 2 2"
 							/>
 						</svg>
-						<h3>Click to upload CSV or zip, or drag and drop</h3>
+						<h3>Click to upload CSV, zip, or .pecan, or drag and drop</h3>
 					</label>
 					<input
 						type="file"
-						accept=".csv,text/csv,.zip,application/zip,application/x-zip-compressed"
+						accept=".csv,text/csv,.zip,application/zip,application/x-zip-compressed,.pecan"
 						id="drop_zone-input"
 						multiple
 						style="display: none"