Merge pull request #56 from firelab/timestamps

add timestamps to frame retrieval API for #45; fix off-by-1 error #57; css for #53

Author: robertmcabee

README.md

@@ -38,7 +38,6 @@ And then there's a redis instance living at port 6379 where all the job status a
 
 ### Todo before MSU handoff
 
-- [ ] Switching to a new station view pauses slide deck playthrough
 - [ ] Add a "clear" button to the map that clears all job assets from the map
 - [ ] Discuss pixel-width of gust fronts written to output file next team meeting
 - [ ] Diff the NFGDA code used in nfgda_service with the original NFGDA code, see if there are any useful features we're missing out on or bugs we introduced

backend/apis/retrieve_frames.py

@@ -3,11 +3,14 @@
 """
 
 import os
-from flask import send_file, abort
+import json
+from flask import send_file, abort, make_response
 
 
 def get_frame(job_id: str, index: int):
-    """Return a single GeoTIFF frame file for the given job and frame index."""
+    """Return a single GeoTIFF frame file for the given job and frame index.
+        Includes an X-Frame-Timestamp header of the radar observation time.
+    """
     job_dir = "/processed_data/" + job_id
     if not os.path.exists(job_dir):
         abort(404, description="Job not found")
@@ -16,8 +19,33 @@ def get_frame(job_id: str, index: int):
     if not os.path.exists(frame_path):
         abort(404, description="Frame not found")
 
-    return send_file(
-        frame_path,
-        mimetype="image/tiff",
-        as_attachment=False
+    response = make_response(
+        send_file(
+            frame_path,
+            mimetype="image/tiff",
+            as_attachment=False,
+        )
     )
+
+    # attach the radar observation timestamp if available
+    timestamp = get_frame_timestamp(job_dir, index)
+    if timestamp is not None:
+        response.headers["X-Frame-Timestamp"] = timestamp
+
+    return response
+
+
+def get_frame_timestamp(job_dir: str, index: int) -> str | None:
+    """Read the per-frame observation timestamp from the job's manifest file.
+    Returns an ISO 8601 UTC string (e.g. "2024-07-07T01:22:24Z") or
+    None if the manifest is missing or the frame index has no entry.
+    """
+    manifest_path = os.path.join(job_dir, "timestamps.json")
+    if not os.path.exists(manifest_path):
+        return None
+    try:
+        with open(manifest_path) as f:
+            manifest = json.load(f)
+        return manifest.get(str(index))
+    except Exception:
+        return None

frontend/assets/github.svg

@@ -49,17 +49,24 @@ export default function App() {
   const fetchRadarData = async () => {
     try {
       // ---- validate request ----
+      //check if station is selected
       if (!selectedStation?.properties?.station_id) {
         setErrorMessage("Please select a radar station first.");
         return;
       }
+      // check if endTime is in the past
+      if (!currentMode && selectedDateTime.isAfter(dayjs().subtract(Number(selectedDuration), "minute"))) {
+        setErrorMessage(
+          `Please select a start time at least ${selectedDuration} minutes in the past`,
+        );
+        return;
+      }
       setErrorMessage("");
       const durationMinutes = Number(selectedDuration);
       const requestBody = {
         stationId: selectedStation.properties.station_id,
       };
       if (!currentMode) {
-        console.log("using historical data");
         requestBody.startUtc = selectedDateTime
           .utc()
           .format("YYYY-MM-DDTHH:mm:ss[Z]");
@@ -68,7 +75,6 @@ export default function App() {
           .utc()
           .format("YYYY-MM-DDTHH:mm:ss[Z]");
       } else {
-        console.log("using current data");
         requestBody.startUtc = dayjs()
           .subtract(durationMinutes + 15, "minute")
           .utc()
@@ -119,23 +125,31 @@ export default function App() {
   };
 
   // fetch frames once the job is completed and the jobId and numFrames are set
-  useEffect(() => {
-    async function fetchFrames() {
-      if (jobStatus !== "COMPLETED" || !jobId || numFrames <= 0) return;
-      console.log(`attempting to fetch ${numFrames} frames for job ${jobId}`);
-      try {
-        const promises = Array.from({ length: numFrames }, (_, i) =>
-          fetch(`/apis/jobs/${jobId}/frames/${i}`)
-            .then((res) => {
-              if (!res.ok) throw new Error(`Failed frame ${i}`);
-              return res.blob();
-            })
-            .then((blob) => URL.createObjectURL(blob)),
-        );
-        const urls = await Promise.all(promises);
-        setFrames(urls);
-        setIsPlaying(true);
-        console.log("Frames fetched successfully: ", urls);
+useEffect(() => {
+  async function fetchFrames() {
+    if (jobStatus !== "COMPLETED" || !jobId || numFrames <= 0) return;
+    console.log(`attempting to fetch ${numFrames} frames for job ${jobId}`);
+    try {
+      const promises = Array.from({ length: numFrames }, async (_, i) => {
+        const res = await fetch(`/apis/jobs/${jobId}/frames/${i}`);
+        if (res.status === 404) {
+            console.warn(`Frame ${i} gave 404 - skipping`);
+            return null;
+          }
+          if (!res.ok) throw new Error(`Failed frame ${i}`);
+          const timestamp = res.headers.get("x-frame-timestamp");
+          const blob = await res.blob();
+          return {
+            url: URL.createObjectURL(blob),
+            timestamp,
+            index: i,
+          };
+        });
+        const frames = await Promise.all(promises);
+        frames.filter(Boolean).sort((a, b) => a.index - b.index);
+        setFrames(frames);
+        setIsPlaying(frames.length > 0);
+        console.log("Frames fetched successfully: ", frames);
       } catch (err) {
         console.error("Error fetching frames:", err);
       }
@@ -219,7 +233,10 @@ export default function App() {
   return (
     <div>
       <div className="flex flex-col md:flex-row w-full">
-        <div className="md:mt-12 p-4 gap-4 md:w-92 w-full flex flex-col">
+        <div className="md:mt-6 p-4 gap-4 md:w-92 w-full flex flex-col">
+          <div className="mb-6 items-center gap-4">
+            <h1 className="text-3xl font-light">Gust Front Web App</h1>
+          </div>
           {/* Station Selector */}
           <RadarStationDropdown
             stations={stations}
@@ -360,11 +377,10 @@ export default function App() {
                       <p className="min-w-fit px-3">{geotiffOpacity}%</p>
                     </div>
                     <div className="w-1/2">
-                      {/*TODO: Actual timestamp will go here: */}
                       <p className="text-sm text-right">
-                        {selectedDateTime
-                          .tz(timezone)
-                          .format("YYYY-MM-DD HH:mm z")}
+                        {frames[currentFrameIndex]?.timestamp
+                          ? `${dayjs(frames[currentFrameIndex].timestamp).tz(timezone).format("YYYY-MM-DD HH:mm z")}`
+                          : "No timestamp available"}
                       </p>
                     </div>
                   </div>
@@ -378,8 +394,8 @@ export default function App() {
           <div
             className={
               jobStatus === "PROCESSING" ||
-              jobStatus === "REQUESTED" ||
-              jobStatus === "PENDING"
+                jobStatus === "REQUESTED" ||
+                jobStatus === "PENDING"
                 ? "opacity-50"
                 : ""
             }
@@ -395,6 +411,12 @@ export default function App() {
           </div>
         </div>
       </div>
+      <footer className=" m-4 absolute bottom-0 left-0 hidden md:block shadow-xl hover:shadow-sm transition-all">
+        <a className="outline-1 hover:text-black opacity-50 hover:opacity-100 transition-all rounded-md p-2 flex gap-2 items-center" href="https://github.com/firelab/gust-front-detection-webapp" target="_blank" rel="noopener noreferrer">
+          <p className="">Code</p>
+          <img src="/assets/github.svg" alt="GitHub" className="w-6 h-6" />
+        </a>
+      </footer>
     </div>
   );
 }

frontend/src/App.jsx

@@ -10,8 +10,8 @@ export default function GeotiffAnimation({ frames, currentIndex, opacity }) {
   useEffect(() => {
     async function processFrames() {
       const processed = await Promise.all(
-        frames.map(async (url) => {
-          const res = await fetch(url);
+        frames.map(async (frame) => {
+          const res = await fetch(frame.url);
           const buf = await res.arrayBuffer();
           const geo = await parseGeoraster(buf);
 

frontend/src/components/GeoTiffAnimation.jsx

@@ -1 +1,5 @@
 @import 'tailwindcss';
+
+@theme {
+  --font-sans: "Roboto", "Helvetica", "Arial", sans-serif;
+}

frontend/src/index.css

@@ -78,7 +78,7 @@ def process_geotiff_output(job_id: str) -> None:
         redis_client.hset(f"job:{job_id}", mapping={"status": "FAILED", "error_message": result})
     else:
         logger.info("successfully generated GeoTIFF series for job %s", job_id)
-        redis_client.hset(f"job:{job_id}", mapping={"status": "COMPLETED", "num_frames": len(os.listdir(f"/processed_data/{job_id}"))})
+        redis_client.hset(f"job:{job_id}", mapping={"status": "COMPLETED", "num_frames": len(os.listdir(f"/processed_data/{job_id}")) - 1}) # subtract 1 for the timestamps.json file
 
 async def run_and_release_job(job_id: str) -> None:
     """Run a job and release the semaphore when finished."""

nfgda_service/main.py

@@ -94,10 +94,21 @@ def generate_geotiff_output(job_id: str, redis_client: redis.Redis):
 
     radar_lon, radar_lat = radar_coords
 
-    # process each file
+    # process each file and collect per-frame observation timestamps
+    timestamps = {}
     for i, file in enumerate(files):
         logger.info(f'processing file {i+1} of {len(files)} into GeoTIFF format')
-        project_data(os.path.join(f"/nfgda_output/{job_id}/nfgda_detection", file), radar_lat, radar_lon, out_dir, i)
+        npz_path = os.path.join(f"/nfgda_output/{job_id}/nfgda_detection", file)
+        ts = extract_timestamp(npz_path)
+        if ts is not None:
+            timestamps[i] = ts
+        project_data(npz_path, radar_lat, radar_lon, out_dir, i)
+
+    # write manifest so the API can serve observation timestamps per frame
+    manifest_path = os.path.join(out_dir, "timestamps.json")
+    with open(manifest_path, "w") as f:
+        json.dump(timestamps, f)
+    logger.info(f"Wrote timestamp manifest with {len(timestamps)} entries to {manifest_path}")
 
 
 def get_radar_coords(station_id: str, redis_client: redis.Redis) -> tuple[float, float]:
@@ -140,6 +151,24 @@ def _reflectivity_to_rgba(refl: np.ndarray, nfout: np.ndarray) -> np.ndarray:
     return rgba
 
 
+def extract_timestamp(npz_path: str) -> str | None:
+    """Return the radar observation time from a detection .npz as an ISO 8601 UTC string,
+    or None if the key is absent or unparseable."""
+    try:
+        data = np.load(npz_path, allow_pickle=True)
+        if "timestamp" not in data:
+            return None
+        ts = data["timestamp"]
+        # numpy.datetime64 (0-d array) -> Python datetime -> ISO string
+        # .item() is required to get the Python scalar; .astype(object) keeps it
+        # as a 0-d ndarray which has no .strftime()
+        ts_dt = ts.astype("datetime64[s]").item()
+        return ts_dt.strftime("%Y-%m-%dT%H:%M:%SZ")
+    except Exception as e:
+        logger.warning(f"Could not extract timestamp from {npz_path}: {e}")
+        return None
+
+
 def project_data(npz_path: str, radar_lat: float, radar_lon: float, out_dir: str, index: int) -> None:
 
     # ---------------------------

nfgda_service/process_output.py

@@ -112,7 +112,8 @@ def main():
         print(f"\n  GET {url}")
         resp = requests.get(url)
         if resp.status_code == 200:
-            print(f"    ✓ frame {index}: {resp.status_code} ({len(resp.content)} bytes)")
+            ts = resp.headers.get("X-Frame-Timestamp", "no timestamp")
+            print(f"    ✓ frame {index}: {resp.status_code} ({len(resp.content)} bytes)  timestamp={ts}")
         else:
             print(f"    ✗ frame {index}: {resp.status_code} — {resp.text[:200]}")