build: Fix release and improve TTL message (#267)

* fix: retention date grace on import

* feat: More granular TTL for shortly expiring datasets

* build: use GITHUB_TOKEN for checkout 4adb429

Author: googlemaps-bot

.releaserc

@@ -1,6 +1,7 @@
 {
   "name": "dune-buggy",
   "version": "1.23.2",
+  "private": true,
   "description": "",
   "main": "index.js",
   "repository": {
@@ -99,4 +100,4 @@
       "last 1 safari version"
     ]
   }
-}
+}

package-lock.json

@@ -20,7 +20,7 @@ import {
 import _ from "lodash";
 import { getQueryStringValue, setQueryStringValue } from "./queryString";
 import "./global.css";
-import { log } from "./Utils";
+import Utils, { log } from "./Utils";
 import { toast, ToastContainer } from "react-toastify";
 import "react-toastify/dist/ReactToastify.css";
 import { ALL_TOGGLES, getVisibleToggles } from "./MapToggles";
@@ -427,7 +427,7 @@ class App extends React.Component {
               const timeLeftMs = retentionDate - now;
               const daysLeft = timeLeftMs / (1000 * 60 * 60 * 24);
               if (daysLeft <= 10) {
-                log(`Dataset ${index + 1} will expire in ${Math.ceil(daysLeft)} days.`, "warn");
+                log(`Dataset ${index + 1} will expire in ${Utils.formatTTLRemaining(timeLeftMs)}.`, "warn");
               }
             }
           }
@@ -829,7 +829,7 @@ class App extends React.Component {
         log(`Dataset ${index + 1} has expired and was deleted (Retention limit reached).`, "error");
         return "Expired";
       } else if (daysLeft <= 10) {
-        log(`Dataset ${index + 1} will expire in ${Math.ceil(daysLeft)} days.`, "warn");
+        log(`Dataset ${index + 1} will expire in ${Utils.formatTTLRemaining(timeLeftMs)}.`, "warn");
         return "Warning";
       }
       return "Valid";

package.json

@@ -27,6 +27,34 @@ class Utils {
     }
     return timeStr;
   }
+
+  /**
+   * Formats a duration for TTL warnings.
+   * > 1 day: "X days"
+   * 1h - 24h: "X hours"
+   * 1m - 60m: "X minutes"
+   * < 1m: "X seconds"
+   */
+  static formatTTLRemaining(ms) {
+    const days = ms / (1000 * 60 * 60 * 24);
+    const hours = ms / (1000 * 60 * 60);
+    const minutes = ms / (1000 * 60);
+    const seconds = ms / 1000;
+
+    if (days >= 1) {
+      const d = Math.ceil(days);
+      return `${d} day${d > 1 ? "s" : ""}`;
+    } else if (hours >= 1) {
+      const h = Math.ceil(hours);
+      return `${h} hour${h > 1 ? "s" : ""}`;
+    } else if (minutes >= 1) {
+      const m = Math.ceil(minutes);
+      return `${m} minute${m > 1 ? "s" : ""}`;
+    } else {
+      const s = Math.ceil(seconds);
+      return `${s} second${s !== 1 ? "s" : ""}`;
+    }
+  }
 }
 
 /**

src/App.js

@@ -229,14 +229,40 @@ function isRestrictedLog(row) {
   return row.jsonPayload?.["@type"]?.includes("Restricted") || false;
 }
 
+function calculateRetentionDate(logsArray) {
+  let oldestTimestamp = Infinity;
+  logsArray.forEach((row) => {
+    const ts = new Date(
+      row.timestamp || row.insertTimestamp || row.jsonPayload?.timestamp || row.jsonpayload?.timestamp
+    ).getTime();
+    if (!isNaN(ts) && ts < oldestTimestamp) {
+      oldestTimestamp = ts;
+    }
+  });
+
+  let retentionDateIdentifier = null;
+  if (oldestTimestamp !== Infinity) {
+    const fiftyFiveDaysMs = 55 * 24 * 60 * 60 * 1000;
+    const oneHourMs = 60 * 60 * 1000;
+    const expirationBasedOnLogs = oldestTimestamp + fiftyFiveDaysMs;
+    const minimumRetention = Date.now() + oneHourMs;
+    const chosenRetention = new Date(Math.max(expirationBasedOnLogs, minimumRetention));
+    retentionDateIdentifier = chosenRetention.toISOString();
+  } else {
+    retentionDateIdentifier = new Date(Date.now() + 60 * 60 * 1000).toISOString();
+  }
+  return retentionDateIdentifier;
+}
+
 export function ensureCorrectFormat(data) {
   let logsArray;
   //Handle if data is not array (like when reading a file).
   if (!Array.isArray(data)) {
-    // If it's already in the correct format, return it as is.
+    // If it's already in the correct format, return it as is, BUT RE-CALCULATE TTL for grace period.
     if (data && data.rawLogs && Array.isArray(data.rawLogs)) {
       return {
         ...data,
+        retentionDate: calculateRetentionDate(data.rawLogs),
         APIKEY: data.APIKEY || DEFAULT_API_KEY,
       };
     } else {
@@ -337,26 +363,8 @@ export function ensureCorrectFormat(data) {
 
   if (!hasPoints) log("Bounds Calculation Failed: Could not find vehicle location data in any row.");
 
-  // Calculate retention date
-  let oldestTimestamp = Infinity;
-  logsArray.forEach((row) => {
-    const ts = new Date(row.timestamp || row.insertTimestamp || row.jsonPayload?.timestamp).getTime();
-    if (!isNaN(ts) && ts < oldestTimestamp) {
-      oldestTimestamp = ts;
-    }
-  });
-
-  let retentionDateIdentifier = null;
-  if (oldestTimestamp !== Infinity) {
-    const fiftyFiveDaysMs = 55 * 24 * 60 * 60 * 1000;
-    const oneHourMs = 60 * 60 * 1000;
-    const expirationBasedOnLogs = oldestTimestamp + fiftyFiveDaysMs;
-    const minimumRetention = Date.now() + oneHourMs;
-    retentionDateIdentifier = new Date(Math.max(expirationBasedOnLogs, minimumRetention)).toISOString();
-  } else {
-    // If no valid timestamps found, default to 1 hour from now for safety
-    retentionDateIdentifier = new Date(Date.now() + 60 * 60 * 1000).toISOString();
-  }
+  // Calculate retention date using the helper
+  const retentionDateIdentifier = calculateRetentionDate(logsArray);
 
   return {
     APIKEY: DEFAULT_API_KEY,

src/Utils.js

@@ -257,4 +257,21 @@ describe("ensureCorrectFormat TTL Logic", () => {
     // Simple regex to check ISO format YYYY-MM-DDTHH:mm:ss.sssZ
     expect(result.retentionDate).toMatch(/\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}.\d{3}Z/);
   });
+
+  it("should recalculate retentionDate on import, ignoring stale values in file", () => {
+    // A file might include a retentionDate.
+    const staleDate = new Date(Date.now() - 100 * ONE_DAY_MS).toISOString();
+    const mockExportedFile = {
+      rawLogs: [{ timestamp: new Date(Date.now() - 200 * ONE_DAY_MS).toISOString(), jsonPayload: { test: 1 } }],
+      retentionDate: staleDate,
+      APIKEY: "abc",
+    };
+
+    const result = ensureCorrectFormat(mockExportedFile);
+    const retention = new Date(result.retentionDate).getTime();
+    const expectedMin = Date.now() + ONE_HOUR_MS;
+
+    expect(result.retentionDate).not.toBe(staleDate);
+    expect(retention).toBeGreaterThanOrEqual(expectedMin - 1000);
+  });
 });