fix: retry+validate live fetches and trim cumulative first-day in daily graph

Hardens the hourly build against the silent-reset bug: fetchWithRetries
retries the live download_history / download_counts URLs three times,
validates the JSON shape, and exits non-zero on final failure unless
ALLOW_HISTORY_RESET=true is explicitly set. Daily-mode chart now always
trims the first datapoint (which is the absolute cumulative count, not a
diff), eliminating the giant first-day spike when history is short.

Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>

Author: abose

build/downloadCounts.js

@@ -1,5 +1,6 @@
 const fs = require('fs');
 const {RELEASE_ASSET_INSTALLER_SUFFIXES, RELEASE_ASSET_UPDATER_SUFFIXES} = require("./constants");
+const {fetchWithRetries} = require("./fetchWithRetries");
 
 const ALL_INSTALLER_SUFFIXES = Object.values(RELEASE_ASSET_INSTALLER_SUFFIXES);
 const ALL_UPDATER_SUFFIXES = Object.values(RELEASE_ASSET_UPDATER_SUFFIXES);
@@ -31,27 +32,17 @@ function computeTotalDownloads(releases) {
     return {totalInstallerDownloads, totalUpdaterDownloads};
 }
 
-function isRunningInGitHubActions() {
-    return process.env.GITHUB_ACTIONS === 'true';
+const DOWNLOAD_COUNTS_URL = "https://public-stats.phcode.io/generated/download_counts.json";
+
+function validateCountsPayload(data) {
+    if (!data || typeof data !== 'object') return 'response was not an object';
+    if (typeof data.totalDownloads !== 'number') return 'missing totalDownloads';
+    if (!data.timestamp) return 'missing timestamp';
+    return null;
 }
 
-const DOWNLOAD_COUNTS_URL = "https://public-stats.phcode.io/generated/download_counts.json";
 async function getCurrentDownloadData() {
-    if(isRunningInGitHubActions()){
-        // in github actions, there can be times when fetch fails in the automated hourly
-        // workflows. In that case we should fail early, else it will reset the history
-        // if we do try catch and return null. We dont want automated workflows resetting
-        // history.
-        const fetchedData = await fetch(DOWNLOAD_COUNTS_URL);
-        return fetchedData.json();
-    }
-    try{
-        const fetchedData = await fetch(DOWNLOAD_COUNTS_URL);
-        return await fetchedData.json();
-    } catch (e) {
-        console.error("No previous data", e);
-    }
-    return null;
+    return fetchWithRetries(DOWNLOAD_COUNTS_URL, validateCountsPayload, 'download_counts');
 }
 
 async function updateDownloadStats(releases) {

build/downloadHistory.js

@@ -1,5 +1,6 @@
 const fs = require('fs');
 const {SUFFIX_TO_NAME_MAP, RELEASE_ASSET_SUFFIXES_ALL} = require("./constants");
+const {fetchWithRetries} = require("./fetchWithRetries");
 
 const ALL_SUFFIXES = Object.values(RELEASE_ASSET_SUFFIXES_ALL);
 function getAssetType(assetName) {
@@ -125,29 +126,21 @@ function toStr(obj) {
     return JSON.stringify(obj, null, 2);
 }
 
-function isRunningInGitHubActions() {
-    return process.env.GITHUB_ACTIONS === 'true';
-}
-
 const DOWNLOAD_HISTORY_URL = "https://public-stats.phcode.io/generated/download_history.json";
-async function getCurrentHistoryData() {
-    if(isRunningInGitHubActions()){
-        // in github actions, there can be times when fetch fails in the automated hourly
-        // workflows. In that case we should fail early, else it will reset the history
-        // if we do try catch and return null. We dont want automated workflows resetting
-        // history.
-        const fetchedData = await fetch(DOWNLOAD_HISTORY_URL);
-        return fetchedData.json();
-    }
-    try{
-        const fetchedData = await fetch(DOWNLOAD_HISTORY_URL);
-        return await fetchedData.json();
-    } catch (e) {
-        console.error("No previous data", e);
+
+function validateHistoryPayload(data) {
+    if (!data || typeof data !== 'object') return 'response was not an object';
+    if (!data.prodReleaseHistory || typeof data.prodReleaseHistory !== 'object') {
+        return 'missing prodReleaseHistory';
     }
+    if (Object.keys(data.prodReleaseHistory).length === 0) return 'prodReleaseHistory is empty';
     return null;
 }
 
+async function getCurrentHistoryData() {
+    return fetchWithRetries(DOWNLOAD_HISTORY_URL, validateHistoryPayload, 'download_history');
+}
+
 async function updateDownloadHistory(releases) {
     const existingHistory = await getCurrentHistoryData();
     const existingProdReleaseHistory = (existingHistory && existingHistory.prodReleaseHistory) ?

build/fetchWithRetries.js

@@ -0,0 +1,41 @@
+async function sleep(ms) {
+    return new Promise(resolve => setTimeout(resolve, ms));
+}
+
+// Fetches `url`, retries on transient failures, and validates the parsed JSON
+// against `validate(data)` (returns a string describing the problem, or null).
+// On final failure: exits the process unless ALLOW_HISTORY_RESET=true is set,
+// in which case it returns null so the caller can proceed with a blank slate.
+async function fetchWithRetries(url, validate, label, attempts = 3) {
+    let lastError;
+    for (let attempt = 1; attempt <= attempts; attempt++) {
+        try {
+            const response = await fetch(url);
+            if (!response.ok) {
+                throw new Error(`HTTP ${response.status} ${response.statusText}`);
+            }
+            const data = await response.json();
+            const problem = validate(data);
+            if (problem) {
+                throw new Error(`unexpected payload: ${problem}`);
+            }
+            return data;
+        } catch (error) {
+            lastError = error;
+            console.warn(`[${label}] attempt ${attempt}/${attempts} failed: ${error.message}`);
+            if (attempt < attempts) {
+                await sleep(1000 * attempt);
+            }
+        }
+    }
+    if (process.env.ALLOW_HISTORY_RESET === 'true') {
+        console.warn(`[${label}] ALLOW_HISTORY_RESET=true; proceeding with blank slate after ${attempts} failed attempts.`);
+        return null;
+    }
+    console.error(`[${label}] fetch failed ${attempts} times. Last error: ${lastError && lastError.message}`);
+    console.error(`[${label}] Refusing to overwrite live history with a blank slate.`);
+    console.error(`[${label}] If you genuinely want to start fresh, re-run with ALLOW_HISTORY_RESET=true.`);
+    process.exit(1);
+}
+
+exports.fetchWithRetries = fetchWithRetries;

docs/download-graphs.html

@@ -249,6 +249,8 @@ <h1>Individual Assets <span class="version-select-text"></span></h1>
         setSelectedOption(timePeriod);
 
         function computeDailyDownloads(stats) {
+            // The first entry is the cumulative count up to that day, not a daily
+            // diff — callers must trim it from the rendered series.
             const dailyDownloads = [];
             if(stats.length >= 1){
                 dailyDownloads.push(stats[0]);
@@ -271,16 +273,15 @@ <h1>Individual Assets <span class="version-select-text"></span></h1>
                 computeDailyData = false;
             }
             let shouldTrimFirstElem = false;
-            if(computeDailyData && (timeSeries.length > days || compareVersions.satisfies(version, "<=3.8.8"))){
-                // if we have more stats, then we add one more day so that we can compute daily downloads
-                // For eg, if we are computing 7 day daily downloads, and we have 8 days stats, we compute the 7th
-                // day downloads by subtracting 7th dya amd 8th day downloads. but the array will have 8 entries,
-                // the first being the absolute 8th day download count, which we should trim.
-
-                // for versions smaller than 3.8.8, we started creating history data mid way and we need to trim the
-                // first element which is the aggregate of several months of downloads, not what we intend for daily
-                // downloads
-                days = days + 1;
+            if(computeDailyData){
+                // In daily mode the first datapoint is the absolute cumulative count
+                // (and for versions <=3.8.8 it's months of aggregated downloads, plus
+                // resets can leave the earliest entry as a huge cumulative baseline).
+                // Always trim it. If we have more history than requested, extend the
+                // slice by one day so the chart still spans the full window.
+                if(timeSeries.length > days){
+                    days = days + 1;
+                }
                 shouldTrimFirstElem = true;
             }
             const trimIndex = Math.max(timeSeries.length - days, 0); // Calculate index to start slicing from