ref(builds-cacher): cache-only Node server, no fetches or writes

Make _webi/builds-cacher.js and _webi/transform-releases.js read
exclusively from ~/.cache/webi/legacy/<name>.json and remove every code
path that fetched from upstream or wrote to disk. The Go cache daemon
(webicached) is now the sole writer; the Node server is a thin reader.

builds-cacher.js:
- Resolve cache files via Os.homedir() + '/.cache/webi/legacy/' instead
  of the cacheDir argument. Drop the 'caches' constructor parameter.
- Remove getLatestBuilds / getLatestBuildsInner — they require()d
  per-package releases.js modules, fetched upstream, and wrote
  <yyyy-mm>/<name>.json + .updated.txt to disk.
- Remove the process.nextTick stale-refresh hook in _doGetPackages.
  Cold reads return what's on disk; if the file is missing, return
  empty meta instead of fetching.
- Remove freshenRandomPackage and its supporting state
  (bc._staleNames, bc._freshenTimeout, bc._staleAge). The hourly
  background freshener competed with webicached for the same files.
- In getProjectTypeByEntry, decide selfhosted vs valid by probing for
  the cache file rather than require()-ing releases.js. Drop the
  not_found / 'PROBLEM/SOLUTION/npm clean-install' diagnostic in
  getProjectsByType — the cache-file probe replaces the module-load
  failure mode.

transform-releases.js:
- Remove Releases.get and the _normalize import. Replace
  getCachedReleases's fetch+race+stale-age machinery with a single
  Fs.readFile of ~/.cache/webi/legacy/<pkg>.json.
- Drop the in-process version re-sort in createFormatsSorter; the
  cache file is already version-sorted by webicached, so the sorter
  only re-orders within the same version.

No callers' public signatures change. Every other file is untouched —
the per-package releases.js modules, _common/*.js fetchers, and
_webi/normalize.js still exist on disk but are no longer reachable
from the request path.

Author: coolaj86

_webi/builds-cacher.js

@@ -3,13 +3,16 @@
 var BuildsCacher = module.exports;
 
 let Fs = require('node:fs/promises');
+let Os = require('node:os');
 let Path = require('node:path');
 
+let LEGACY_CACHE_DIR = Path.join(Os.homedir(), '.cache/webi/legacy');
+
 let HostTargets = require('./build-classifier/host-targets.js');
 let Lexver = require('./build-classifier/lexver.js');
 let Triplet = require('./build-classifier/triplet.js');
 
-var ALIAS_RE = /^alias: (\w+)$/m;
+var ALIAS_RE = /^alias: ([\w.-]+)$/m;
 
 var LEGACY_ARCH_MAP = {
   '*': 'ANYARCH',
@@ -126,61 +129,8 @@ async function readFirstBytes(path) {
   return str;
 }
 
-let promises = {};
-async function getLatestBuilds(Releases, installersDir, cacheDir, name, date) {
-  console.info(`[INFO] getLatestBuilds: ${name}`);
-
-  if (!Releases) {
-    Releases = require(`${installersDir}/${name}/releases.js`);
-  }
-  // TODO update all releases files with module.exports.xxxx = 'foo';
-  if (!Releases.latest) {
-    Releases.latest = Releases;
-  }
-
-  let id = `${cacheDir}/${name}`;
-  if (!promises[id]) {
-    promises[id] = Promise.resolve();
-  }
-
-  promises[id] = promises[id].then(async function () {
-    return await getLatestBuildsInner(Releases, cacheDir, name, date);
-  });
-
-  return await promises[id];
-}
-
-async function getLatestBuildsInner(Releases, cacheDir, name, date) {
-  let data = await Releases.latest();
-
-  if (!date) {
-    date = new Date();
-  }
-  let isoDate = date.toISOString();
-  let yearMonth = isoDate.slice(0, 7);
-
-  // TODO hash file
-  let dataFile = `${cacheDir}/${yearMonth}/${name}.json`;
-  // TODO fsstat releases.js vs require-ing time as well
-  let tsFile = `${cacheDir}/${yearMonth}/${name}.updated.txt`;
-
-  let dirPath = Path.dirname(dataFile);
-  await Fs.mkdir(dirPath, { recursive: true });
-
-  let json = JSON.stringify(data, null, 2);
-  await Fs.writeFile(dataFile, json, 'utf8');
-
-  let seconds = date.valueOf();
-  let ms = seconds / 1000;
-  let msStr = ms.toFixed(3);
-  await Fs.writeFile(tsFile, msStr, 'utf8');
-
-  return data;
-}
-
-BuildsCacher.create = function ({ ALL_TERMS, installers, caches }) {
+BuildsCacher.create = function ({ ALL_TERMS, installers }) {
   let installersDir = installers;
-  let cacheDir = caches;
 
   if (!ALL_TERMS) {
     ALL_TERMS = Triplet.TERMS_PRIMARY_MAP;
@@ -195,7 +145,6 @@ BuildsCacher.create = function ({ ALL_TERMS, installers, caches }) {
   bc._triplets = {};
   bc._targetsByBuildIdCache = {};
   bc._caches = {};
-  bc._staleAge = 15 * 60 * 1000;
   bc._allFormats = {};
   bc._allTriplets = {};
   // Per-name lock: serializes cold-cache getPackages so concurrent
@@ -219,19 +168,6 @@ BuildsCacher.create = function ({ ALL_TERMS, installers, caches }) {
     let entries = await Fs.readdir(installersDir, { withFileTypes: true });
     for (let entry of entries) {
       let meta = await bc.getProjectTypeByEntry(entry);
-      if (meta.type === 'not_found') {
-        let err = meta.detail;
-        console.error('');
-        console.error('PROBLEM');
-        console.error(`    ${err.message}`);
-        console.error('');
-        console.error('SOLUTION');
-        console.error('    npm clean-install');
-        console.error('');
-        throw new Error(
-          '[SANITY FAIL] should never have missing modules in prod',
-        );
-      }
       dirs[meta.type][entry.name] = meta.detail;
     }
 
@@ -300,19 +236,17 @@ BuildsCacher.create = function ({ ALL_TERMS, installers, caches }) {
       return { type: 'alias', detail: link };
     }
 
-    let releasesPath = Path.join(path, 'releases.js');
-    try {
-      void require(releasesPath);
-    } catch (err) {
-      if (err.code !== 'MODULE_NOT_FOUND') {
-        return { type: 'errors', detail: err };
-      }
-
-      if (err.message.includes(`Cannot find module '${releasesPath}'`)) {
-        return { type: 'selfhosted', detail: true };
-      }
-
-      return { type: 'not_found', detail: err };
+    let cacheFile = `${LEGACY_CACHE_DIR}/${entry.name}.json`;
+    let hasCacheFile = await Fs.access(cacheFile).then(
+      function () {
+        return true;
+      },
+      function () {
+        return false;
+      },
+    );
+    if (!hasCacheFile) {
+      return { type: 'selfhosted', detail: true };
     }
 
     return { type: 'valid', detail: true };
@@ -337,14 +271,9 @@ BuildsCacher.create = function ({ ALL_TERMS, installers, caches }) {
     return p;
   };
 
-  async function _doGetPackages({ Releases, name, date }) {
-    if (!date) {
-      date = new Date();
-    }
-    let isoDate = date.toISOString();
-    let yearMonth = isoDate.slice(0, 7);
-    let dataFile = `${cacheDir}/${yearMonth}/${name}.json`;
-    let tsFile = `${cacheDir}/${yearMonth}/${name}.updated.txt`;
+  async function _doGetPackages({ name }) {
+    let dataFile = `${LEGACY_CACHE_DIR}/${name}.json`;
+    let tsFile = `${LEGACY_CACHE_DIR}/${name}.updated.txt`;
 
     let tsDate;
     {
@@ -398,7 +327,7 @@ BuildsCacher.create = function ({ ALL_TERMS, installers, caches }) {
       }
     }
     if (!projInfo) {
-      projInfo = await getLatestBuilds(Releases, installersDir, cacheDir, name);
+      return meta;
     }
     let latestProjInfo = await BuildsCacher.transformAndUpdate(
       name,
@@ -409,64 +338,9 @@ BuildsCacher.create = function ({ ALL_TERMS, installers, caches }) {
     );
     bc._caches[name] = latestProjInfo;
 
-    process.nextTick(async function () {
-      let now = date.valueOf();
-      let age = now - projInfo.updated;
-
-      let fresh = age < bc._staleAge;
-      if (fresh) {
-        return;
-      }
-
-      projInfo = await getLatestBuilds(Releases, installersDir, cacheDir, name);
-      let latestProjInfo = BuildsCacher.transformAndUpdate(
-        name,
-        projInfo,
-        meta,
-        date,
-        bc,
-      );
-      bc._caches[name] = latestProjInfo;
-    });
-
-    return projInfo;
+    return latestProjInfo;
   }
 
-  // Makes sure that packages are updated once an hour, on average
-  bc._staleNames = [];
-  bc._freshenTimeout = null;
-  bc.freshenRandomPackage = async function (minDelay) {
-    if (!minDelay) {
-      minDelay = 15 * 1000;
-    }
-
-    if (bc._staleNames.length === 0) {
-      let dirs = await bc.getProjectsByType();
-      bc._staleNames = Object.keys(dirs.valid);
-      bc._staleNames.sort(function () {
-        return 0.5 - Math.random();
-      });
-    }
-
-    let name = bc._staleNames.pop();
-    void (await bc.getPackages({
-      //Releases: Releases,
-      name: name,
-      date: new Date(),
-    }));
-    console.info(`[INFO] freshenRandomPackage: ${name}`);
-
-    let hour = 60 * 60 * 1000;
-    let delay = minDelay;
-    let spread = hour / bc._staleNames.length;
-    let seed = Math.random();
-    delay += seed * spread;
-
-    clearTimeout(bc._freshenTimeout);
-    bc._freshenTimeout = setTimeout(bc.freshenRandomPackage, delay);
-    bc._freshenTimeout.unref();
-  };
-
   /**
    * Given a list of acceptable formats, get the sorted list of of formats.
    * Actually used (as per node _webi/lint-builds.js):

_webi/builds.js

@@ -15,11 +15,8 @@ let bc = BuildsCacher.create({
   caches: CACHE_DIR,
   installers: INSTALLERS_DIR,
 });
-bc.freshenRandomPackage(600 * 1000);
 
 Builds.init = async function () {
-  bc.freshenRandomPackage(600 * 1000);
-
   let dirs = await bc.getProjectsByType();
   let projNames = Object.keys(dirs.valid);