move pruning old versions to beginning of update, make number of versions to keep configurable as KEEP_VERSIONS

Author: tkalvas

task/StorageCleanup.js

@@ -43,9 +43,10 @@ function isCorrectRouter(basePath, file, routerId) {
 }
 
 /*
- * Keeps 10 valid latest versions and removes the rest.
+ * Keeps KEEP_VERSIONS (default 10) valid latest versions and removes the rest.
  */
 function deleteOldVersions(sourceDir, routerId, tag) {
+  const savedCount = process.env.KEEP_VERSIONS ?? '10';
   const basePath = `${sourceDir}/${tag}`;
   if (!fs.existsSync(basePath)) {
     return Promise(res => res());
@@ -55,7 +56,7 @@ function deleteOldVersions(sourceDir, routerId, tag) {
       .readdirSync(basePath)
       .filter(file => isCorrectRouter(basePath, file, routerId))
       .sort(sortByDate)
-      .slice(0, -10);
+      .slice(0, -savedCount);
     return filesToDelete.map(file => del(`${basePath}/${file}/${routerId}`));
   });
 }

task/Update.js

@@ -139,6 +139,8 @@ async function handleCleanup() {
  * This function only builds the street graph with OSM and DEM data.
  */
 async function buildStreetOnlyGraph(name) {
+  await handleCleanup();
+
   await handleSeeding();
 
   await handleOsmAndDemUpdate();
@@ -174,6 +176,8 @@ async function buildStreetOnlyGraph(name) {
  * This function does the whole build.
  */
 async function buildGraph(name) {
+  await handleCleanup();
+
   await handleSeeding();
 
   await handleOsmAndDemUpdate();
@@ -201,8 +205,6 @@ async function buildGraph(name) {
 
   buildAndDeployDockerImages(date);
 
-  await handleCleanup();
-
   if (global.hasFailures) {
     updateSlackMessage(
       `${name} data updated, but partially falling back to older data :boom:`,
@@ -216,6 +218,8 @@ async function buildGraph(name) {
  * This function builds the graph from prebuilt street graph data.
  */
 async function buildWithPrebuiltStreetGraph(name) {
+  await handleCleanup();
+
   await handleSeeding();
 
   await start('gtfs:update');
@@ -241,8 +245,6 @@ async function buildWithPrebuiltStreetGraph(name) {
 
   buildAndDeployDockerImages(date);
 
-  await handleCleanup();
-
   if (global.hasFailures) {
     updateSlackMessage(
       `${name} data updated from prebuilt street only graph, but partially falling back to older data :boom:`,