Change how OSM preprocessing is done. Remove downloads related to preprocessing files.

Author: VillePihlava

config.js

@@ -68,10 +68,6 @@ const osm = {
   ...extraOSM,
 };
 
-const osmPreprocessingURLs = {
-  hsl: 'https://geocoding.blob.core.windows.net/vrk/osm-preprocessing-hsl.txt',
-};
-
 const dem = {
   waltti:
     'https://elevdata.blob.core.windows.net/elevation/waltti/waltti-10m-elevation-model_20190927.tif',
@@ -88,14 +84,6 @@ module.exports = {
   osm: router.osm.map(id => {
     return { id, url: osm[id] };
   }), // array of id, url (OSM data) pairs
-  osmPreprocessingInstructions: router.osm
-    .filter(id => {
-      return osmPreprocessingURLs[id];
-    })
-    .map(id => {
-      return { id, url: osmPreprocessingURLs[id] };
-    }), // array of id, url (OSM preprocessing instruction data) pairs
-  osmPreprocessingURLs,
   dem: router.dem ? [{ id: router.dem, url: dem[router.dem] }] : null, // currently only one DEM file is used
   dataToolImage: `hsldevcom/otp-data-tools:${process.env.TOOLS_TAG || 'v3'}`,
   dataDir: `${process.cwd()}/data`,

gulpfile.js

@@ -30,7 +30,6 @@ const storageCleanup = require('./task/StorageCleanup');
 const seedSourceDir = `${config.dataDir}/router-${config.router.id}`; // e.g. data/router-hsl
 
 const osmDlDir = `${config.dataDir}/downloads/osm`;
-const osmPreprocessingDlDir = `${config.dataDir}/downloads/osm-preprocessing`;
 const demDlDir = `${config.dataDir}/downloads/dem`;
 const gtfsDlDir = `${config.dataDir}/downloads/gtfs`;
 
@@ -60,10 +59,6 @@ gulp.task('osm:download', async cb => {
   createDir(osmDlDir);
   createDir(osmDir);
   await dl(config.osm, osmDlDir);
-  if (config.osmPreprocessingInstructions) {
-    createDir(osmPreprocessingDlDir);
-    await dl(config.osmPreprocessingInstructions, osmPreprocessingDlDir);
-  }
   cb();
 });
 
@@ -75,7 +70,7 @@ gulp.task(
       gulp
         .src(`${osmDlDir}/*`, noBuf)
         .pipe(validateBlobSize())
-        .pipe(runOSMPreprocessing(osmPreprocessingDlDir))
+        .pipe(runOSMPreprocessing(`${config.router.id}/osm-preprocessing`))
         .pipe(testOTPFile())
         .pipe(gulp.dest(osmDir)),
     () => del(tmpDir),

hsl/osm-preprocessing/hsl.txt

@@ -0,0 +1,3 @@
+osmconvert hsl.pbf -o=hsl.o5m
+osmfilter hsl.o5m --modify-tags="ref=H0071 to ref=no ref=H0091 to ref=no ref=H0072 to ref=no" -o=hsl2.o5m
+osmconvert hsl2.o5m -o=hsl.pbf

task/OSMPreprocessing.js

@@ -4,12 +4,7 @@ const readline = require('readline');
 const fse = require('fs-extra');
 const exec = require('child_process').exec;
 const through = require('through2');
-const {
-  dataDir,
-  constants,
-  dataToolImage,
-  osmPreprocessingURLs,
-} = require('../config');
+const { dataDir, constants, dataToolImage } = require('../config');
 const { postSlackMessage, createDir } = require('../util');
 
 async function readPreprocessingInstructions(preprocessingInstructionsFile) {
@@ -45,19 +40,23 @@ async function readPreprocessingInstructions(preprocessingInstructionsFile) {
 function preprocessWithFile(
   osmFile,
   quiet = false,
-  osmPreprocessingDlDir,
+  osmPreprocessingDir,
   osmId,
   osmFileName,
 ) {
   const lastLog = [];
 
   return new Promise((resolve, reject) => {
-    const preprocessingInstructionsFile = `${osmPreprocessingDlDir}/${osmId}.txt`;
+    const preprocessingInstructionsFile = `${osmPreprocessingDir}/${osmId}.txt`;
 
     if (!fs.existsSync(osmFile)) {
       reject(new Error(`${osmFile} does not exist!\n`));
     } else if (!fs.existsSync(preprocessingInstructionsFile)) {
-      reject(new Error(`${preprocessingInstructionsFile} does not exist!\n`));
+      reject(
+        new Error(
+          `No OSM preprocessing instructions for ${osmId}. ${preprocessingInstructionsFile} does not exist!\n`,
+        ),
+      );
     } else {
       createDir(`${dataDir}/tmp`);
       fs.mkdtemp(`${dataDir}/tmp/osm-preprocessing`, (err, folder) => {
@@ -138,7 +137,7 @@ function preprocessWithFile(
 }
 
 module.exports = {
-  runOSMPreprocessing: osmPreprocessingDlDir => {
+  runOSMPreprocessing: osmPreprocessingDir => {
     return through.obj(function (file, encoding, callback) {
       const osmFile = file.history[file.history.length - 1];
       if (process.env.SKIP_OSM_PREPROCESSING) {
@@ -150,19 +149,7 @@ module.exports = {
       const osmFileName = osmFile.split('/').pop();
       // This can be, for example, hsl, finland, or southFinland.
       const osmId = osmFileName.split('.')[0];
-      if (!osmPreprocessingURLs[osmId]) {
-        process.stdout.write(
-          'No OSM preprocessing instructions for ' + osmId + '\n',
-        );
-        return callback(null, file);
-      }
-      preprocessWithFile(
-        osmFile,
-        true,
-        osmPreprocessingDlDir,
-        osmId,
-        osmFileName,
-      )
+      preprocessWithFile(osmFile, true, osmPreprocessingDir, osmId, osmFileName)
         .then(outputContents => {
           if (outputContents) {
             file.contents = outputContents;
@@ -171,7 +158,8 @@ module.exports = {
             callback(null, null);
           }
         })
-        .catch(() => {
+        .catch(err => {
+          process.stdout.write(err.message);
           callback(null, null);
         });
     });