Merge pull request #495 from HSLdevcom/modify-hsl-preprocessing

AB#235 - Add documentation for OSM preprocessing and change instructions file to bash script

Author: vesameskanen

README.md

@@ -63,6 +63,8 @@ It is possible to change the behaviour of the data builder by defining environme
   - `USE_PREBUILT_STREET_GRAPH` to use the prebuilt street graph to finish a complete graph build
   - All other values default to `NO_SPLIT_BUILD` which indicates that the build is run as normal
 - (Optional) `USE_SEEDED_OSM` skips OSM updating and uses existing seed version
+- (Optional) `SKIP_OSM_PREPROCESSING` skips OSM preprocessing even if an instruction file is defined
+- (Optional) `SKIP_OTP_TESTS` skips OTP tests
 
 ### Data processing steps
 
@@ -175,3 +177,16 @@ It is possible to change the behaviour of the data builder by defining environme
 Contains tools, such as the OneBusAway gtfs filter, for gtfs manipulation.
 It uses the [opentransitsoftwarefoundation/onebusaway-gtfs-transformer-cli](https://registry.hub.docker.com/r/opentransitsoftwarefoundation/onebusaway-gtfs-transformer-cli) as the base image.
 These tools are packaged inside a docker container and are used during the data build process.
+
+#### OSM preprocessing
+
+OSM preprocessing is done if a bash script is defined for a specific config and a specific OSM file.
+See [hsl.sh](hsl/osm-preprocessing/hsl.sh) for an example.
+
+When creating OSM preprocessing instructions you should:
+1. Name the bash file as follows: `<osm_id_of_osm_file>.sh`. Valid file names can be e.g. `hsl.sh` or `southFinland.sh`.
+2. Place the file in the `osm-preprocessing` directory of the config you want to use.
+3. Make sure that the name of the output file is the same as the input file e.g. `hsl.pbf`.
+4. Make sure that you do not reuse input and output filenames in commands.
+For example, do not use: `osmfilter hsl.o5m -o=hsl.o5m ...` you can instead use `osmfilter hsl.o5m -o=hsl2.o5m ...`.
+5. Test the script by running it locally and verifying that the output makes sense.

gulpfile.js

@@ -62,15 +62,26 @@ gulp.task('osm:download', async cb => {
   cb();
 });
 
+gulp.task('osm:copyPreprocessingFiles', () =>
+  gulp
+    .src(`${config.router.id}/osm-preprocessing/*.sh`, noBuf)
+    .pipe(gulp.dest(`${config.dataDir}/${config.router.id}/osm-preprocessing`)),
+);
+
 gulp.task(
   'osm:update',
   gulp.series(
+    'osm:copyPreprocessingFiles',
     'osm:download',
     () =>
       gulp
         .src(`${osmDlDir}/*`, noBuf)
         .pipe(validateBlobSize())
-        .pipe(runOSMPreprocessing(`${config.router.id}/osm-preprocessing`))
+        .pipe(
+          runOSMPreprocessing(
+            `${config.dataDir}/${config.router.id}/osm-preprocessing`,
+          ),
+        )
         .pipe(testOTPFile())
         .pipe(gulp.dest(osmDir)),
     () => del(tmpDir),

hsl/osm-preprocessing/hsl.sh

@@ -0,0 +1,13 @@
+#!/usr/bin/env bash
+
+set -x
+
+osmconvert hsl.pbf -o=hsl.o5m
+osmfilter hsl.o5m -o=hsl2.o5m --modify-tags="ref=H0071 to ref=no \
+                                             ref=H0091 to ref=no \
+                                             ref=H0072 to ref=no \
+                                             ref=H0079 to ref=no \
+                                             ref=H0082 to ref=no \
+                                             ref=H0083 to ref=no \
+                                             ref=H0084 to ref=no"
+osmconvert hsl2.o5m -o=hsl.pbf

hsl/osm-preprocessing/hsl.txt

@@ -1,41 +1,12 @@
 const fs = require('fs');
-const { once } = require('events');
-const readline = require('readline');
 const fse = require('fs-extra');
 const exec = require('child_process').exec;
 const through = require('through2');
 const { dataDir, constants, dataToolImage } = require('../config');
 const { postSlackMessage, createDir } = require('../util');
 
-async function readPreprocessingInstructions(preprocessingInstructionsFile) {
-  const preprocessingInstructions = [];
-  const errorLines = [];
-  const rl = readline.createInterface({
-    input: fs.createReadStream(preprocessingInstructionsFile),
-  });
-  rl.on('line', line => {
-    if (/^(osmconvert|osmfilter|osmupdate).*$/.test(line)) {
-      preprocessingInstructions.push(line);
-    } else if (line === '') {
-      process.stdout.write(
-        'Skipping empty line in ' + preprocessingInstructionsFile + '\n',
-      );
-    } else {
-      errorLines.push(line);
-    }
-  });
-  await once(rl, 'close');
-  if (errorLines.length > 0) {
-    throw Error(
-      `${errorLines.join(', ')} are not valid preprocessing instructions!\n`,
-    );
-  }
-  return `-c '${preprocessingInstructions.join(' && ')}'`;
-}
-
 /**
- * Runs the instructions listed in the OSM preprocessing file.
- * Only the osmfilter, osmconvert, and osmupdate commands can be used.
+ * Runs the instructions listed in the OSM preprocessing bash script.
  */
 function preprocessWithFile(
   osmFile,
@@ -47,7 +18,7 @@ function preprocessWithFile(
   const lastLog = [];
 
   return new Promise((resolve, reject) => {
-    const preprocessingInstructionsFile = `${osmPreprocessingDir}/${osmId}.txt`;
+    const preprocessingInstructionsFile = `${osmPreprocessingDir}/${osmId}.sh`;
 
     if (!fs.existsSync(osmFile)) {
       reject(new Error(`${osmFile} does not exist!\n`));
@@ -73,15 +44,13 @@ function preprocessWithFile(
         const r = fs.createReadStream(osmFile);
         r.on('end', async () => {
           try {
-            const concatenatedInstructions =
-              await readPreprocessingInstructions(
-                preprocessingInstructionsFile,
-              );
             process.stdout.write(
-              'Running command: ' + concatenatedInstructions + '\n',
+              'Running commands from file: ' +
+                preprocessingInstructionsFile +
+                '\n',
             );
             const preprocessingCommand = exec(
-              `docker run -v ${folder}:/tmp/osm-preprocessing:rw -w /tmp/osm-preprocessing --rm --entrypoint /bin/bash ${dataToolImage} ${concatenatedInstructions}`,
+              `docker run -v ${folder}:/tmp/osm-preprocessing:rw -v ${preprocessingInstructionsFile}:/tmp/preprocessing.sh:ro -w /tmp/osm-preprocessing --rm --entrypoint /bin/bash ${dataToolImage} /tmp/preprocessing.sh`,
               { maxBuffer: constants.BUFFER_SIZE },
             );
             preprocessingCommand.on('exit', function (c) {