add capability for netex data, introduce VR_bussit in finland as the first netex data

Author: tkalvas

.gitignore

@@ -2,9 +2,13 @@ data
 storage
 node_modules/
 .DS_Store
+.pnp.cjs
+.pnp.loader.mjs
+.yarn/
+yarn.lock
 router-finland.zip
 router-hsl.zip
 router-waltti.zip
 router-waltti-alt.zip
-
+*~
 .tmp

README.md

@@ -17,17 +17,14 @@ the databuilding process can also be called directly from the source tree. The o
 required external dependency is Docker. Docker is used for launching external
 commands that do, for example, data manipulation.
 
-install gulp cli:
-`yarn global add gulp-cli`
-
-install app deps:
-`yarn`
+install application and development dependecies:
+`yarn install`
 
 update osm data:
-`ROUTER_NAME=hsl gulp osm:update`
+`ROUTER_NAME=hsl yarn gulp osm:update`
 
 download new gtfs data for waltti:
-`ROUTER_NAME=waltti gulp gtfs:dl`
+`ROUTER_NAME=waltti yarn gulp gtfs:dl`
 
 #### Configuration
 

finland/config.js

@@ -127,5 +127,13 @@ module.exports = {
       false,
     ),
   ],
+  netex: [
+    {
+      id: 'VR_bussit',
+      url: 'https://mobility.mobility-database.fintraffic.fi/static/vr_bussit.zip',
+      groupFilePattern: '(line)_.*\\.xml',
+      sharedFilePattern: '_.*\\.xml',
+    },
+  ],
   osm: ['finland', 'estonia'],
 };

gulpfile.js

@@ -8,6 +8,7 @@ const prepareFit = require('./task/PrepareFit');
 const mapFit = require('./task/MapFit');
 const { validateBlobSize } = require('./task/BlobValidation');
 const { testOTPFile } = require('./task/OTPTest');
+const { testNetexFile } = require('./task/NetexTest');
 const { runOSMPreprocessing } = require('./task/OSMPreprocessing');
 const seed = require('./task/Seed');
 const {
@@ -22,6 +23,7 @@ const {
   buildOTPStreetOnlyGraphTask,
 } = require('./task/BuildOTPGraph');
 const { renameGTFSFile } = require('./task/GTFSRename');
+const { renameNetexFile } = require('./task/NetexRename');
 const { replaceGTFSFilesTask } = require('./task/GTFSReplace');
 const { extractFilesTask, addFilesTask } = require('./task/ZipTask');
 const { createDir } = require('./util');
@@ -32,10 +34,12 @@ const seedSourceDir = `${config.dataDir}/router-${config.router.id}`; // e.g. da
 const osmDlDir = `${config.dataDir}/downloads/osm`;
 const demDlDir = `${config.dataDir}/downloads/dem`;
 const gtfsDlDir = `${config.dataDir}/downloads/gtfs`;
+const netexDlDir = `${config.dataDir}/downloads/netex`;
 
 const osmDir = `${config.dataDir}/ready/osm`;
 const demDir = `${config.dataDir}/ready/dem`;
 const gtfsDir = `${config.dataDir}/ready/gtfs`;
+const netexDir = `${config.dataDir}/ready/netex`;
 
 const gtfsSeedDir = `${config.dataDir}/seed`;
 const fitDir = `${config.dataDir}/fit`;
@@ -49,6 +53,34 @@ const renamedDir = `${config.dataDir}/renamed`;
 
 const noBuf = { buffer: false }; // options for gulp src
 
+/**
+ * Download netex data
+ */
+gulp.task('netex:download', async cb => {
+  if (!config.router.netex) {
+    return Promise.resolve();
+  }
+  createDir(netexDlDir);
+  createDir(netexDir);
+  await dl(config.router.netex, netexDlDir);
+  cb();
+});
+
+gulp.task('netex:rename', () =>
+  gulp
+    .src(`${netexDlDir}/*`, noBuf)
+    .pipe(renameNetexFile())
+    .pipe(gulp.dest(netexDir)),
+);
+
+gulp.task(
+  'netex:update',
+  gulp.series(
+    'netex:download',
+    'netex:rename',
+  ),
+);
+
 /**
  * Download osm data
  */
@@ -231,6 +263,17 @@ gulp.task(
   ),
 );
 
+gulp.task('netex:del', () => del(netexDir));
+
+gulp.task(
+  'netex:seed',
+  gulp.series('netex:del', () =>
+    gulp
+      .src(`${seedSourceDir}/*-netex.zip`, noBuf)
+      .pipe(gulp.dest(netexDir)),
+  ),
+);
+
 gulp.task('osm:del', () => del(osmDir));
 
 gulp.task(
@@ -271,6 +314,7 @@ gulp.task(
     'dem:seed',
     'osm:seed',
     'gtfs:seed',
+    'netex:seed',
     'seed:cleanup',
   ),
 );

package.json

@@ -37,6 +37,7 @@
     "@eslint/js": "^9.18.0",
     "eslint": "^9.18.0",
     "globals": "^15.14.0",
+    "gulp-cli": "^3.1.0",
     "prettier": "^3.4.2"
   }
 }

task/NetexRename.js

@@ -0,0 +1,15 @@
+const through = require('through2');
+
+module.exports = {
+  renameNetexFile: () => {
+    return through.obj(function (file, encoding, callback) {
+      if (!file.stem.includes('-netex')) {
+        file.stem = file.stem + '-netex';
+      }
+      if (file.extname !== '.zip') {
+        file.extname = '.zip';
+      }
+      callback(null, file);
+    });
+  },
+};

task/NetexTest.js

@@ -0,0 +1,17 @@
+const through = require('through2');
+
+module.exports = {
+  testNetexFile: () => {
+    return through.obj(function (file, encoding, callback) {
+      const otpFile = file.history[file.history.length - 1];
+      if (process.env.SKIP_OTP_TESTS) {
+        process.stdout.write(
+          'OTP test skipped because the SKIP_OTP_TESTS environment variable is set\n',
+        );
+        return callback(null, file);
+      }
+      // TODO add test
+      return callback(null, file);
+    });
+  },
+};

task/PrepareRouterData.js

@@ -22,6 +22,31 @@ const extraUpdaters =
     ? JSON.parse(process.env.EXTRA_UPDATERS)
     : {};
 
+function createAndProcessBuildConfig(router) {
+  process.stdout.write('copying build-config.json...\n');
+  const configName = `${router.id}/build-config.json`;
+  const buildConfig = JSON.parse(fs.readFileSync(configName, 'utf8'));
+  const transitFeeds = buildConfig.transitFeeds || [];
+  if (router.netex) {
+    router.netex.forEach(src => {
+      const feed = {
+	type: 'netex',
+	feedId: src.id,
+	source: 'file:///var/opentripplanner/' + src.id + '-netex.zip',
+	groupFilePattern: src.groupFilePattern,
+	sharedFilePattern: src.sharedFilePattern,
+      };
+      transitFeeds.push(feed);
+    });
+    buildConfig.transitFeeds = transitFeeds;
+  }
+  const file = new Vinyl({
+    path: 'build-config.json',
+    contents: Buffer.from(JSON.stringify(buildConfig, null, 2)),
+  });
+  return file;
+}
+
 // Prepares router-config.json data for opentripplanner and applies edits/additions made in EXTRA_UPDATERS env var
 function createAndProcessRouterConfig(router) {
   process.stdout.write('copying router-config.json...\n');
@@ -68,8 +93,8 @@ function prepareRouterData(router) {
     'Collecting data and configuration files for graph build\n',
   );
 
-  stream.push(createFile(router, 'build-config.json', router.id));
   stream.push(createFile(router, 'otp-config.json', router.id));
+  stream.push(createAndProcessBuildConfig(router));
   stream.push(createAndProcessRouterConfig(router));
   router.osm.forEach(osmId => {
     const name = osmId + '.pbf';
@@ -83,6 +108,12 @@ function prepareRouterData(router) {
     const name = src.id + '-gtfs.zip';
     stream.push(createFile(router, name, `${dataDir}/ready/gtfs`));
   });
+  if (router.netex) {
+    router.netex.forEach(src => {
+      const name = src.id + '-netex.zip';
+      stream.push(createFile(router, name, `${dataDir}/ready/netex`));
+    });
+  }
   stream.end();
 
   return stream;
@@ -99,8 +130,8 @@ function prepareRouterDataForStreetOnlyGraphBuild(router) {
     'Collecting data and configuration files for street only graph build\n',
   );
 
-  stream.push(createFile(router, 'build-config.json', router.id));
   stream.push(createFile(router, 'otp-config.json', router.id));
+  stream.push(createAndProcessBuildConfig(router));
   stream.push(createAndProcessRouterConfig(router));
   router.osm.forEach(osmId => {
     const name = osmId + '.pbf';
@@ -145,6 +176,12 @@ function prepareRouterDataForPrebuiltStreetGraphBuild(router) {
     const name = src.id + '-gtfs.zip';
     stream.push(createFile(router, name, `${dataDir}/ready/gtfs`));
   });
+  if (router.netex) {
+    router.netex.forEach(src => {
+      const name = src.id + '-netex.zip';
+      stream.push(createFile(router, name, `${dataDir}/ready/netex`));
+    });
+  }
 
   const osmDirectories = getDirectories(
     `${storageDir}/osm-builds/${process.env.DOCKER_TAG}`,

task/Update.js

@@ -183,6 +183,7 @@ async function buildGraph(name) {
   await handleOsmAndDemUpdate();
 
   await start('gtfs:update');
+  await start('netex:update');
 
   process.stdout.write('Build routing graph\n');
   await start('router:buildGraph');
@@ -223,6 +224,7 @@ async function buildWithPrebuiltStreetGraph(name) {
   await handleSeeding();
 
   await start('gtfs:update');
+  await start('netex:update');
 
   process.stdout.write('Build routing graph from prebuilt street only graph\n');
   await start('router:buildWithPrebuiltStreetGraph');