Import script trams

.github/workflows/java-tests.yml

@@ -31,6 +31,20 @@ jobs:
           distribution: temurin
           cache: maven
 
+      - name: Set up Python 3.14
+        uses: actions/setup-python@v5
+        with:
+          python-version: "3.14"
+
+      - name: Create stop-registry importer virtualenv
+        # The Spring Batch job runs the external stop-registry importer Python script as
+        # its final step. Create the virtualenv and install its dependencies so the step
+        # can run during the Java integration tests (see the `ci` profile config).
+        run: |
+          python -m venv stop-registry-importer/.venv-stop-registry
+          stop-registry-importer/.venv-stop-registry/bin/python -m pip install --upgrade pip
+          stop-registry-importer/.venv-stop-registry/bin/pip install -r stop-registry-importer/requirements.txt
+
       - name: Verify whether test postgresql db is up
         uses: HSLdevcom/jore4-tools/github-actions/healthcheck@healthcheck-v1
         with:

.gitignore

@@ -45,4 +45,7 @@ digiroad_stops*.csv
 
 #possible jore3 backup file
 jore3dump/*
-!jore3dump/put-bak-file-here.txt
+!jore3dump/put-bak-file-here.txt
+
+#stop place import script environment file
+.env

Dockerfile

@@ -22,6 +22,18 @@ ARG APPINSIGHTS_VERSION=3.7.7
 # expose server port
 EXPOSE 8080
 
+# install Python 3 + pip for the stop-registry importer script that is run as the
+# final step of the Spring Batch import job
+RUN apt-get update \
+ && apt-get install -y --no-install-recommends python3 python3-pip \
+ && rm -rf /var/lib/apt/lists/*
+
+# install stop-registry importer Python dependencies and copy the script
+COPY ./stop-registry-importer/requirements.txt /opt/stop-registry-importer/requirements.txt
+RUN pip3 install --no-cache-dir --break-system-packages \
+        -r /opt/stop-registry-importer/requirements.txt
+COPY --chmod=444 ./stop-registry-importer/importer.py /opt/stop-registry-importer/importer.py
+
 # download script for reading Docker secrets
 ADD --chmod=555 https://raw.githubusercontent.com/HSLdevcom/jore4-tools/main/docker/read-secrets.sh /tmp/read-secrets.sh
 

README.md

@@ -529,3 +529,40 @@ If a test case fails because the `com.microsoft.sqlserver.jdbc.SQLServerExceptio
 the error message says that it cannot find a database object, the problem is that the script which
 creates the source MSSQL database (_docker/mssql_init/populate.sql_) was changed. You can solve this problem by running the command:
 `./development.sh recreate` at command prompt.
+
+---
+
+## Jore 3 stop import script
+
+### Requirements
+
+You need to have Docker installed on your system to run the script.
+
+Also the python virtual environment must be set up to run the tests (´./developer.sh setup:python`).
+
+### How to use
+
+The script is automatically run as part of the import process.
+
+To manually run the stop registry importer script you need to have a Jore 3 database, a populated Jore 4 routes database and the `jore4-hasura` and `jore4-tiamat` microservices running.
+
+By default the script runs from the local Jore 3 `mssqltestdb` database and uses the base local `jore4-hasura` service as the target.
+You can change the source database and target Hasura instance by creating a `.env` file in the same directory as the script.
+
+Set the values for variables you want to set:
+
+```
+HASURA_API_URL=
+HASURA_ADMIN_SECRET=
+SOURCE_DB_USERNAME=
+SOURCE_DB_PASSWORD=
+SOURCE_DB_HOSTNAME=
+SOURCE_DB_PORT=
+SOURCE_DB_DATABASE=
+```
+
+You should have run the base Jore 3 importer first which ensures the Jore 4 database has the required scheduled stop points. Then run the stop registry import script which will match scheduled stop points in the Jore 4 routes database with stops in the Jore 3 database and generate GraphQL mutations to Hasura/Tiamat according to the data. The script will also link the generated stop registry stops with the scheduled stop points by their NeTEx ID using Hasura for the mutation.
+
+Running the script will produce multiple errors as there are many stops in the Jore 3 database with overlapping validity. These can be ingored as only the most recent one is the one which ends up being imported.
+
+To run the script simply run `run-stop-registry-importer.sh` in the stop-registry-importer directory, this will create and run a Docker container with the environment set up.

development.sh

@@ -12,6 +12,16 @@ cd "$(dirname "$0")" # Setting the working directory as the script directory
 # variable.
 DOCKER_COMPOSE_BUNDLE_REF=${BUNDLE_REF:-main}
 
+STOP_REGISTRY_IMPORTER_DIR="./stop-registry-importer"
+STOP_REGISTRY_VENV_DIR="${STOP_REGISTRY_IMPORTER_DIR}/.venv-stop-registry"
+STOP_REGISTRY_REQUIREMENTS_FILE="${STOP_REGISTRY_IMPORTER_DIR}/requirements.txt"
+STOP_REGISTRY_REQUIREMENTS_IN_FILE="${STOP_REGISTRY_IMPORTER_DIR}/requirements.in"
+
+# Python/pip executables inside the stop-registry virtualenv. These are
+# populated by `ensure_python_venv` so that every Python-related command uses
+# the correct interpreter and isolated environment.
+STOP_REGISTRY_VENV_PYTHON="${STOP_REGISTRY_VENV_DIR}/bin/python"
+
 # Define a Docker Compose project name to distinguish
 # the docker environment of this project from others
 export COMPOSE_PROJECT_NAME=jore3-importer
@@ -55,6 +65,14 @@ print_usage() {
   generate:jooq
     Generate JOOQ classes.
 
+  python:setup
+    Creates/updates Python virtualenv for stop-registry importer and installs dependencies.
+
+  python:update-reqs
+    Recompiles stop-registry-importer/requirements.txt from requirements.in using
+    pip-tools (pip-compile). Run this after changing requirements.in. You must run python:setup
+    to install the updated dependencies after review.
+
   stop
     Stop the dependencies and the dockerized application.
 
@@ -202,7 +220,8 @@ seed_tram_infra_links() {
 }
 
 start_all() {
-  $DOCKER_COMPOSE_CMD up --build -d importer-jooq-database importer-test-database jore4-mssqltestdb jore4-hasura jore4-testdb jore4-jore3importer jore4-mapmatchingdb jore4-mapmatching
+  start_deps
+  $DOCKER_COMPOSE_CMD up --build -d  jore4-jore3importer
 }
 
 start_deps() {
@@ -212,7 +231,7 @@ start_deps() {
   # jore4-mssqltestdb - The Jore 3 MSSQL database which contains the source data which is read by the importer
   # jore4-hasura - Hasura. We have to start Hasura because it ensures that db migrations are run to the Jore 4 database.
   # jore4-testdb - Jore 4 database. This is the destination database of the import process.
-  $DOCKER_COMPOSE_CMD up --build -d importer-jooq-database importer-test-database jore4-mssqltestdb jore4-hasura jore4-testdb jore4-mapmatchingdb jore4-mapmatching
+  $DOCKER_COMPOSE_CMD up --build -d importer-jooq-database importer-test-database jore4-mssqltestdb jore4-hasura jore4-testdb jore4-mapmatchingdb jore4-mapmatching jore4-tiamat jore4-auth jore4-idp
 }
 
 stop() {
@@ -245,6 +264,58 @@ generate_jooq() {
   mvn clean generate-sources -Pci
 }
 
+upload_zones() {
+  echo "Uploading municipality and fare zones to Tiamat"
+
+  curl --silent --output /dev/null --show-error --fail -X POST -H"Content-Type: application/xml" -d @netex/hsl-zones-netex.xml localhost:3010/services/stop_places/netex
+}
+
+setup_python() {
+  ensure_python_venv
+
+  echo "Installing dependencies from ${STOP_REGISTRY_REQUIREMENTS_FILE}..."
+  "$STOP_REGISTRY_VENV_PYTHON" -m pip install -r "$STOP_REGISTRY_REQUIREMENTS_FILE"
+}
+
+# Ensures the stop-registry virtualenv exists and that pip is up to date.
+#
+# This is the common entry point for all Python commands: it guarantees that
+# subsequent calls to "$STOP_REGISTRY_VENV_PYTHON" use the correct interpreter
+# and isolated environment regardless of the host Python setup.
+ensure_python_venv() {
+  if [ ! -d "$STOP_REGISTRY_VENV_DIR" ]; then
+    echo "Creating Python virtualenv in ${STOP_REGISTRY_VENV_DIR}..."
+    python3 -m venv "$STOP_REGISTRY_VENV_DIR"
+  fi
+
+  if [ ! -x "$STOP_REGISTRY_VENV_PYTHON" ]; then
+    echo "ERROR: Python executable not found in virtualenv: ${STOP_REGISTRY_VENV_PYTHON}" >&2
+    echo "Try removing ${STOP_REGISTRY_VENV_DIR} and running 'python:setup' again." >&2
+    exit 1
+  fi
+
+  "$STOP_REGISTRY_VENV_PYTHON" -m pip install --upgrade pip
+}
+
+# Recompiles requirements.txt from requirements.in using pip-tools and installs
+# the resolved dependency set into the virtualenv.
+update_python_requirements() {
+  ensure_python_venv
+
+  if [ ! -f "$STOP_REGISTRY_REQUIREMENTS_IN_FILE" ]; then
+    echo "ERROR: requirements input file not found: ${STOP_REGISTRY_REQUIREMENTS_IN_FILE}" >&2
+    exit 1
+  fi
+
+  "$STOP_REGISTRY_VENV_PYTHON" -m pip install --upgrade pip-tools
+
+  echo "Compiling ${STOP_REGISTRY_REQUIREMENTS_FILE} from ${STOP_REGISTRY_REQUIREMENTS_IN_FILE}..."
+  "$STOP_REGISTRY_VENV_PYTHON" -m piptools compile \
+    --strip-extras \
+    --output-file "$STOP_REGISTRY_REQUIREMENTS_FILE" \
+    "$STOP_REGISTRY_REQUIREMENTS_IN_FILE"
+}
+
 ### Control flow
 
 COMMAND=${1:-}
@@ -257,19 +328,31 @@ fi
 case $COMMAND in
   start)
     download_docker_compose_bundle
+    setup_python
     start_all
+    upload_zones
     ;;
 
   start:deps)
     download_docker_compose_bundle
+    setup_python
     start_deps
+    upload_zones
     ;;
 
   generate:jooq)
     wait_for_test_databases_to_be_ready
     generate_jooq
     ;;
 
+  python:setup)
+    setup_python
+    ;;
+
+  python:update-reqs)
+    update_python_requirements
+    ;;
+
   stop)
     stop
     ;;
@@ -280,7 +363,9 @@ case $COMMAND in
 
   recreate)
     remove
+    setup_python
     start_deps
+    upload_zones
     ;;
 
   list)

docker/docker-compose.custom.yml

@@ -35,5 +35,22 @@ services:
     # volume to easily use jore3 database dump
     volumes:
       - ../jore3dump:/mnt/jore3dump
+
+  jore4-tiamat:
+    # Pin tiamat to a compatible version.
+    #image: "hsldevcom/jore4-tiamat:main--20250317-1136ff34a5b6030b0e9c85b0373f5ddc02946267"
+    depends_on:
+      jore4-testdb:
+        condition: service_healthy
+
+  jore4-hasura:
+    # pin compatible version of jore4 data model
+    #image: "hsldevcom/jore4-hasura:hsl-main--20250317-60eaf1217447890591ca3bbe262c96f2cc68ba57"
+    depends_on:
+      jore4-testdb:
+        condition: service_healthy
+      jore4-tiamat:
+        condition: service_healthy
+
 networks:
-    jore4:
+  jore4:

profiles/ci/config.properties

@@ -73,3 +73,11 @@ digiroad.stop.csv.file.url=
 
 # The map matching API URL
 map.matching.api.baseUrl=http://localhost:3005
+
+# Stop-registry importer execution (paths relative to the repository root, where
+# Maven runs in CI). The Python virtualenv is created by the CI workflow before the
+# tests are run; see .github/workflows/java-tests.yml.
+stop.registry.importer.python.command=.venv-stop-registry/bin/python
+stop.registry.importer.script.path=importer.py
+stop.registry.importer.working.directory=stop-registry-importer
+stop.registry.importer.timeout.hours=2

profiles/dev/config.properties

@@ -58,6 +58,12 @@ digiroad.stop.csv.file.url=https://stjore4dev001.blob.core.windows.net/jore4-dig
 # The base URL of the Map Matching API
 map.matching.api.baseUrl=http://localhost:3005
 
+# Stop-registry importer execution (local workspace paths)
+stop.registry.importer.python.command=.venv-stop-registry/bin/python
+stop.registry.importer.script.path=importer.py
+stop.registry.importer.working.directory=stop-registry-importer
+stop.registry.importer.timeout.hours=2
+
 #
 # TEST SETTINGS
 #

profiles/prod/config.properties

@@ -41,3 +41,9 @@ digiroad.stop.csv.file.url=https://stjore4dev001.blob.core.windows.net/jore4-dig
 
 # The base URL of the map matching API
 map.matching.api.baseUrl=
+
+# Stop-registry importer execution (container paths)
+stop.registry.importer.python.command=python3
+stop.registry.importer.script.path=/opt/stop-registry-importer/importer.py
+stop.registry.importer.working.directory=/opt/stop-registry-importer
+stop.registry.importer.timeout.hours=2

src/main/java/fi/hsl/jore/importer/config/jobs/JobConfig.java

@@ -67,6 +67,7 @@
 import fi.hsl.jore.importer.feature.batch.stop_place.StopPlaceImportProcessor;
 import fi.hsl.jore.importer.feature.batch.stop_place.StopPlaceImportReader;
 import fi.hsl.jore.importer.feature.batch.stop_place.support.IStopPlaceImportRepository;
+import fi.hsl.jore.importer.feature.batch.stop_registry.RunStopRegistryImporterTasklet;
 import fi.hsl.jore.importer.feature.infrastructure.link.dto.Jore3Link;
 import fi.hsl.jore.importer.feature.infrastructure.link_shape.dto.Jore3LinkShape;
 import fi.hsl.jore.importer.feature.infrastructure.node.dto.Jore3Node;
@@ -110,16 +111,19 @@
 import org.springframework.batch.core.step.Step;
 import org.springframework.batch.core.step.builder.StepBuilder;
 import org.springframework.batch.core.step.skip.AlwaysSkipItemSkipPolicy;
+import org.springframework.beans.factory.annotation.Value;
 import org.springframework.boot.autoconfigure.EnableAutoConfiguration;
 import org.springframework.boot.batch.autoconfigure.BatchTransactionManager;
 import org.springframework.context.annotation.Bean;
 import org.springframework.context.annotation.ComponentScan;
 import org.springframework.context.annotation.Configuration;
+import org.springframework.context.annotation.PropertySource;
 import org.springframework.transaction.PlatformTransactionManager;
 
 @Configuration
 @EnableAutoConfiguration
 @ComponentScan(basePackages = "fi.hsl.jore.importer.feature")
+@PropertySource("classpath:application.properties")
 public class JobConfig {
     public static final String JOB_NAME = "importJoreJob";
 
@@ -148,7 +152,9 @@ public Job importJob(
             final Flow importScheduledStopPointsFlow,
             final Flow importStopPlacesFlow,
             // Export data from the importer staging DB to Jore 4 DB.
-            final Flow jore4ExportFlow) {
+            final Flow jore4ExportFlow,
+            // Final step: run the external Python stop-registry importer.
+            final Flow stopRegistryImportFlow) {
         return new JobBuilder(JOB_NAME, jobRepository)
                 .start(importNodesFlow)
                 .next(importLinksFlow)
@@ -162,10 +168,32 @@ public Job importJob(
                 .next(importScheduledStopPointsFlow)
                 .next(importStopPlacesFlow)
                 .next(jore4ExportFlow)
+                .next(stopRegistryImportFlow)
                 .end()
                 .build();
     }
 
+    @Bean
+    public Step runStopRegistryImporterStep(
+            @Value("${stop.registry.importer.python.command:python3}") final String pythonCommand,
+            @Value("${stop.registry.importer.script.path:importer.py}") final String scriptPath,
+            @Value("${stop.registry.importer.working.directory:stop-registry-importer}") final String workingDir,
+            @Value("${stop.registry.importer.timeout.hours:2}") final long timeoutHours) {
+        return new StepBuilder("runStopRegistryImporterStep", jobRepository)
+                .allowStartIfComplete(true)
+                .tasklet(
+                        new RunStopRegistryImporterTasklet(pythonCommand, scriptPath, workingDir, timeoutHours),
+                        transactionManager)
+                .build();
+    }
+
+    @Bean
+    public Flow stopRegistryImportFlow(final Step runStopRegistryImporterStep) {
+        return new FlowBuilder<SimpleFlow>("stopRegistryImportFlow")
+                .start(runStopRegistryImporterStep)
+                .build();
+    }
+
     @Bean
     public Flow importNodesFlow(final Step prepareNodesStep, final Step importNodesStep, final Step commitNodesStep) {