Extract tram links from QGIS, export mbtiles

README.md

@@ -40,7 +40,7 @@ of the Docker container by running:
 ```
 
 The shapefiles are, by default, imported into a database schema named `digiroad`.
-The schema name can be changed in `set_env_vars.sh` script.
+The schema name can be changed in `set_env.sh` script.
 
 Within the script execution further processing for Digiroad data is done as well.
 
@@ -260,6 +260,32 @@ exported with (assuming Digiroad shapefiles have already been imported):
 ./export_mbtiles_dr_pysakki.sh
 ```
 
+## Import MML tram network
+
+1. Download MML's tram network from their website:
+- [Lataa paikkatietoaineistoja](https://asiointi.maanmittauslaitos.fi/karttapaikka/tiedostopalvelu?lang=fi)
+  - Maastotietokanta
+    - Valitse tiedostomuoto: ```GeoPackage**```
+    - Piirrä oma alue
+      - valitse teema: ```Raideliikenne```
+    - Lisää ostoskoriin
+    - Download via the download link in the email you receive once the download is available.
+
+2. Open the GeoPackage file in QGIS (drag to Layers)
+2. Open Python console
+3. Open the ```generate-tram-infralinks-from-qgis.py``` file into the Python File viewer
+4. Select the MML GeoPackage layer
+5. Run the python script
+
+    The sql containing the infra links, insertable to the database, can be found in ```/tmp/tram_infraLinks.sql```. Leave it there or copy to ```sql/```.
+
+6. run ```export_mbtiles_tram_links.sh``` with the date the tram link material was obtained on:
+```sh
+  ./export_mbtiles_tram_links.sh 2026-01-28
+```
+
+8. upload the ```sql/tram_infralinks.sql``` and ```workdir/mbtiles/tram_links_<MML_TRAM_IMPORT_DATE>_<today>.mbtiles``` to blob storage ```stjore4dev001 / jore4-ui```
+
 ## License
 
 The project license is in [`LICENSE`](./LICENSE).

export_mbtiles_tram_links.sh

@@ -0,0 +1,85 @@
+#!/usr/bin/env bash
+
+# Stop on first error. Have meaningful error messages.
+set -euo pipefail
+
+# Source common environment variables and functions.
+source "$(dirname "$0")/set_env.sh"
+
+DB_TABLE_NAME="tram_links"
+
+usage() {
+  echo "Usage: $(basename "$0") <MML_TRAM_IMPORT_DATE>"
+  echo "  MML_TRAM_IMPORT_DATE must be in format YYYY-MM-DD"
+}
+
+if [[ "${1:-}" == "" ]]; then
+  usage
+  exit 1
+fi
+
+MML_TRAM_IMPORT_DATE="$1"
+
+if [[ ! "$MML_TRAM_IMPORT_DATE" =~ ^20[2-9][0-9]-(0[1-9]|1[0-2])-(0[1-9]|[12][0-9]|3[01])$ ]]; then
+  echo "Invalid date: $MML_TRAM_IMPORT_DATE"
+  usage
+  exit 1
+fi
+
+MBTILES_MAX_ZOOM_LEVEL=17
+MBTILES_LAYER_NAME=$DB_TABLE_NAME
+MBTILES_DESCRIPTION="Tram track links"
+
+MBTILES_OUTPUT_DIR="${WORK_DIR}/mbtiles"
+GEOJSON_OUTPUT_DIR="${MBTILES_OUTPUT_DIR}/geojson_input"
+SQL_INPUT_DIR="${CWD}/sql"
+
+print_and_run_cmd mkdir -p "$GEOJSON_OUTPUT_DIR"
+print_and_run_cmd mkdir -p "$SQL_INPUT_DIR"
+print_and_run_cmd mkdir -p "${WORK_DIR}/shp"
+
+OUTPUT_FILE_BASENAME="${DB_TABLE_NAME}_${MML_TRAM_IMPORT_DATE}_$(date "+%Y-%m-%d")"
+
+GEOJSON_OUTPUT_FILE="${OUTPUT_FILE_BASENAME}.geojson"
+MBTILES_OUTPUT_FILE="${OUTPUT_FILE_BASENAME}.mbtiles"
+
+if [ ! -f "${SQL_INPUT_DIR}/tram_infraLinks.sql" ]; then
+  if [ ! -f "/tmp/tram_infraLinks.sql" ]; then
+    echo "Expected SQL file for processing tram links does not exist: /tmp/tram_infraLinks.sql"
+    exit 1
+  fi
+  print_and_run_cmd mv "/tmp/tram_infraLinks.sql" "${SQL_INPUT_DIR}/tram_infraLinks.sql"
+else
+  echo "Using existing file: ${SQL_INPUT_DIR}/tram_infraLinks.sql"
+fi
+
+print_and_run_cmd docker_kill
+print_and_run_cmd docker_run "${WORK_DIR}/shp"
+
+# install pgcrypto extension for generating UUIDs
+time docker_exec postgres "exec $PSQL -v ON_ERROR_STOP=1 -c 'CREATE EXTENSION IF NOT EXISTS pgcrypto;'"
+
+# import tram infra links
+time print_and_run_cmd docker_exec postgres "exec $PSQL -v ON_ERROR_STOP=1 -f /tmp/sql/tram_infraLinks.sql"
+
+# Export filtered links directly from PostGIS to GeoJSON.
+rm -f "${GEOJSON_OUTPUT_DIR}/$GEOJSON_OUTPUT_FILE"
+time docker_exec "$CURRUSER" \
+  "exec ogr2ogr -f GeoJSON -lco COORDINATE_PRECISION=7 /tmp/mbtiles/geojson_input/$GEOJSON_OUTPUT_FILE \
+  \"PG:host=$DB_HOST port=$DB_PORT dbname=$DB_NAME user=$DB_USERNAME \" \
+  -sql \"SELECT \
+      infrastructure_link_id::text AS id, \
+      external_link_id AS link_id,\
+      ST_Force2D(shape::geometry) AS geom \
+    FROM infrastructure_network.infrastructure_link \
+    WHERE external_link_source = 'temp_hsl_tram'\" \
+  -nln $MBTILES_LAYER_NAME"
+
+# Convert from GeoJSON to MBTiles.
+rm -f "${MBTILES_OUTPUT_DIR}/${MBTILES_OUTPUT_FILE}"
+rm -f "${MBTILES_OUTPUT_DIR}/${MBTILES_OUTPUT_FILE}-journal"
+time docker_exec "$CURRUSER" \
+  "tippecanoe /tmp/mbtiles/geojson_input/$GEOJSON_OUTPUT_FILE -o /tmp/$MBTILES_OUTPUT_FILE -z$MBTILES_MAX_ZOOM_LEVEL -X -l $MBTILES_LAYER_NAME -n \"$MBTILES_DESCRIPTION\" -f && exec mv /tmp/$MBTILES_OUTPUT_FILE /tmp/mbtiles/$MBTILES_OUTPUT_FILE"
+
+# Stop Docker container.
+ print_and_run_cmd docker_stop

generate-tram-infralinks-from-qgis.py

@@ -0,0 +1,211 @@
+# python
+
+# This script exports selected line features from the active QGIS layer
+# to a SQL file that can be used to insert them into the 'infrastructure_link'
+# table in a PostGIS database. It calculates the ellipsoidal length of each
+# feature using the GRS80 ellipsoid and transforms geometries to EPSG:4326
+# for storage as geography(LineStringZ,4326).
+#
+# Configuration options at the top of the script allow customization of
+# the output file path, table name, direction, external link source,
+# ID field, and batch size for inserts.
+#
+# Open the script in the QGIS Python console and run it with an active layer
+# containing selected (with one of the selection tools) line features to
+# generate the SQL file. If no features are selected, the entire layer will
+# be exported.
+
+
+from qgis.core import (
+    QgsProject,
+    QgsCoordinateReferenceSystem,
+    QgsCoordinateTransform,
+    QgsDistanceArea,
+)
+from qgis.utils import iface
+
+import uuid
+
+# Config
+output_path = '/tmp/tram_infraLinks.sql'
+table_name = 'infrastructure_network.infrastructure_link'
+direction = 'bidirectional'
+external_link_source = 'temp_hsl_tram'
+id_field = 'mtk_id'
+batch_size = 50
+
+layer = iface.activeLayer()
+if layer is None:
+    raise RuntimeError('No active layer.')
+
+proj = QgsProject.instance()
+src_crs = layer.crs()
+dst_crs = QgsCoordinateReferenceSystem('EPSG:4326')
+transform = QgsCoordinateTransform(src_crs, dst_crs, proj)
+
+# Ellipsoidal length calculator on GRS80 (EPSG:7019)
+dist = QgsDistanceArea()
+dist.setSourceCrs(src_crs, proj.transformContext())
+ok = False
+try:
+    ok = dist.setEllipsoid('EPSG:7019')
+except Exception:
+    ok = False
+if not ok:
+    dist.setEllipsoid('GRS80')  # fallback
+
+def sql_quote(val):
+    if val is None:
+        return 'NULL'
+    return "'" + str(val).replace("'", "''") + "'"
+
+# Determine feature source: selected or full layer
+selected_count = layer.selectedFeatureCount()
+if selected_count > 0:
+  features = list(layer.selectedFeatures())
+  export_count = selected_count
+else:
+  print("No features selected; exporting entire layer.")
+  features = list(layer.getFeatures())
+  export_count = layer.featureCount()
+
+lines = []
+lines.append('-- SQL export generated by QGIS Python console')
+lines.append(f'-- Source layer: {layer.name()}')
+lines.append(f'-- Exported features (source): {export_count}')
+lines.append('BEGIN;')
+
+# Add infralink type
+lines.append("""
+-- Create schema and tables if they don't exist. Needed in digiroad-import repo when generating mbtiles.
+CREATE SCHEMA IF NOT EXISTS infrastructure_network;
+
+CREATE TABLE IF NOT EXISTS infrastructure_network.external_source (
+  value text NOT NULL
+);
+
+CREATE TABLE infrastructure_network.vehicle_submode_on_infrastructure_link (
+    infrastructure_link_id uuid NOT NULL,
+    vehicle_submode text NOT NULL
+);
+
+CREATE TABLE IF NOT EXISTS infrastructure_network.infrastructure_link (
+    infrastructure_link_id uuid DEFAULT public.gen_random_uuid() NOT NULL,
+    direction text NOT NULL,
+    shape public.geography(LineStringZ,4326) NOT NULL,
+    estimated_length_in_metres double precision,
+    external_link_id text NOT NULL,
+    external_link_source text NOT NULL
+);
+
+-- Add the temporary link external source type is it doesn't exist
+INSERT INTO infrastructure_network.external_source VALUES ('temp_hsl_tram') ON CONFLICT DO NOTHING;
+
+""")
+
+
+batch_values = []
+
+# Diagnostics
+written = 0            # rows added to VALUES
+seen = 0               # features iterated
+failed_transform = 0
+skipped_empty = 0
+skipped_non_line = 0
+non_tram = 0
+
+def flush_batch():
+    if not batch_values:
+        return
+    values_sql = ',\n  '.join(batch_values)
+    insert_sql = (
+        f"INSERT INTO {table_name} "
+        "(infrastructure_link_id, direction, shape, estimated_length_in_metres, external_link_id, external_link_source)\n"
+        f"VALUES\n  {values_sql};"
+    )
+    lines.append(insert_sql)
+    batch_values.clear()
+
+field_names = layer.fields().names()
+
+for f in features:
+    if f['kohdeluokka'] not in (14141, 14142):
+        non_tram += 1
+        continue
+
+    seen += 1
+    geom = f.geometry()
+    if geom is None or geom.isEmpty():
+        skipped_empty += 1
+        continue
+
+    # 1) Ellipsoidal length in metres
+    try:
+        length_m = dist.measureLength(geom)
+    except Exception:
+        length_m = None
+
+    # 2) Try to merge multipart lines to single, if any
+    try:
+        merged = geom.lineMerge()
+        if merged and not merged.isEmpty():
+            geom = merged
+    except Exception:
+        pass
+
+    # 3) Transform geometry to EPSG:4326 for WKT Z output
+    try:
+        geom.transform(transform)
+    except Exception:
+        failed_transform += 1
+        continue
+
+    wkt = geom.asWkt()  # LINESTRING Z (...) expected
+    up = wkt.upper()
+    if not up.startswith('LINESTRING'):
+        # Skip non-lines if target column is geography(LineStringZ,4326)
+        skipped_non_line += 1
+        continue
+
+    external_id = f[id_field] if id_field in field_names else None
+    length_sql = 'NULL' if length_m is None else f"{float(length_m)}"
+    ext_id_sql = sql_quote(external_id)
+
+    infrastructure_link_id = str(uuid.uuid4())
+
+    # geography from EWKT in 4326 with Z
+    shape_sql = (
+        f"CAST(ST_GeogFromText('SRID=4326;{wkt}') AS geography(LineStringZ,4326))"
+    )
+
+    value_sql = (
+        f"('{infrastructure_link_id}', '{direction}', {shape_sql}, {length_sql}, {ext_id_sql}, '{external_link_source}')"
+    )
+    batch_values.append(value_sql)
+    written += 1
+
+    if len(batch_values) >= batch_size:
+        flush_batch()
+
+flush_batch()
+
+
+lines.append("""
+-- Select all infralink ids from 'infrastructure_link' table and insert them to the 'vehicle_submode_on_infrastructure_link' table
+-- along with static 'generic_tram' vehicle submode info
+
+INSERT INTO infrastructure_network.vehicle_submode_on_infrastructure_link
+  SELECT il.infrastructure_link_id,
+         'generic_tram' AS vehicle_submode
+  FROM infrastructure_network.infrastructure_link il
+  WHERE il.external_link_source = 'temp_hsl_tram'
+ON CONFLICT DO NOTHING;
+
+
+COMMIT;
+""")
+
+with open(output_path, 'w', encoding='utf-8') as fh:
+    fh.write('\n'.join(lines))
+
+print(f"Wrote SQL to {output_path} with {written} value rows from {seen} features; skipped non_tram: {non_tram}, skipped empty: {skipped_empty}, failed_transform: {failed_transform}, non-line: {skipped_non_line}.")

import_digiroad_shapefiles.sh

@@ -44,30 +44,10 @@ rm -fr "${DOWNLOAD_TARGET_DIR:?}/${AREA}"
 unzip -u "$DOWNLOAD_TARGET_FILE" Dokumentit/* -d "$DOWNLOAD_TARGET_DIR"
 
 # Remove possibly running/existing Docker container.
-docker kill "$DOCKER_CONTAINER_NAME" &> /dev/null || true
-docker rm -v "$DOCKER_CONTAINER_NAME" &> /dev/null || true
-
-# Create directories that will be mounted to Docker container.
-mkdir -p "$WORK_DIR"/csv
-mkdir -p "$WORK_DIR"/mbtiles
-mkdir -p "$WORK_DIR"/pgdump
-
-# Create and start new Docker container. Mount all directories as volumes that
-# are needed by various processing scripts.
-docker run \
-  --name "$DOCKER_CONTAINER_NAME" \
-  -p 127.0.0.1:${DOCKER_CONTAINER_PORT}:5432 \
-  -e POSTGRES_HOST_AUTH_METHOD=trust \
-  -v "$CWD"/fixup/digiroad:/tmp/gpkg \
-  -v "$CWD"/sql:/tmp/sql \
-  -v "$SHP_FILE_DIR":/tmp/shp \
-  -v "$WORK_DIR"/csv:/tmp/csv \
-  -v "$WORK_DIR"/mbtiles:/tmp/mbtiles \
-  -v "$WORK_DIR"/pgdump:/tmp/pgdump \
-  -d "$DOCKER_IMAGE"
-
-# Wait for PostgreSQL server to be ready.
-docker_exec postgres "exec $PG_WAIT"
+docker_kill
+
+# Create and start new Docker container.
+docker_run "$SHP_FILE_DIR"
 
 # Create digiroad import schema into database.
 docker_exec postgres "exec $PSQL -nt -c \"CREATE SCHEMA ${DB_SCHEMA_NAME_DIGIROAD};\""