Changed the name to gtfs-datasets-comparer

Author: jcpitre

functions-python/batch_process_dataset/src/pipeline_tasks.py

@@ -11,7 +11,7 @@
 from shared.helpers.utils import (
     create_http_task,
     create_http_pmtiles_builder_task,
-    create_http_gtfs_change_tracker_task,
+    create_http_gtfs_datasets_comparer_task,
 )
 
 
@@ -172,7 +172,7 @@ def create_pipeline_tasks(dataset: Gtfsdataset, db_session: Session) -> None:
                 dataset_stable_id,
             )
         else:
-            create_http_gtfs_change_tracker_task(
+            create_http_gtfs_datasets_comparer_task(
                 feed_stable_id=stable_id,
                 base_dataset_stable_id=previous_dataset.stable_id,
                 new_dataset_stable_id=dataset_stable_id,

functions-python/batch_process_dataset/tests/test_batch_process_dataset_main.py

@@ -449,9 +449,9 @@ def mock_remove_side_effect(path):
     @patch.dict(
         os.environ, {"FEEDS_CREDENTIALS": '{"test_stable_id": "test_credentials"}'}
     )
-    @patch("pipeline_tasks.create_http_gtfs_change_tracker_task")
+    @patch("pipeline_tasks.create_http_gtfs_datasets_comparer_task")
     @with_db_session(db_url=default_db_url)
-    def test_process(self, mock_gtfs_change_tracker_task, db_session):
+    def test_process(self, mock_gtfs_datasets_comparer_task, db_session):
         feeds = db_session.query(Gtfsfeed).all()
         feed_id = feeds[0].id
 

functions-python/gtfs_change_tracker/function_config.json

@@ -1,5 +1,5 @@
 {
-  "name": "gtfs-change-tracker",
+  "name": "gtfs-datasets-comparer",
   "description": "Tracks changes between two GTFS datasets and stores a structured changelog in GCS and the database",
   "entry_point": "gtfs_change_tracker",
   "timeout": 540,

functions-python/helpers/utils.py

@@ -535,13 +535,13 @@ def create_http_pmtiles_builder_task(
     )
 
 
-def create_http_gtfs_change_tracker_task(
+def create_http_gtfs_datasets_comparer_task(
     feed_stable_id: str,
     base_dataset_stable_id: str,
     new_dataset_stable_id: str,
 ) -> None:
     """
-    Create a Cloud Task to run the gtfs-change-tracker function for a pair of datasets.
+    Create a Cloud Task to run the gtfs-datasets-comparer function for a pair of datasets.
     """
     from google.cloud import tasks_v2
     import json
@@ -563,7 +563,7 @@ def create_http_gtfs_change_tracker_task(
     create_http_task(
         client,
         body,
-        f"https://{gcp_region}-{project_id}.cloudfunctions.net/gtfs-change-tracker-{gcp_env}",
+        f"https://{gcp_region}-{project_id}.cloudfunctions.net/gtfs-datasets-comparer-{gcp_env}",
         project_id,
         gcp_region,
         queue_name,

infra/batch/main.tf

@@ -45,7 +45,7 @@ locals {
 
   function_pmtiles_builder_config = jsondecode(file("${path.module}/../../functions-python/pmtiles_builder/function_config.json"))
 
-  function_gtfs_change_tracker_config = jsondecode(file("${path.module}/../../functions-python/gtfs_change_tracker/function_config.json"))
+  function_gtfs_datasets_comparer_config = jsondecode(file("${path.module}/../../functions-python/gtfs_datasets_comparer/function_config.json"))
 }
 
 data "google_vpc_access_connector" "vpc_connector" {
@@ -274,11 +274,11 @@ resource "google_cloud_tasks_queue" "pmtiles_builder_task_queue" {
   }
 }
 
-# Task queue to invoke gtfs_change_tracker function
-resource "google_cloud_tasks_queue" "gtfs_change_tracker_task_queue" {
+# Task queue to invoke gtfs_datasets_comparer function
+resource "google_cloud_tasks_queue" "gtfs_datasets_comparer_task_queue" {
   project  = var.project_id
   location = var.gcp_region
-  name     = "gtfs-change-tracker-queue-${var.environment}-${local.deployment_timestamp}"
+  name     = "gtfs-datasets-comparer-queue-${var.environment}-${local.deployment_timestamp}"
 
   rate_limits {
     max_concurrent_dispatches = 10
@@ -332,7 +332,7 @@ resource "google_cloudfunctions2_function" "pubsub_function" {
       MATERIALIZED_VIEW_QUEUE = google_cloud_tasks_queue.refresh_materialized_view_task_queue.name
       PMTILES_BUILDER_QUEUE = google_cloud_tasks_queue.pmtiles_builder_task_queue.name
       REVERSE_GEOLOCATION_QUEUE = "reverse-geolocation-processor-task-queue"
-      GTFS_CHANGE_TRACKER_QUEUE = google_cloud_tasks_queue.gtfs_change_tracker_task_queue.name
+      GTFS_CHANGE_TRACKER_QUEUE = google_cloud_tasks_queue.gtfs_datasets_comparer_task_queue.name
       WEB_REVALIDATION_QUEUE = google_cloud_tasks_queue.web_revalidation_task_queue.name
     }
     dynamic "secret_environment_variables" {
@@ -493,10 +493,10 @@ resource "google_cloud_run_service_iam_member" "pmtiles_builder_invoker" {
   member   = "serviceAccount:${google_service_account.functions_service_account.email}"
 }
 
-resource "google_cloud_run_service_iam_member" "gtfs_change_tracker_invoker" {
+resource "google_cloud_run_service_iam_member" "gtfs_datasets_comparer_invoker" {
   project  = var.project_id
   location = var.gcp_region
-  service  = "${local.function_gtfs_change_tracker_config.name}-${var.environment}"
+  service  = "${local.function_gtfs_datasets_comparer_config.name}-${var.environment}"
   role     = "roles/run.invoker"
   member   = "serviceAccount:${google_service_account.functions_service_account.email}"
 }

infra/functions-python/main.tf

@@ -66,8 +66,8 @@ locals {
   function_pmtiles_builder_config = jsondecode(file("${path.module}/../../functions-python/pmtiles_builder/function_config.json"))
   function_pmtiles_builder_zip = "${path.module}/../../functions-python/pmtiles_builder/.dist/pmtiles_builder.zip"
 
-  function_gtfs_change_tracker_config = jsondecode(file("${path.module}/../../functions-python/gtfs_change_tracker/function_config.json"))
-  function_gtfs_change_tracker_zip = "${path.module}/../../functions-python/gtfs_change_tracker/.dist/gtfs_change_tracker.zip"
+  function_gtfs_datasets_comparer_config = jsondecode(file("${path.module}/../../functions-python/gtfs_datasets_comparer/function_config.json"))
+  function_gtfs_datasets_comparer_zip = "${path.module}/../../functions-python/gtfs_datasets_comparer/.dist/gtfs_datasets_comparer.zip"
 }
 
 locals {
@@ -80,7 +80,7 @@ locals {
     local.function_export_csv_config.secret_environment_variables,
     local.function_tasks_executor_config.secret_environment_variables,
     local.function_pmtiles_builder_config.secret_environment_variables,
-    local.function_gtfs_change_tracker_config.secret_environment_variables
+    local.function_gtfs_datasets_comparer_config.secret_environment_variables
   )
 
   # Remove duplicates by key, keeping the first occurrence
@@ -227,10 +227,10 @@ resource "google_storage_bucket_object" "pmtiles_builder_zip" {
 }
 
 # 16. GTFS Change Tracker
-resource "google_storage_bucket_object" "gtfs_change_tracker_zip" {
+resource "google_storage_bucket_object" "gtfs_datasets_comparer_zip" {
   bucket = google_storage_bucket.functions_bucket.name
-  name   = "gtfs-change-tracker-${substr(filebase64sha256(local.function_gtfs_change_tracker_zip), 0, 10)}.zip"
-  source = local.function_gtfs_change_tracker_zip
+  name   = "gtfs-datasets-comparer-${substr(filebase64sha256(local.function_gtfs_datasets_comparer_zip), 0, 10)}.zip"
+  source = local.function_gtfs_datasets_comparer_zip
 }
 
 # Web app revalidation secret
@@ -1045,20 +1045,20 @@ resource "google_cloudfunctions2_function_iam_member" "pmtiles_builder_invoker"
   member         = "serviceAccount:${google_service_account.functions_service_account.email}"
 }
 
-# Grant execution permission to batchfunctions service account to the gtfs_change_tracker function
-resource "google_cloudfunctions2_function_iam_member" "gtfs_change_tracker_invoker_batch_sa" {
+# Grant execution permission to batchfunctions service account to the gtfs_datasets_comparer function
+resource "google_cloudfunctions2_function_iam_member" "gtfs_datasets_comparer_invoker_batch_sa" {
   project        = var.project_id
   location       = var.gcp_region
-  cloud_function = google_cloudfunctions2_function.gtfs_change_tracker.name
+  cloud_function = google_cloudfunctions2_function.gtfs_datasets_comparer.name
   role           = "roles/cloudfunctions.invoker"
   member         = "serviceAccount:${local.batchfunctions_sa_email}"
 }
 
-# Grant execution permission to the functions service account to the gtfs_change_tracker function
-resource "google_cloudfunctions2_function_iam_member" "gtfs_change_tracker_invoker" {
+# Grant execution permission to the functions service account to the gtfs_datasets_comparer function
+resource "google_cloudfunctions2_function_iam_member" "gtfs_datasets_comparer_invoker" {
   project        = var.project_id
   location       = var.gcp_region
-  cloud_function = google_cloudfunctions2_function.gtfs_change_tracker.name
+  cloud_function = google_cloudfunctions2_function.gtfs_datasets_comparer.name
   role           = "roles/cloudfunctions.invoker"
   member         = "serviceAccount:${google_service_account.functions_service_account.email}"
 }
@@ -1257,21 +1257,21 @@ resource "google_cloudfunctions2_function" "pmtiles_builder" {
 }
 
 
-# 16. functions/gtfs_change_tracker cloud function
-resource "google_cloudfunctions2_function" "gtfs_change_tracker" {
-  name        = "${local.function_gtfs_change_tracker_config.name}-${var.environment}"
+# 16. functions/gtfs_datasets_comparer cloud function
+resource "google_cloudfunctions2_function" "gtfs_datasets_comparer" {
+  name        = "${local.function_gtfs_datasets_comparer_config.name}-${var.environment}"
   project     = var.project_id
-  description = local.function_gtfs_change_tracker_config.description
+  description = local.function_gtfs_datasets_comparer_config.description
   location    = var.gcp_region
   depends_on  = [google_secret_manager_secret_iam_member.secret_iam_member]
 
   build_config {
     runtime     = var.python_runtime
-    entry_point = local.function_gtfs_change_tracker_config.entry_point
+    entry_point = local.function_gtfs_datasets_comparer_config.entry_point
     source {
       storage_source {
         bucket = google_storage_bucket.functions_bucket.name
-        object = google_storage_bucket_object.gtfs_change_tracker_zip.name
+        object = google_storage_bucket_object.gtfs_datasets_comparer_zip.name
       }
     }
   }
@@ -1285,19 +1285,19 @@ resource "google_cloudfunctions2_function" "gtfs_change_tracker" {
       # GTFS_DIFF_DUCKDB_TMPDIR: directs DuckDB spill files to the in-memory volume.
       GTFS_DIFF_DUCKDB_TMPDIR = "/tmp/in-memory"
     }
-    available_memory                 = local.function_gtfs_change_tracker_config.memory
-    timeout_seconds                  = local.function_gtfs_change_tracker_config.timeout
-    available_cpu                    = local.function_gtfs_change_tracker_config.available_cpu
-    max_instance_request_concurrency = local.function_gtfs_change_tracker_config.max_instance_request_concurrency
-    max_instance_count               = local.function_gtfs_change_tracker_config.max_instance_count
-    min_instance_count               = local.function_gtfs_change_tracker_config.min_instance_count
+    available_memory                 = local.function_gtfs_datasets_comparer_config.memory
+    timeout_seconds                  = local.function_gtfs_datasets_comparer_config.timeout
+    available_cpu                    = local.function_gtfs_datasets_comparer_config.available_cpu
+    max_instance_request_concurrency = local.function_gtfs_datasets_comparer_config.max_instance_request_concurrency
+    max_instance_count               = local.function_gtfs_datasets_comparer_config.max_instance_count
+    min_instance_count               = local.function_gtfs_datasets_comparer_config.min_instance_count
     service_account_email            = google_service_account.functions_service_account.email
-    ingress_settings                 = local.function_gtfs_change_tracker_config.ingress_settings
+    ingress_settings                 = local.function_gtfs_datasets_comparer_config.ingress_settings
     vpc_connector                    = data.google_vpc_access_connector.vpc_connector.id
     vpc_connector_egress_settings    = "PRIVATE_RANGES_ONLY"
 
     dynamic "secret_environment_variables" {
-      for_each = local.function_gtfs_change_tracker_config.secret_environment_variables
+      for_each = local.function_gtfs_datasets_comparer_config.secret_environment_variables
       content {
         key        = secret_environment_variables.value["key"]
         project_id = var.project_id
@@ -1311,17 +1311,17 @@ resource "google_cloudfunctions2_function" "gtfs_change_tracker" {
 # google_cloudfunctions2_function does not expose volume mounts in its schema.
 # This terraform_data resource mounts both the datasets GCS bucket and an in-memory tmpfs
 # on the underlying Cloud Run service after the function is deployed.
-resource "terraform_data" "gtfs_change_tracker_gcs_mount" {
+resource "terraform_data" "gtfs_datasets_comparer_gcs_mount" {
   triggers_replace = {
-    function_name = google_cloudfunctions2_function.gtfs_change_tracker.name
+    function_name = google_cloudfunctions2_function.gtfs_datasets_comparer.name
     bucket        = "${var.datasets_bucket_name}-${var.environment}"
     region        = var.gcp_region
     project       = var.project_id
   }
 
   provisioner "local-exec" {
     command = <<-EOT
-      MOUNTS=$(gcloud run services describe ${google_cloudfunctions2_function.gtfs_change_tracker.name} \
+      MOUNTS=$(gcloud run services describe ${google_cloudfunctions2_function.gtfs_datasets_comparer.name} \
         --project ${var.project_id} \
         --region ${var.gcp_region} \
         --format='value(spec.template.spec.volumes[].name)' 2>/dev/null)
@@ -1337,12 +1337,12 @@ resource "terraform_data" "gtfs_change_tracker_gcs_mount" {
       if echo "$MOUNTS" | grep -q "in-memory"; then
         echo "In-memory volume already mounted, skipping."
       else
-        ARGS="$ARGS --add-volume name=in-memory,type=in-memory,size-limit=${var.gtfs_change_tracker_in_memory_size}"
+        ARGS="$ARGS --add-volume name=in-memory,type=in-memory,size-limit=${var.gtfs_datasets_comparer_in_memory_size}"
         ARGS="$ARGS --add-volume-mount volume=in-memory,mount-path=/tmp/in-memory"
       fi
 
       if [ -n "$ARGS" ]; then
-        gcloud run services update ${google_cloudfunctions2_function.gtfs_change_tracker.name} \
+        gcloud run services update ${google_cloudfunctions2_function.gtfs_datasets_comparer.name} \
           --project ${var.project_id} \
           --region ${var.gcp_region} \
           $ARGS \
@@ -1351,7 +1351,7 @@ resource "terraform_data" "gtfs_change_tracker_gcs_mount" {
     EOT
   }
 
-  depends_on = [google_cloudfunctions2_function.gtfs_change_tracker]
+  depends_on = [google_cloudfunctions2_function.gtfs_datasets_comparer]
 }
 
 # Create the Pub/Sub topic used for publishing messages about rebuilding missing bounding boxes

infra/functions-python/vars.tf

@@ -135,8 +135,8 @@ variable "brevo_api_announcements_list_id" {
     description = "Brevo list ID for API announcements"
     default     = ""
 }
-variable "gtfs_change_tracker_in_memory_size" {
+variable "gtfs_datasets_comparer_in_memory_size" {
     type        = string
-    description = "Size limit for the gtfs_change_tracker in-memory tmpfs volume"
+    description = "Size limit for the gtfs_datasets_comparer in-memory tmpfs volume"
     default     = "3Gi"
 }