Skip to content

Commit acf9e5d

Browse files
davidgamezdavidgamez
authored
fix: add common variables to all functions (#1676)
1 parent 5b0be37 commit acf9e5d

4 files changed

Lines changed: 122 additions & 23 deletions

File tree

‎api/src/shared/common/gcp_utils.py‎

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -194,11 +194,10 @@ def create_http_task_with_name(
194194
dispatch_deadline=duration_pb2.Duration(seconds=timeout_s),
195195
)
196196
try:
197-
response = client.create_task(parent=parent, task=task)
197+
client.create_task(parent=parent, task=task)
198198
logging.info("Task created with task_name: %s", task_name)
199199
except Exception as e:
200200
if "Requested entity already exists" in str(e):
201201
logging.info("Task already exists for %s, skipping.", task_name)
202202
else:
203203
logging.error("Error creating task: %s", e)
204-
logging.error("response: %s", response)

‎infra/batch/main.tf‎

Lines changed: 8 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -173,11 +173,13 @@ resource "google_cloudfunctions2_function" "batch_datasets" {
173173
vpc_connector_egress_settings = "PRIVATE_RANGES_ONLY"
174174

175175
environment_variables = {
176+
ENVIRONMENT = var.environment
177+
PROJECT_ID = var.project_id
178+
GCP_REGION = var.gcp_region
179+
SERVICE_ACCOUNT_EMAIL = google_service_account.functions_service_account.email
176180
PUBSUB_TOPIC_NAME = google_pubsub_topic.pubsub_topic.name
177-
PROJECT_ID = var.project_id
178181
# prevents multiline logs from being truncated on GCP console
179182
PYTHONNODEBUGRANGES = 0
180-
ENVIRONMENT = var.environment
181183
}
182184
dynamic "secret_environment_variables" {
183185
for_each = local.function_batch_datasets_config.secret_environment_variables
@@ -296,15 +298,15 @@ resource "google_cloudfunctions2_function" "pubsub_function" {
296298
vpc_connector_egress_settings = "PRIVATE_RANGES_ONLY"
297299

298300
environment_variables = {
301+
ENVIRONMENT = var.environment
302+
PROJECT_ID = var.project_id
303+
GCP_REGION = var.gcp_region
304+
SERVICE_ACCOUNT_EMAIL = google_service_account.functions_service_account.email
299305
DATASETS_BUCKET_NAME = google_storage_bucket.datasets_bucket.name
300306
# prevents multiline logs from being truncated on GCP console
301307
PYTHONNODEBUGRANGES = 0
302308
DB_REUSE_SESSION = "True"
303-
ENVIRONMENT = var.environment
304309
PUBLIC_HOSTED_DATASETS_URL = local.public_hosted_datasets_url
305-
PROJECT_ID = var.project_id
306-
GCP_REGION = var.gcp_region
307-
SERVICE_ACCOUNT_EMAIL = google_service_account.functions_service_account.email
308310
MATERIALIZED_VIEW_QUEUE = google_cloud_tasks_queue.refresh_materialized_view_task_queue.name
309311
PMTILES_BUILDER_QUEUE = google_cloud_tasks_queue.pmtiles_builder_task_queue.name
310312
REVERSE_GEOLOCATION_QUEUE = "reverse-geolocation-processor-task-queue"

‎infra/functions-python/main.tf‎

Lines changed: 38 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -394,6 +394,9 @@ resource "google_cloudfunctions2_function" "compute_validation_report_counters"
394394

395395
environment_variables = {
396396
ENVIRONMENT = var.environment
397+
PROJECT_ID = var.project_id
398+
GCP_REGION = var.gcp_region
399+
SERVICE_ACCOUNT_EMAIL = google_service_account.functions_service_account.email
397400
PYTHONNODEBUGRANGES = 0
398401
}
399402

@@ -439,7 +442,10 @@ resource "google_cloudfunctions2_function" "gbfs_validator_batch" {
439442
}
440443
service_config {
441444
environment_variables = {
442-
PROJECT_ID = var.project_id
445+
ENVIRONMENT = var.environment
446+
PROJECT_ID = var.project_id
447+
GCP_REGION = var.gcp_region
448+
SERVICE_ACCOUNT_EMAIL = google_service_account.functions_service_account.email
443449
PUBSUB_TOPIC_NAME = google_pubsub_topic.validate_gbfs_feed.name
444450
PYTHONNODEBUGRANGES = 0
445451
FEEDS_LIMIT = lower(var.environment) == "dev" ? "10" : null
@@ -595,10 +601,11 @@ resource "google_cloudfunctions2_function" "gbfs_validator_pubsub" {
595601
vpc_connector_egress_settings = "PRIVATE_RANGES_ONLY"
596602
environment_variables = {
597603
ENVIRONMENT = var.environment
598-
BUCKET_NAME = google_storage_bucket.gbfs_snapshots_bucket.name
599604
PROJECT_ID = var.project_id
600605
GCP_REGION = var.gcp_region
601606
SERVICE_ACCOUNT_EMAIL = google_service_account.functions_service_account.email
607+
BUCKET_NAME = google_storage_bucket.gbfs_snapshots_bucket.name
608+
SERVICE_ACCOUNT_EMAIL = google_service_account.functions_service_account.email
602609
QUEUE_NAME = google_cloud_tasks_queue.reverse_geolocation_task_queue_processor.name
603610
}
604611
dynamic "secret_environment_variables" {
@@ -632,10 +639,14 @@ resource "google_cloudfunctions2_function" "operations_api" {
632639
}
633640
service_config {
634641
environment_variables = {
642+
ENVIRONMENT = var.environment
635643
PROJECT_ID = var.project_id
644+
GCP_REGION = var.gcp_region
645+
SERVICE_ACCOUNT_EMAIL = google_service_account.functions_service_account.email
636646
PYTHONNODEBUGRANGES = 0
637647
GOOGLE_CLIENT_ID = var.operations_oauth2_client_id
638648
DATASET_PROCESSING_TOPIC_NAME = "datasets-batch-topic-${var.environment}"
649+
WEB_REVALIDATION_QUEUE = google_cloud_tasks_queue.web_revalidation_task_queue.name
639650
}
640651
available_memory = local.function_operations_api_config.memory
641652
timeout_seconds = local.function_operations_api_config.timeout
@@ -687,6 +698,9 @@ resource "google_cloudfunctions2_function" "backfill_dataset_service_date_range"
687698
environment_variables = {
688699
# prevents multiline logs from being truncated on GCP console
689700
ENVIRONMENT = var.environment
701+
PROJECT_ID = var.project_id
702+
GCP_REGION = var.gcp_region
703+
SERVICE_ACCOUNT_EMAIL = google_service_account.functions_service_account.email
690704
PYTHONNODEBUGRANGES = 0
691705
}
692706
dynamic "secret_environment_variables" {
@@ -725,9 +739,11 @@ resource "google_cloudfunctions2_function" "export_csv" {
725739
}
726740
service_config {
727741
environment_variables = {
728-
DATASETS_BUCKET_NAME = data.google_storage_bucket.datasets_bucket.name
729-
PROJECT_ID = var.project_id
730742
ENVIRONMENT = var.environment
743+
PROJECT_ID = var.project_id
744+
GCP_REGION = var.gcp_region
745+
SERVICE_ACCOUNT_EMAIL = google_service_account.functions_service_account.email
746+
DATASETS_BUCKET_NAME = data.google_storage_bucket.datasets_bucket.name
731747
}
732748
available_memory = local.function_export_csv_config.memory
733749
timeout_seconds = local.function_export_csv_config.timeout
@@ -779,11 +795,11 @@ resource "google_cloudfunctions2_function" "update_feed_status" {
779795
vpc_connector_egress_settings = "PRIVATE_RANGES_ONLY"
780796

781797
environment_variables = {
798+
ENVIRONMENT = var.environment
782799
PROJECT_ID = var.project_id
783800
GCP_REGION = var.gcp_region
784-
ENVIRONMENT = var.environment
785-
MATERIALIZED_VIEW_QUEUE = google_cloud_tasks_queue.refresh_materialized_view_task_queue.name
786-
SERVICE_ACCOUNT_EMAIL = google_service_account.functions_service_account.email
801+
SERVICE_ACCOUNT_EMAIL = google_service_account.functions_service_account.email
802+
MATERIALIZED_VIEW_QUEUE = google_cloud_tasks_queue.refresh_materialized_view_task_queue.name
787803
# prevents multiline logs from being truncated on GCP console
788804
PYTHONNODEBUGRANGES = 0
789805
}
@@ -868,6 +884,10 @@ resource "google_cloudfunctions2_function" "reverse_geolocation_populate" {
868884
}
869885
service_config {
870886
environment_variables = {
887+
ENVIRONMENT = var.environment
888+
PROJECT_ID = var.project_id
889+
GCP_REGION = var.gcp_region
890+
SERVICE_ACCOUNT_EMAIL = google_service_account.functions_service_account.email
871891
PYTHONNODEBUGRANGES = 0
872892
DB_REUSE_SESSION = "True"
873893
}
@@ -916,11 +936,11 @@ resource "google_cloudfunctions2_function" "reverse_geolocation_processor" {
916936
service_config {
917937
environment_variables = {
918938
PYTHONNODEBUGRANGES = 0
939+
ENVIRONMENT = var.environment
919940
PROJECT_ID = var.project_id
920941
GCP_REGION = var.gcp_region
921-
ENVIRONMENT = var.environment
922-
MATERIALIZED_VIEW_QUEUE = google_cloud_tasks_queue.refresh_materialized_view_task_queue.name
923942
SERVICE_ACCOUNT_EMAIL = google_service_account.functions_service_account.email
943+
MATERIALIZED_VIEW_QUEUE = google_cloud_tasks_queue.refresh_materialized_view_task_queue.name
924944
DATASETS_BUCKET_NAME_GTFS = "${var.datasets_bucket_name}-${var.environment}"
925945
DATASETS_BUCKET_NAME_GBFS = "${var.gbfs_bucket_name}-${var.environment}"
926946
WEB_REVALIDATION_QUEUE = google_cloud_tasks_queue.web_revalidation_task_queue.name
@@ -1012,11 +1032,12 @@ resource "google_cloudfunctions2_function" "reverse_geolocation_batch" {
10121032
service_config {
10131033
environment_variables = {
10141034
PYTHONNODEBUGRANGES = 0
1035+
ENVIRONMENT = var.environment
10151036
PROJECT_ID = var.project_id
1016-
DATASETS_BUCKET_NAME = "${var.datasets_bucket_name}-${var.environment}"
1017-
QUEUE_NAME = google_cloud_tasks_queue.reverse_geolocation_task_queue_processor.name
10181037
GCP_REGION = var.gcp_region
10191038
SERVICE_ACCOUNT_EMAIL = google_service_account.functions_service_account.email
1039+
DATASETS_BUCKET_NAME = "${var.datasets_bucket_name}-${var.environment}"
1040+
QUEUE_NAME = google_cloud_tasks_queue.reverse_geolocation_task_queue_processor.name
10201041
}
10211042
available_memory = local.function_reverse_geolocation_config.available_memory
10221043
timeout_seconds = local.function_reverse_geolocation_config.timeout
@@ -1081,17 +1102,17 @@ resource "google_cloudfunctions2_function" "tasks_executor" {
10811102
}
10821103
service_config {
10831104
environment_variables = {
1084-
PROJECT_ID = var.project_id
10851105
ENVIRONMENT = var.environment
1106+
PROJECT_ID = var.project_id
1107+
GCP_REGION = var.gcp_region
1108+
SERVICE_ACCOUNT_EMAIL = google_service_account.functions_service_account.email
10861109
BOUNDING_BOXES_PUBSUB_TOPIC_NAME = google_pubsub_topic.rebuild_missing_bounding_boxes.name
10871110
DATASET_PROCESSING_TOPIC_NAME = "datasets-batch-topic-${var.environment}"
10881111
MATERIALIZED_VIEW_QUEUE = google_cloud_tasks_queue.refresh_materialized_view_task_queue.name
10891112
DATASETS_BUCKET_NAME = "${var.datasets_bucket_name}-${var.environment}"
10901113
GBFS_SNAPSHOTS_BUCKET_NAME = google_storage_bucket.gbfs_snapshots_bucket.name
10911114
PMTILES_BUILDER_QUEUE = google_cloud_tasks_queue.pmtiles_builder_task_queue.name
10921115
TASK_RUN_SYNC_QUEUE = google_cloud_tasks_queue.task_run_sync_queue.name
1093-
SERVICE_ACCOUNT_EMAIL = google_service_account.functions_service_account.email
1094-
GCP_REGION = var.gcp_region
10951116
TDG_API_TOKEN = var.tdg_api_token
10961117
WEB_REVALIDATION_QUEUE = google_cloud_tasks_queue.web_revalidation_task_queue.name
10971118
WEB_APP_REVALIDATE_URL = var.web_app_revalidate_url
@@ -1148,8 +1169,10 @@ resource "google_cloudfunctions2_function" "pmtiles_builder" {
11481169
}
11491170
service_config {
11501171
environment_variables = {
1151-
PROJECT_ID = var.project_id
11521172
ENVIRONMENT = var.environment
1173+
PROJECT_ID = var.project_id
1174+
GCP_REGION = var.gcp_region
1175+
SERVICE_ACCOUNT_EMAIL = google_service_account.functions_service_account.email
11531176
PUBSUB_TOPIC_NAME = "rebuild-bounding-boxes-topic"
11541177
MATERIALIZED_VIEW_QUEUE = google_cloud_tasks_queue.refresh_materialized_view_task_queue.name
11551178
DATASETS_BUCKET_NAME = "${var.datasets_bucket_name}-${var.environment}"

‎scripts/init-local-folder.sh‎

Lines changed: 75 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,75 @@
1+
#!/bin/bash
2+
#
3+
#
4+
# MobilityData 2026
5+
#
6+
# Licensed under the Apache License, Version 2.0 (the "License");
7+
# you may not use this file except in compliance with the License.
8+
# You may obtain a copy of the License at
9+
#
10+
# http://www.apache.org/licenses/LICENSE-2.0
11+
#
12+
# Unless required by applicable law or agreed to in writing, software
13+
# distributed under the License is distributed on an "AS IS" BASIS,
14+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15+
# See the License for the specific language governing permissions and
16+
# limitations under the License.
17+
18+
# This script initializes a local development environment for a new branch/folder.
19+
# It rebuilds the local and test databases, regenerates SQLAlchemy models,
20+
# FastAPI stubs, and Operations API schema and stubs.
21+
#
22+
# Usage:
23+
# ./init-local-folder.sh [--populate-db]
24+
#
25+
# Options:
26+
# --populate-db Download the latest CSV from GCS and populate the main database.
27+
# Without this flag, the database is only rebuilt with Liquibase migrations.
28+
#
29+
# Dependencies:
30+
# docker, docker-compose, wget, openapi-generator (setup-openapi-generator.sh), yq (v4+)
31+
32+
# relative path
33+
SCRIPT_PATH="$(dirname -- "${BASH_SOURCE[0]}")"
34+
35+
POPULATE_DB=false
36+
while [[ $# -gt 0 ]]; do
37+
case "$1" in
38+
--populate-db) POPULATE_DB=true; shift ;;
39+
--help)
40+
echo "Usage: $0 [--populate-db]"
41+
echo " --populate-db Download and populate the main DB with the latest CSV data."
42+
exit 0 ;;
43+
*) echo "Unknown option: $1" >&2; exit 1 ;;
44+
esac
45+
done
46+
47+
echo "==> Rebuilding main local database..."
48+
if [ "$POPULATE_DB" = true ]; then
49+
"$SCRIPT_PATH/docker-localdb-rebuild-data.sh" --populate-db
50+
else
51+
"$SCRIPT_PATH/docker-localdb-rebuild-data.sh"
52+
fi
53+
54+
echo "==> Rebuilding test database..."
55+
"$SCRIPT_PATH/docker-localdb-rebuild-data.sh" --use-test-db
56+
57+
echo "==> Generating SQLAlchemy models (db-gen)..."
58+
"$SCRIPT_PATH/db-gen.sh"
59+
60+
echo "==> Generating FastAPI stubs (api-gen)..."
61+
"$SCRIPT_PATH/api-gen.sh"
62+
63+
echo "==> Setting up the OpenApi Generator"
64+
"$SCRIPT_PATH/setup-openapi-generator.sh"
65+
66+
echo "==> Syncing Operations API schema..."
67+
"$SCRIPT_PATH/api-operations-update-schema.sh"
68+
69+
echo "==> Generating Operations API stubs (api-operations-gen)..."
70+
"$SCRIPT_PATH/api-operations-gen.sh"
71+
72+
echo "==> Setting up all Python functions..."
73+
"$SCRIPT_PATH/function-python-setup.sh" --all
74+
75+
echo "==> Local folder initialization complete."

0 commit comments

Comments
 (0)