From 82fe5fa188bf3b8154130779bd6e01a0e04b73c7 Mon Sep 17 00:00:00 2001 From: David Gamez Diaz <1192523+davidgamez@users.noreply.github.com> Date: Thu, 31 Jul 2025 12:02:47 -0400 Subject: [PATCH 1/2] add support for filtering feed ids in the gbfs batch function --- functions-python/gbfs_validator/src/main.py | 34 +++++++++++++++-- .../tests/test_gbfs_validator.py | 37 +++++++++++++++++++ 2 files changed, 67 insertions(+), 4 deletions(-) diff --git a/functions-python/gbfs_validator/src/main.py b/functions-python/gbfs_validator/src/main.py index ea4bf9673..a98f68193 100644 --- a/functions-python/gbfs_validator/src/main.py +++ b/functions-python/gbfs_validator/src/main.py @@ -101,9 +101,11 @@ def gbfs_validator_pubsub(cloud_event: CloudEvent): @with_db_session @functions_framework.http -def gbfs_validator_batch(_, db_session: Session): +def gbfs_validator_batch(request, db_session: Session): """ - HTTP Cloud Function to trigger the GBFS Validator function for multiple datasets. + HTTP Cloud Function to trigger the GBFS Validator function for multiple datasets. + When the request is a POST request with a JSON body containing `feed_ids`, + it processes only those feeds. Otherwise, it processes all feeds in the database. @param _: The request object. @return: The response of the function. """ @@ -113,9 +115,23 @@ def gbfs_validator_batch(_, db_session: Session): logging.error("PUBSUB_TOPIC_NAME environment variable not set.") return "PUBSUB_TOPIC_NAME environment variable not set.", 500 - # Get all GBFS feeds from the database try: - gbfs_feeds = fetch_all_gbfs_feeds(db_session) + feed_ids = None + if request and request.method == "POST" and request.is_json: + request_json = request.get_json() + feed_ids = request_json.get("feed_ids") if request_json else None + else: + logging.info("Request body not provided or not a valid JSON.") + except Exception as e: + logging.error("Error parsing request body: %s", e) + return "Invalid request body.", 400 + + try: + if feed_ids: + gbfs_feeds = fetch_gbfs_feeds_by_ids(db_session, feed_ids) + else: + # Get all GBFS feeds from the database + gbfs_feeds = fetch_all_gbfs_feeds(db_session) except Exception: return "Error getting all GBFS feeds.", 500 @@ -150,3 +166,13 @@ def gbfs_validator_batch(_, db_session: Session): f"GBFS Validator batch function triggered successfully for {len(feeds_data)} feeds.", 200, ) + + +def fetch_gbfs_feeds_by_ids(db_session, feed_ids): + """Fetch GBFS feeds by their IDs from the database.""" + gbfs_feeds = ( + db_session.query(Gbfsfeed) + .filter(Gbfsfeed.id.in_(feed_ids), Gbfsfeed.status != "deprecated") + .all() + ) + return gbfs_feeds diff --git a/functions-python/gbfs_validator/tests/test_gbfs_validator.py b/functions-python/gbfs_validator/tests/test_gbfs_validator.py index ce1306577..dc86577c2 100644 --- a/functions-python/gbfs_validator/tests/test_gbfs_validator.py +++ b/functions-python/gbfs_validator/tests/test_gbfs_validator.py @@ -179,3 +179,40 @@ def test_gbfs_validator_batch_publish_exception( # Call the function result = gbfs_validator_batch(None) self.assertEqual(result[1], 500) + + @patch.dict( + os.environ, + { + "PUBSUB_TOPIC_NAME": "mock-topic", + }, + ) + @patch("main.pubsub_v1.PublisherClient") + @patch("main.fetch_gbfs_feeds_by_ids") + def test_gbfs_validator_batch_by_feed_ids( + self, mock_fetch_gbfs_feeds_by_ids, mock_publisher_client + ): + # Prepare mocks + mock_session = MagicMock() + # mock_database.return_value.start_db_session.return_value = mock_session + + mock_publisher = MagicMock() + mock_publisher_client.return_value = mock_publisher + + mock_feed = MagicMock() + mock_feed.stable_id = "mock-stable-id" + mock_feed.id = str(uuid.uuid4()) + mock_feed.auto_discovery_url = "http://mock-url.com" + mock_feed.gbfsversions = [MagicMock(version="1.0")] + mock_feed_2 = copy.deepcopy(mock_feed) + mock_feed_2.gbfsversions = [] + mock_fetch_gbfs_feeds_by_ids.return_value = [mock_feed, mock_feed_2] + request = MagicMock() + request.method = "POST" + request.is_json = True + request.get_json.return_value = {"feed_ids": [mock_feed.id, mock_feed_2.id]} + # Call the function + result = gbfs_validator_batch(request, db_session=mock_session) + self.assertEqual(result[1], 200) + + mock_fetch_gbfs_feeds_by_ids.assert_called_once() + self.assertEqual(mock_publisher.publish.call_count, 2) From db60fafb048062448a468fd97c43b6594b435f8c Mon Sep 17 00:00:00 2001 From: David Gamez Diaz <1192523+davidgamez@users.noreply.github.com> Date: Thu, 31 Jul 2025 14:34:42 -0400 Subject: [PATCH 2/2] use stable_id instead of feed id --- functions-python/gbfs_validator/README.md | 9 ++++++++- functions-python/gbfs_validator/src/main.py | 20 +++++++++++-------- .../tests/test_gbfs_validator.py | 16 +++++++-------- 3 files changed, 28 insertions(+), 17 deletions(-) diff --git a/functions-python/gbfs_validator/README.md b/functions-python/gbfs_validator/README.md index 4f5926619..07254ed91 100644 --- a/functions-python/gbfs_validator/README.md +++ b/functions-python/gbfs_validator/README.md @@ -30,7 +30,7 @@ The message published by the batch function to the Pub/Sub topic follows this fo ### Functionality Details -- **`gbfs-validator-batch`**: Triggered per execution ID, this function iterates over all GBFS feeds, preparing and publishing individual messages to the Pub/Sub topic. +- **`gbfs-validator-batch`**: Triggered per execution ID, when the request is a POST request with a JSON body containing `feed_stable_ids`, it publishes events related to only those feeds. Otherwise, it publishes avents of all feeds to the Pub/Sub topic. - **`gbfs-validator-pubsub`**: Triggered per feed, this function performs the following steps: 1. **Access the autodiscovery URL and update versions**: The function accesses the autodiscovery URL to update the **GBFSVersions** table. 2. **Measure latency and validate the feed**: For each version, the function measures the response latency and validates the feed. The validation summary is stored in GCP, and the total error count is extracted and saved in the **GBFSValidationReport**. @@ -46,6 +46,13 @@ The `gbfs-validator-batch` function requires the following environment variables - **`PROJECT_ID`**: The Google Cloud Project ID used to construct the full topic path. - **`FEEDS_DATABASE_URL`**: The database connection string for accessing the GBFS feeds. +Optional request body parameters for the batch function: +```json +{ + "feed_stable_ids": ["feed_id_1", "feed_id_2"] +} +``` + ### Pub/Sub Function Environment Variables The `gbfs-validator-pubsub` function requires the following environment variables: diff --git a/functions-python/gbfs_validator/src/main.py b/functions-python/gbfs_validator/src/main.py index a98f68193..c4ed1d7c9 100644 --- a/functions-python/gbfs_validator/src/main.py +++ b/functions-python/gbfs_validator/src/main.py @@ -104,7 +104,7 @@ def gbfs_validator_pubsub(cloud_event: CloudEvent): def gbfs_validator_batch(request, db_session: Session): """ HTTP Cloud Function to trigger the GBFS Validator function for multiple datasets. - When the request is a POST request with a JSON body containing `feed_ids`, + When the request is a POST request with a JSON body containing `feed_stable_ids`, it processes only those feeds. Otherwise, it processes all feeds in the database. @param _: The request object. @return: The response of the function. @@ -116,10 +116,12 @@ def gbfs_validator_batch(request, db_session: Session): return "PUBSUB_TOPIC_NAME environment variable not set.", 500 try: - feed_ids = None + feed_stable_ids = None if request and request.method == "POST" and request.is_json: request_json = request.get_json() - feed_ids = request_json.get("feed_ids") if request_json else None + feed_stable_ids = ( + request_json.get("feed_stable_ids") if request_json else None + ) else: logging.info("Request body not provided or not a valid JSON.") except Exception as e: @@ -127,8 +129,8 @@ def gbfs_validator_batch(request, db_session: Session): return "Invalid request body.", 400 try: - if feed_ids: - gbfs_feeds = fetch_gbfs_feeds_by_ids(db_session, feed_ids) + if feed_stable_ids: + gbfs_feeds = fetch_gbfs_feeds_by_stable_ids(db_session, feed_stable_ids) else: # Get all GBFS feeds from the database gbfs_feeds = fetch_all_gbfs_feeds(db_session) @@ -168,11 +170,13 @@ def gbfs_validator_batch(request, db_session: Session): ) -def fetch_gbfs_feeds_by_ids(db_session, feed_ids): - """Fetch GBFS feeds by their IDs from the database.""" +def fetch_gbfs_feeds_by_stable_ids(db_session, feed_stable_ids): + """Fetch GBFS feeds by their IDs and not deprecated from the database""" gbfs_feeds = ( db_session.query(Gbfsfeed) - .filter(Gbfsfeed.id.in_(feed_ids), Gbfsfeed.status != "deprecated") + .filter( + Gbfsfeed.stable_id.in_(feed_stable_ids), Gbfsfeed.status != "deprecated" + ) .all() ) return gbfs_feeds diff --git a/functions-python/gbfs_validator/tests/test_gbfs_validator.py b/functions-python/gbfs_validator/tests/test_gbfs_validator.py index dc86577c2..e35c4723a 100644 --- a/functions-python/gbfs_validator/tests/test_gbfs_validator.py +++ b/functions-python/gbfs_validator/tests/test_gbfs_validator.py @@ -75,7 +75,6 @@ def test_gbfs_validator_batch( ): # Prepare mocks mock_session = MagicMock() - # mock_database.return_value.start_db_session.return_value = mock_session mock_publisher = MagicMock() mock_publisher_client.return_value = mock_publisher @@ -187,13 +186,12 @@ def test_gbfs_validator_batch_publish_exception( }, ) @patch("main.pubsub_v1.PublisherClient") - @patch("main.fetch_gbfs_feeds_by_ids") - def test_gbfs_validator_batch_by_feed_ids( - self, mock_fetch_gbfs_feeds_by_ids, mock_publisher_client + @patch("main.fetch_gbfs_feeds_by_stable_ids") + def test_gbfs_validator_batch_by_feed_stable_ids( + self, fetch_gbfs_feeds_by_stable_ids, mock_publisher_client ): # Prepare mocks mock_session = MagicMock() - # mock_database.return_value.start_db_session.return_value = mock_session mock_publisher = MagicMock() mock_publisher_client.return_value = mock_publisher @@ -205,14 +203,16 @@ def test_gbfs_validator_batch_by_feed_ids( mock_feed.gbfsversions = [MagicMock(version="1.0")] mock_feed_2 = copy.deepcopy(mock_feed) mock_feed_2.gbfsversions = [] - mock_fetch_gbfs_feeds_by_ids.return_value = [mock_feed, mock_feed_2] + fetch_gbfs_feeds_by_stable_ids.return_value = [mock_feed, mock_feed_2] request = MagicMock() request.method = "POST" request.is_json = True - request.get_json.return_value = {"feed_ids": [mock_feed.id, mock_feed_2.id]} + request.get_json.return_value = { + "feed_stable_ids": [mock_feed.id, mock_feed_2.id] + } # Call the function result = gbfs_validator_batch(request, db_session=mock_session) self.assertEqual(result[1], 200) - mock_fetch_gbfs_feeds_by_ids.assert_called_once() + fetch_gbfs_feeds_by_stable_ids.assert_called_once() self.assertEqual(mock_publisher.publish.call_count, 2)