diff --git a/functions-python/backfill_dataset_service_date_range/src/main.py b/functions-python/backfill_dataset_service_date_range/src/main.py index 908ffca54..2fc3c417f 100644 --- a/functions-python/backfill_dataset_service_date_range/src/main.py +++ b/functions-python/backfill_dataset_service_date_range/src/main.py @@ -94,8 +94,14 @@ def backfill_datasets(session: "Session"): response.raise_for_status() json_data = response.json() else: - logging.info("Blob found, downloading from blob") - json_data = json.loads(dataset_blob.download_as_string()) + try: + logging.info("Blob found, downloading from blob") + json_data = json.loads(dataset_blob.download_as_string()) + except Exception as e: + logging.error(f"Error downloading blob: {e} trying json report url") + response = requests.get(json_report_url) + response.raise_for_status() + json_data = response.json() extracted_service_start_date = ( json_data.get("summary", {}) diff --git a/functions-python/backfill_dataset_service_date_range/tests/test_backfill_dataset_service_date_range_main.py b/functions-python/backfill_dataset_service_date_range/tests/test_backfill_dataset_service_date_range_main.py index 2d539e08f..4af8268bc 100644 --- a/functions-python/backfill_dataset_service_date_range/tests/test_backfill_dataset_service_date_range_main.py +++ b/functions-python/backfill_dataset_service_date_range/tests/test_backfill_dataset_service_date_range_main.py @@ -83,6 +83,58 @@ def test_backfill_datasets(mock_get, mock_storage_client): mock_session.commit.assert_called_once() +@patch("logging.error", autospec=True) +@patch("google.cloud.storage.Client", autospec=True) +@patch("requests.get") +def test_backfill_datasets_error_commit(mock_get, mock_storage_client, mock_logger): + # Mock the storage client and bucket + mock_bucket = MagicMock() + mock_client_instance = mock_storage_client.return_value + mock_client_instance.bucket.return_value = mock_bucket + mock_blob = MagicMock() + mock_blob.exists.return_value = False + mock_bucket.blob.return_value = mock_blob + + mock_session = MagicMock() + mock_dataset = Mock(spec=Gtfsdataset) + mock_dataset.id = 1 + mock_dataset.stable_id = "mdb-392-202406181921" + mock_dataset.service_date_range_end = None + mock_dataset.service_date_range_start = None + mock_dataset.validation_reports = [ + MagicMock( + validator_version="6.0.0", + validated_at="2022-01-01T00:00:00Z", + json_report="http://example-2.com/report.json", + ) + ] + + mock_query = MagicMock() + mock_query.options.return_value = mock_query + mock_query.filter.return_value = mock_query + mock_query.all.return_value = [mock_dataset] + mock_session.query.return_value = mock_query + mock_session.commit.side_effect = Exception("Commit failed") + + mock_response = Mock() + mock_response.status_code = 200 + mock_response.json.return_value = { + "summary": { + "feedInfo": { + "feedServiceWindowStart": "2023-01-01", + "feedServiceWindowEnd": "2023-12-31", + } + } + } + mock_get.return_value = mock_response + + try: + backfill_datasets(mock_session) + except Exception: + mock_session.rollback.assert_called_once() + mock_session.close.assert_called_once() + + @patch("google.cloud.storage.Client", autospec=True) @patch("requests.get") def test_backfill_datasets_no_validation_reports(mock_get, mock_storage_client): diff --git a/functions-python/validation_report_processor/.coveragerc b/functions-python/process_validation_report/.coveragerc similarity index 100% rename from functions-python/validation_report_processor/.coveragerc rename to functions-python/process_validation_report/.coveragerc diff --git a/functions-python/validation_report_processor/.env.rename_me b/functions-python/process_validation_report/.env.rename_me similarity index 100% rename from functions-python/validation_report_processor/.env.rename_me rename to functions-python/process_validation_report/.env.rename_me diff --git a/functions-python/validation_report_processor/README.md b/functions-python/process_validation_report/README.md similarity index 100% rename from functions-python/validation_report_processor/README.md rename to functions-python/process_validation_report/README.md diff --git a/functions-python/validation_report_processor/function_config.json b/functions-python/process_validation_report/function_config.json similarity index 100% rename from functions-python/validation_report_processor/function_config.json rename to functions-python/process_validation_report/function_config.json diff --git a/functions-python/validation_report_processor/requirements.txt b/functions-python/process_validation_report/requirements.txt similarity index 100% rename from functions-python/validation_report_processor/requirements.txt rename to functions-python/process_validation_report/requirements.txt diff --git a/functions-python/validation_report_processor/requirements_dev.txt b/functions-python/process_validation_report/requirements_dev.txt similarity index 100% rename from functions-python/validation_report_processor/requirements_dev.txt rename to functions-python/process_validation_report/requirements_dev.txt diff --git a/functions-python/validation_report_processor/src/__init__.py b/functions-python/process_validation_report/src/__init__.py similarity index 100% rename from functions-python/validation_report_processor/src/__init__.py rename to functions-python/process_validation_report/src/__init__.py diff --git a/functions-python/validation_report_processor/src/main.py b/functions-python/process_validation_report/src/main.py similarity index 95% rename from functions-python/validation_report_processor/src/main.py rename to functions-python/process_validation_report/src/main.py index 092e2b6b3..9029aaf4e 100644 --- a/functions-python/validation_report_processor/src/main.py +++ b/functions-python/process_validation_report/src/main.py @@ -148,6 +148,20 @@ def generate_report_entities( dataset = get_dataset(dataset_stable_id, session) dataset.validation_reports.append(validation_report_entity) + + if ( + "summary" in json_report + and "feedInfo" in json_report["summary"] + and "feedServiceWindowStart" in json_report["summary"]["feedInfo"] + and "feedServiceWindowEnd" in json_report["summary"]["feedInfo"] + ): + dataset.service_date_range_start = json_report["summary"]["feedInfo"][ + "feedServiceWindowStart" + ] + dataset.service_date_range_end = json_report["summary"]["feedInfo"][ + "feedServiceWindowEnd" + ] + for feature_name in json_report["summary"]["gtfsFeatures"]: feature = get_feature(feature_name, session) feature.validations.append(validation_report_entity) diff --git a/functions-python/validation_report_processor/tests/test_validation_report.py b/functions-python/process_validation_report/tests/test_validation_report.py similarity index 100% rename from functions-python/validation_report_processor/tests/test_validation_report.py rename to functions-python/process_validation_report/tests/test_validation_report.py diff --git a/infra/functions-python/main.tf b/infra/functions-python/main.tf index 5d7f45208..048952051 100644 --- a/infra/functions-python/main.tf +++ b/infra/functions-python/main.tf @@ -34,8 +34,8 @@ locals { vpc_connector_name = lower(var.environment) == "dev" ? "vpc-connector-qa" : "vpc-connector-${lower(var.environment)}" vpc_connector_project = lower(var.environment) == "dev" ? "mobility-feeds-qa" : var.project_id - function_process_validation_report_config = jsondecode(file("${path.module}/../../functions-python/validation_report_processor/function_config.json")) - function_process_validation_report_zip = "${path.module}/../../functions-python/validation_report_processor/.dist/validation_report_processor.zip" + function_process_validation_report_config = jsondecode(file("${path.module}/../../functions-python/process_validation_report/function_config.json")) + function_process_validation_report_zip = "${path.module}/../../functions-python/process_validation_report/.dist/process_validation_report.zip" public_hosted_datasets_url = lower(var.environment) == "prod" ? "https://${var.public_hosted_datasets_dns}" : "https://${var.environment}-${var.public_hosted_datasets_dns}" function_update_validation_report_config = jsondecode(file("${path.module}/../../functions-python/update_validation_report/function_config.json"))