diff --git a/functions-python/README.md b/functions-python/README.md index 3e3526e8a..c82e711ef 100644 --- a/functions-python/README.md +++ b/functions-python/README.md @@ -156,6 +156,13 @@ or ``` scripts/api-tests.sh --folder functions-python ``` + +To run the tests and generate the html coverage report: +``` +scripts/api-tests.sh --folder functions-python --html_report +``` +_The coverage reports are located in `{project}/scripts/coverage_reports` as individual folder per function._ + This will - run the `function-python-setup.sh` script for the function (ie create the `shared` and `test_shared` folders with symlinks) - Create a python virtual environment in the function folder, e.g.: `functions-python/batch_datasets/venv` diff --git a/functions-python/helpers/tests/test_transform.py b/functions-python/helpers/tests/test_transform.py index 4fb81711c..f421614e4 100644 --- a/functions-python/helpers/tests/test_transform.py +++ b/functions-python/helpers/tests/test_transform.py @@ -1,4 +1,4 @@ -from transform import to_boolean +from transform import to_boolean, get_nested_value def test_to_boolean(): @@ -19,3 +19,44 @@ def test_to_boolean(): assert to_boolean(None) is False assert to_boolean([]) is False assert to_boolean({}) is False + + +def test_get_nested_value(): + # Test case 1: Nested dictionary with string value + assert get_nested_value({"a": {"b": {"c": "d"}}}, ["a", "b", "c"]) == "d" + + # Test case 2: Nested dictionary with integer value + assert get_nested_value({"a": {"b": {"c": 1}}}, ["a", "b", "c"]) == 1 + + # Test case 3: Nested dictionary with float value + assert get_nested_value({"a": {"b": {"c": 1.5}}}, ["a", "b", "c"]) == 1.5 + + # Test case 4: Nested dictionary with boolean value + assert get_nested_value({"a": {"b": {"c": True}}}, ["a", "b", "c"]) is True + + # Test case 5: Nested dictionary with string value that needs trimming + assert get_nested_value({"a": {"b": {"c": " d "}}}, ["a", "b", "c"]) == "d" + + # Test case 6: Key not found in the dictionary + assert get_nested_value({"a": {"b": {"c": "d"}}}, ["a", "b", "x"]) is None + assert ( + get_nested_value({"a": {"b": {"c": "d"}}}, ["a", "b", "x"], "default") + == "default" + ) + assert get_nested_value({"a": {"b": {"c": "d"}}}, ["a", "b", "x"], []) == [] + + # Test case 7: Intermediate key not found in the dictionary + assert get_nested_value({"a": {"b": {"c": "d"}}}, ["a", "x", "c"]) is None + assert ( + get_nested_value({"a": {"b": {"c": "d"}}}, ["a", "x", "c"], "default") + == "default" + ) + assert get_nested_value({"a": {"b": {"c": "d"}}}, ["a", "x", "c"], []) == [] + + # Test case 8: Empty keys list + assert get_nested_value({"a": {"b": {"c": "d"}}}, []) is None + assert get_nested_value({"a": {"b": {"c": "d"}}}, [], {}) == {} + + # Test case 9: Non-dictionary data + assert get_nested_value("not a dict", ["a", "b", "c"]) is None + assert get_nested_value("not a dict", ["a", "b", "c"], []) == [] diff --git a/functions-python/helpers/transform.py b/functions-python/helpers/transform.py index e7d5264f9..9b96c63b8 100644 --- a/functions-python/helpers/transform.py +++ b/functions-python/helpers/transform.py @@ -13,6 +13,7 @@ # See the License for the specific language governing permissions and # limitations under the License. # +from typing import List, Optional def to_boolean(value): @@ -24,3 +25,31 @@ def to_boolean(value): if isinstance(value, str): return value.lower() in ["true", "1", "yes", "y"] return False + + +def get_nested_value( + data: dict, keys: List[str], default_value: Optional[any] = None +) -> Optional[any]: + """ + Retrieve the value from a nested dictionary given a list of keys. + + Args: + data (dict): The dictionary to search. + keys (List[str]): The list of keys representing the path to the field. + default_value: The value to return if the field is not found. + + Returns: + Optional[any]: The value if found and valid, otherwise None. The str values are trimmed. + """ + if not keys: + return default_value + current_data = data + for key in keys: + if isinstance(current_data, dict) and key in current_data: + current_data = current_data[key] + else: + return default_value + if isinstance(current_data, str): + result = current_data.strip() + return result if result else default_value + return current_data diff --git a/functions-python/process_validation_report/src/main.py b/functions-python/process_validation_report/src/main.py index 9029aaf4e..58b35cfee 100644 --- a/functions-python/process_validation_report/src/main.py +++ b/functions-python/process_validation_report/src/main.py @@ -27,6 +27,7 @@ Gtfsdataset, ) from shared.helpers.logger import Logger +from shared.helpers.transform import get_nested_value logging.basicConfig(level=logging.INFO) @@ -149,20 +150,9 @@ def generate_report_entities( dataset = get_dataset(dataset_stable_id, session) dataset.validation_reports.append(validation_report_entity) - if ( - "summary" in json_report - and "feedInfo" in json_report["summary"] - and "feedServiceWindowStart" in json_report["summary"]["feedInfo"] - and "feedServiceWindowEnd" in json_report["summary"]["feedInfo"] - ): - dataset.service_date_range_start = json_report["summary"]["feedInfo"][ - "feedServiceWindowStart" - ] - dataset.service_date_range_end = json_report["summary"]["feedInfo"][ - "feedServiceWindowEnd" - ] - - for feature_name in json_report["summary"]["gtfsFeatures"]: + populate_service_date(dataset, json_report) + + for feature_name in get_nested_value(json_report, ["summary", "gtfsFeatures"], []): feature = get_feature(feature_name, session) feature.validations.append(validation_report_entity) entities.append(feature) @@ -179,6 +169,23 @@ def generate_report_entities( return entities +def populate_service_date(dataset, json_report): + """ + Populates the service date range of the dataset based on the JSON report. + The service date range is extracted from the feedServiceWindowStart and feedServiceWindowEnd fields, + if both are present and not empty. + """ + feed_service_window_start = get_nested_value( + json_report, ["summary", "feedInfo", "feedServiceWindowStart"] + ) + feed_service_window_end = get_nested_value( + json_report, ["summary", "feedInfo", "feedServiceWindowEnd"] + ) + if feed_service_window_start and feed_service_window_end: + dataset.service_date_range_start = feed_service_window_start + dataset.service_date_range_end = feed_service_window_end + + def create_validation_report_entities(feed_stable_id, dataset_stable_id, version): """ Creates and stores entities based on a validation report. diff --git a/functions-python/process_validation_report/tests/test_validation_report.py b/functions-python/process_validation_report/tests/test_validation_report.py index 483abfd19..a035b78fe 100644 --- a/functions-python/process_validation_report/tests/test_validation_report.py +++ b/functions-python/process_validation_report/tests/test_validation_report.py @@ -19,6 +19,7 @@ get_dataset, create_validation_report_entities, process_validation_report, + populate_service_date, ) faker = Faker() @@ -206,3 +207,75 @@ def test_process_validation_report_invalid_request( __, status = process_validation_report(request) self.assertEqual(status, 400) create_validation_report_entities_mock.assert_not_called() + + def test_populate_service_date_valid_dates(self): + """Test populate_service_date function with valid date values.""" + dataset = Gtfsdataset( + id=faker.word(), feed_id=faker.word(), stable_id=faker.word(), latest=True + ) + json_report = { + "summary": { + "feedInfo": { + "feedServiceWindowStart": "2024-01-01", + "feedServiceWindowEnd": "2024-12-31", + } + } + } + + populate_service_date(dataset, json_report) + + self.assertEqual(dataset.service_date_range_start, "2024-01-01") + self.assertEqual(dataset.service_date_range_end, "2024-12-31") + + def test_populate_service_date_valid_empty_dates(self): + """Test populate_service_date function.""" + dataset = Gtfsdataset( + id=faker.word(), feed_id=faker.word(), stable_id=faker.word(), latest=True + ) + json_report = { + "summary": { + "feedInfo": { + "feedServiceWindowStart": "", + "feedServiceWindowEnd": "2024-12-31", + } + } + } + populate_service_date(dataset, json_report) + self.assertEqual(dataset.service_date_range_start, None) + self.assertEqual(dataset.service_date_range_end, None) + + json_report = { + "summary": { + "feedInfo": { + "feedServiceWindowStart": "2024-12-31", + "feedServiceWindowEnd": "", + } + } + } + populate_service_date(dataset, json_report) + self.assertEqual(dataset.service_date_range_start, None) + self.assertEqual(dataset.service_date_range_end, None) + + json_report = { + "summary": { + "feedInfo": { + "feedServiceWindowStart": "2024-12-31", + "feedServiceWindowEnd": None, + } + } + } + populate_service_date(dataset, json_report) + self.assertEqual(dataset.service_date_range_start, None) + self.assertEqual(dataset.service_date_range_end, None) + + json_report = { + "summary": { + "feedInfo": { + "feedServiceWindowStart": None, + "feedServiceWindowEnd": "2024-12-31", + } + } + } + populate_service_date(dataset, json_report) + self.assertEqual(dataset.service_date_range_start, None) + self.assertEqual(dataset.service_date_range_end, None)