fix: process validation report fails when service date fields are empty (#978)

Author: davidgamez

functions-python/README.md

@@ -156,6 +156,13 @@ or
 ```
 scripts/api-tests.sh --folder functions-python
 ```
+
+To run the tests and generate the html coverage report:
+```
+scripts/api-tests.sh --folder functions-python --html_report
+```
+_The coverage reports are located in `{project}/scripts/coverage_reports` as individual folder per function._
+
 This will 
 - run the `function-python-setup.sh` script for the function (ie create the `shared` and `test_shared` folders with symlinks) 
 - Create a python virtual environment in the function folder, e.g.: `functions-python/batch_datasets/venv`

functions-python/helpers/tests/test_transform.py

@@ -1,4 +1,4 @@
-from transform import to_boolean
+from transform import to_boolean, get_nested_value
 
 
 def test_to_boolean():
@@ -19,3 +19,44 @@ def test_to_boolean():
     assert to_boolean(None) is False
     assert to_boolean([]) is False
     assert to_boolean({}) is False
+
+
+def test_get_nested_value():
+    # Test case 1: Nested dictionary with string value
+    assert get_nested_value({"a": {"b": {"c": "d"}}}, ["a", "b", "c"]) == "d"
+
+    # Test case 2: Nested dictionary with integer value
+    assert get_nested_value({"a": {"b": {"c": 1}}}, ["a", "b", "c"]) == 1
+
+    # Test case 3: Nested dictionary with float value
+    assert get_nested_value({"a": {"b": {"c": 1.5}}}, ["a", "b", "c"]) == 1.5
+
+    # Test case 4: Nested dictionary with boolean value
+    assert get_nested_value({"a": {"b": {"c": True}}}, ["a", "b", "c"]) is True
+
+    # Test case 5: Nested dictionary with string value that needs trimming
+    assert get_nested_value({"a": {"b": {"c": "  d  "}}}, ["a", "b", "c"]) == "d"
+
+    # Test case 6: Key not found in the dictionary
+    assert get_nested_value({"a": {"b": {"c": "d"}}}, ["a", "b", "x"]) is None
+    assert (
+        get_nested_value({"a": {"b": {"c": "d"}}}, ["a", "b", "x"], "default")
+        == "default"
+    )
+    assert get_nested_value({"a": {"b": {"c": "d"}}}, ["a", "b", "x"], []) == []
+
+    # Test case 7: Intermediate key not found in the dictionary
+    assert get_nested_value({"a": {"b": {"c": "d"}}}, ["a", "x", "c"]) is None
+    assert (
+        get_nested_value({"a": {"b": {"c": "d"}}}, ["a", "x", "c"], "default")
+        == "default"
+    )
+    assert get_nested_value({"a": {"b": {"c": "d"}}}, ["a", "x", "c"], []) == []
+
+    # Test case 8: Empty keys list
+    assert get_nested_value({"a": {"b": {"c": "d"}}}, []) is None
+    assert get_nested_value({"a": {"b": {"c": "d"}}}, [], {}) == {}
+
+    # Test case 9: Non-dictionary data
+    assert get_nested_value("not a dict", ["a", "b", "c"]) is None
+    assert get_nested_value("not a dict", ["a", "b", "c"], []) == []

functions-python/helpers/transform.py

@@ -13,6 +13,7 @@
 #  See the License for the specific language governing permissions and
 #  limitations under the License.
 #
+from typing import List, Optional
 
 
 def to_boolean(value):
@@ -24,3 +25,31 @@ def to_boolean(value):
     if isinstance(value, str):
         return value.lower() in ["true", "1", "yes", "y"]
     return False
+
+
+def get_nested_value(
+    data: dict, keys: List[str], default_value: Optional[any] = None
+) -> Optional[any]:
+    """
+    Retrieve the value from a nested dictionary given a list of keys.
+
+    Args:
+        data (dict): The dictionary to search.
+        keys (List[str]): The list of keys representing the path to the field.
+        default_value: The value to return if the field is not found.
+
+    Returns:
+        Optional[any]: The value if found and valid, otherwise None. The str values are trimmed.
+    """
+    if not keys:
+        return default_value
+    current_data = data
+    for key in keys:
+        if isinstance(current_data, dict) and key in current_data:
+            current_data = current_data[key]
+        else:
+            return default_value
+    if isinstance(current_data, str):
+        result = current_data.strip()
+        return result if result else default_value
+    return current_data

functions-python/process_validation_report/src/main.py

@@ -27,6 +27,7 @@
     Gtfsdataset,
 )
 from shared.helpers.logger import Logger
+from shared.helpers.transform import get_nested_value
 
 logging.basicConfig(level=logging.INFO)
 
@@ -149,20 +150,9 @@ def generate_report_entities(
     dataset = get_dataset(dataset_stable_id, session)
     dataset.validation_reports.append(validation_report_entity)
 
-    if (
-        "summary" in json_report
-        and "feedInfo" in json_report["summary"]
-        and "feedServiceWindowStart" in json_report["summary"]["feedInfo"]
-        and "feedServiceWindowEnd" in json_report["summary"]["feedInfo"]
-    ):
-        dataset.service_date_range_start = json_report["summary"]["feedInfo"][
-            "feedServiceWindowStart"
-        ]
-        dataset.service_date_range_end = json_report["summary"]["feedInfo"][
-            "feedServiceWindowEnd"
-        ]
-
-    for feature_name in json_report["summary"]["gtfsFeatures"]:
+    populate_service_date(dataset, json_report)
+
+    for feature_name in get_nested_value(json_report, ["summary", "gtfsFeatures"], []):
         feature = get_feature(feature_name, session)
         feature.validations.append(validation_report_entity)
         entities.append(feature)
@@ -179,6 +169,23 @@ def generate_report_entities(
     return entities
 
 
+def populate_service_date(dataset, json_report):
+    """
+    Populates the service date range of the dataset based on the JSON report.
+    The service date range is extracted from the feedServiceWindowStart and feedServiceWindowEnd fields,
+     if both are present and not empty.
+    """
+    feed_service_window_start = get_nested_value(
+        json_report, ["summary", "feedInfo", "feedServiceWindowStart"]
+    )
+    feed_service_window_end = get_nested_value(
+        json_report, ["summary", "feedInfo", "feedServiceWindowEnd"]
+    )
+    if feed_service_window_start and feed_service_window_end:
+        dataset.service_date_range_start = feed_service_window_start
+        dataset.service_date_range_end = feed_service_window_end
+
+
 def create_validation_report_entities(feed_stable_id, dataset_stable_id, version):
     """
     Creates and stores entities based on a validation report.

functions-python/process_validation_report/tests/test_validation_report.py

@@ -19,6 +19,7 @@
     get_dataset,
     create_validation_report_entities,
     process_validation_report,
+    populate_service_date,
 )
 
 faker = Faker()
@@ -206,3 +207,75 @@ def test_process_validation_report_invalid_request(
         __, status = process_validation_report(request)
         self.assertEqual(status, 400)
         create_validation_report_entities_mock.assert_not_called()
+
+    def test_populate_service_date_valid_dates(self):
+        """Test populate_service_date function with valid date values."""
+        dataset = Gtfsdataset(
+            id=faker.word(), feed_id=faker.word(), stable_id=faker.word(), latest=True
+        )
+        json_report = {
+            "summary": {
+                "feedInfo": {
+                    "feedServiceWindowStart": "2024-01-01",
+                    "feedServiceWindowEnd": "2024-12-31",
+                }
+            }
+        }
+
+        populate_service_date(dataset, json_report)
+
+        self.assertEqual(dataset.service_date_range_start, "2024-01-01")
+        self.assertEqual(dataset.service_date_range_end, "2024-12-31")
+
+    def test_populate_service_date_valid_empty_dates(self):
+        """Test populate_service_date function."""
+        dataset = Gtfsdataset(
+            id=faker.word(), feed_id=faker.word(), stable_id=faker.word(), latest=True
+        )
+        json_report = {
+            "summary": {
+                "feedInfo": {
+                    "feedServiceWindowStart": "",
+                    "feedServiceWindowEnd": "2024-12-31",
+                }
+            }
+        }
+        populate_service_date(dataset, json_report)
+        self.assertEqual(dataset.service_date_range_start, None)
+        self.assertEqual(dataset.service_date_range_end, None)
+
+        json_report = {
+            "summary": {
+                "feedInfo": {
+                    "feedServiceWindowStart": "2024-12-31",
+                    "feedServiceWindowEnd": "",
+                }
+            }
+        }
+        populate_service_date(dataset, json_report)
+        self.assertEqual(dataset.service_date_range_start, None)
+        self.assertEqual(dataset.service_date_range_end, None)
+
+        json_report = {
+            "summary": {
+                "feedInfo": {
+                    "feedServiceWindowStart": "2024-12-31",
+                    "feedServiceWindowEnd": None,
+                }
+            }
+        }
+        populate_service_date(dataset, json_report)
+        self.assertEqual(dataset.service_date_range_start, None)
+        self.assertEqual(dataset.service_date_range_end, None)
+
+        json_report = {
+            "summary": {
+                "feedInfo": {
+                    "feedServiceWindowStart": None,
+                    "feedServiceWindowEnd": "2024-12-31",
+                }
+            }
+        }
+        populate_service_date(dataset, json_report)
+        self.assertEqual(dataset.service_date_range_start, None)
+        self.assertEqual(dataset.service_date_range_end, None)