CDD-3295: Add in_reporting delay and fix unit tests

Author: tushortz

metrics/api/serializers/charts/dual_category_charts.py

@@ -12,7 +12,9 @@
     DEFAULT_Y_AXIS,
     ChartAxisFields,
     ChartTypes,
+    DataSourceFileType,
     DEFAULT_Y_AXIS_MINIMUM_VAlUE,
+    extract_metric_group_from_metric,
 )
 from metrics.domain.models.charts.dual_category_charts import (
     DualCategoryChartRequestParams,
@@ -67,24 +69,31 @@ def validate(cls, attrs: dict) -> dict:
         """Validate primary_field_values based on the selected x-axis."""
         x_axis = attrs.get("x_axis") or DEFAULT_X_AXIS
         primary_field_values = attrs.get("primary_field_values") or []
+        metric = attrs["static_fields"]["metric"]
+        metric_group = extract_metric_group_from_metric(metric=metric)
+        is_timeseries_data = DataSourceFileType[metric_group].is_timeseries
 
-        # When x-axis is date, primary_field_values should not be provided as the chart will be a timeseries
-        if x_axis == ChartAxisFields.date.name and primary_field_values:
-            raise serializers.ValidationError(
-                {
-                    "primary_field_values": (
-                        "This field should not be provided when x_axis is 'date'."
-                    )
-                }
-            )
+        if is_timeseries_data:
+            if primary_field_values:
+                raise serializers.ValidationError(
+                    {
+                        "primary_field_values": (
+                            "This field should not be provided for timeseries data."
+                        )
+                    }
+                )
+            if x_axis != ChartAxisFields.date.name:
+                raise serializers.ValidationError(
+                    {
+                        "x_axis": (
+                            "This field should be set to 'date' for timeseries data."
+                        )
+                    }
+                )
 
-        if x_axis != ChartAxisFields.date.name and not primary_field_values:
+        elif not is_timeseries_data and not primary_field_values:
             raise serializers.ValidationError(
-                {
-                    "primary_field_values": (
-                        "This field is required when x_axis is not 'date'."
-                    )
-                }
+                {"primary_field_values": ("This field is required for headline data.")}
             )
 
         return attrs

metrics/api/views/charts/dual_category_charts.py

@@ -47,8 +47,7 @@
     "file_format": "svg",
     "chart_height": 200,
     "chart_width": 320,
-    "x_axis": "sex",
-    "primary_field_values": ["m", "f"],
+    "x_axis": "date",
     "y_axis": "metric",
     "x_axis_title": "",
     "y_axis_title": "",

metrics/domain/charts/reporting_delay_period.py

@@ -1,6 +1,6 @@
 import contextlib
 import logging
-from datetime import datetime
+from datetime import date, datetime
 
 import plotly
 from plotly.graph_objs import Scatter
@@ -55,9 +55,13 @@ def _get_last_x_value_at_end_of_reporting_delay_period(
 
 def get_x_value_at_start_of_reporting_delay_period(
     chart_plots_data: list[PlotGenerationData],
-) -> str:
-    index: int = chart_plots_data[0].start_of_reporting_delay_period_index
-    return chart_plots_data[0].x_axis_values[index]
+) -> date:
+    values = [
+        plot.x_axis_values[plot.start_of_reporting_delay_period_index]
+        for plot in chart_plots_data
+    ]
+
+    return min(values)
 
 
 def add_reporting_delay_period(

metrics/domain/charts/stacked_bar/generation.py

@@ -3,6 +3,7 @@
 import plotly.graph_objects as go
 
 from metrics.domain.charts.chart_settings.dual_category import DualCategoryChartSettings
+from metrics.domain.charts.reporting_delay_period import add_reporting_delay_period
 from metrics.domain.models.plots import ChartGenerationPayload
 
 
@@ -20,6 +21,9 @@ def generate_stacked_bar(
         The figure for the stacked bar chart.
     """
     figure = go.Figure()
+    settings = DualCategoryChartSettings(
+        chart_generation_payload=chart_generation_payload,
+    )
 
     grouped: dict[str, dict] = defaultdict(
         lambda: {"x_axis_values": [], "y_axis_values": []}
@@ -41,9 +45,13 @@ def generate_stacked_bar(
             )
         )
 
-    settings = DualCategoryChartSettings(
-        chart_generation_payload=chart_generation_payload,
-    )
-    figure.update_layout(**settings.get_stacked_bar_chart_config())
+    if settings.is_date_type_x_axis:
+        add_reporting_delay_period(
+            chart_plots_data=chart_generation_payload.plots,
+            figure=figure,
+        )
+
+    layout_args = settings.get_stacked_bar_chart_config()
+    figure.update_layout(**layout_args)
 
     return figure

tests/integration/metrics/api/views/charts/test_dual_category_charts.py

@@ -25,14 +25,14 @@ def path(self) -> str:
         ],
     )
     @pytest.mark.django_db
-    def test_returns_correct_response_for_age_based_chart(
+    def test_returns_correct_response_for_timeseries_chart(
         self,
         query_params: dict[str, bool],
         response_header: str,
         admin_user: User,
     ):
         """
-        Given a valid payload to create a chart
+        Given a valid payload to create a timeseries dual-category chart
         When the `POST /api/charts/dual-category/v1/` endpoint is hit
             with the given preview query params
         Then an HTTP 200 OK response is returned
@@ -44,18 +44,17 @@ def test_returns_correct_response_for_age_based_chart(
         static_fields = valid_payload["static_fields"]
 
         for age in ("00-04", "05-11"):
-            for sex in ("m", "f"):
-                CoreTimeSeriesFactory.create_record(
-                    topic_name=static_fields["topic"],
-                    metric_name=static_fields["metric"],
-                    stratum_name=static_fields["stratum"],
-                    age_name=age,
-                    geography_name=static_fields["geography"],
-                    geography_type_name=static_fields["geography_type"],
-                    sex=sex,
-                    date=static_fields["date_from"],
-                    metric_value=100,
-                )
+            CoreTimeSeriesFactory.create_record(
+                topic_name=static_fields["topic"],
+                metric_name=static_fields["metric"],
+                stratum_name=static_fields["stratum"],
+                age_name=age,
+                geography_name=static_fields["geography"],
+                geography_type_name=static_fields["geography_type"],
+                sex=static_fields["sex"],
+                date=static_fields["date_from"],
+                metric_value=100,
+            )
 
         # When
         response: Response = client.post(

tests/unit/metrics/api/serializers/charts/test_dual_category_charts.py

@@ -16,6 +16,12 @@
 from metrics.domain.models.charts.dual_category_charts import (
     DualCategoryChartRequestParams,
 )
+from tests.fakes.factories.metrics.metric_factory import FakeMetricFactory
+from tests.fakes.managers.metric_manager import FakeMetricManager
+from tests.fakes.managers.topic_manager import FakeTopicManager
+
+HEADLINE_METRIC = "COVID-19_headline_vaccines_spring24Uptake"
+TIMESERIES_METRIC = "COVID-19_cases_rateRollingMean"
 
 
 class TestDualCategoryChartSegmentSerializer:
@@ -201,26 +207,26 @@ def test_missing_label_is_still_deemed_valid(self):
 class TestDualCategoryChartSerializer:
     # Success cases
     @pytest.mark.parametrize(
-        "x_axis,primary_field_values",
+        "metric,x_axis,primary_field_values",
         [
-            ("age", ["m", "f"]),
-            # Test that primary_field_values can be omitted when x_axis is date
-            ("date", None),
-            # Test defaulting to date when x_axis is not provided
-            (None, None),
+            (HEADLINE_METRIC, "age", ["m", "f"]),
+            (TIMESERIES_METRIC, "date", None),
+            (TIMESERIES_METRIC, None, None),
         ],
     )
-    def test_validation_with_valid_x_axis_and_primary_field_values_combination(
-        self, x_axis, primary_field_values
+    def test_validation_with_valid_metric_and_primary_field_values_combination(
+        self, metric: str, x_axis: str | None, primary_field_values: list[str] | None
     ):
         """
-        Given a payload containing varying x_axis and primary_field_values
+        Given a payload containing a valid metric and primary_field_values combination
             passed to a `DualCategoryChartSerializer` object
         When `validate()` is called from the serializer
-        Then no ValidationError is raised and the data is returned as these are valid combination of fields
+        Then no ValidationError is raised and the data is returned unchanged
         """
         # Given
         valid_payload = EXAMPLE_DUAL_CATEGORY_CHART_REQUEST_PAYLOAD.copy()
+        valid_payload["static_fields"] = valid_payload["static_fields"].copy()
+        valid_payload["static_fields"]["metric"] = metric
         valid_payload["x_axis"] = x_axis
         valid_payload["primary_field_values"] = primary_field_values
 
@@ -233,16 +239,15 @@ def test_validation_with_valid_x_axis_and_primary_field_values_combination(
         assert is_valid == valid_payload
 
     # Failure cases
-    def test_validation_with_x_axis_and_primary_field_values_present(self):
+    def test_validation_with_primary_field_values_for_timeseries_data(self):
         """
-        Given a payload containing x_axis and primary_field_values
+        Given a payload containing primary_field_values for a timeseries metric
             passed to a `DualCategoryChartSerializer` object
         When `validate()` is called from the serializer
-        Then a `ValidationError` is raised as primary_field_values should not be provided when x_axis value is date
+        Then a `ValidationError` is raised as primary_field_values should not be provided
         """
         # Given
         invalid_payload = EXAMPLE_DUAL_CATEGORY_CHART_REQUEST_PAYLOAD.copy()
-        invalid_payload["x_axis"] = "date"
         invalid_payload["primary_field_values"] = ["m", "f"]
 
         serializer = DualCategoryChartSerializer()
@@ -252,22 +257,43 @@ def test_validation_with_x_axis_and_primary_field_values_present(self):
             serializer.validate(attrs=invalid_payload)
 
         assert exc_info.value.detail["primary_field_values"] == (
-            "This field should not be provided when x_axis is 'date'."
+            "This field should not be provided for timeseries data."
+        )
+
+    def test_validation_with_non_date_x_axis_for_timeseries_data(self):
+        """
+        Given a payload containing a non-date x_axis for a timeseries metric
+            passed to a `DualCategoryChartSerializer` object
+        When `validate()` is called from the serializer
+        Then a `ValidationError` is raised as x_axis must be date for timeseries data
+        """
+        # Given
+        invalid_payload = EXAMPLE_DUAL_CATEGORY_CHART_REQUEST_PAYLOAD.copy()
+        invalid_payload["x_axis"] = "age"
+
+        serializer = DualCategoryChartSerializer()
+
+        # When / Then
+        with pytest.raises(ValidationError) as exc_info:
+            serializer.validate(attrs=invalid_payload)
+
+        assert exc_info.value.detail["x_axis"] == (
+            "This field should be set to 'date' for timeseries data."
         )
 
-    def test_validation_with_primary_field_values_missing_when_x_axis_is_not_date(self):
+    def test_validation_with_primary_field_values_missing_for_headline_data(self):
         """
-        Given a payload containing x_axis but no primary_field_values
+        Given a payload containing a headline metric but no primary_field_values
             passed to a `DualCategoryChartSerializer` object
         When `validate()` is called from the serializer
-        Then a `ValidationError` is raised as primary_field_values should be provided when x_axis value is not date
+        Then a `ValidationError` is raised as primary_field_values are required
         """
         # Given
         invalid_payload = EXAMPLE_DUAL_CATEGORY_CHART_REQUEST_PAYLOAD.copy()
+        invalid_payload["static_fields"] = invalid_payload["static_fields"].copy()
+        invalid_payload["static_fields"]["metric"] = HEADLINE_METRIC
         invalid_payload["x_axis"] = "age"
-        invalid_payload.pop(
-            "primary_field_values"
-        )  # Remove primary_field_values to simulate it being missing
+        invalid_payload.pop("primary_field_values", None)
 
         serializer = DualCategoryChartSerializer()
 
@@ -276,7 +302,7 @@ def test_validation_with_primary_field_values_missing_when_x_axis_is_not_date(se
             serializer.validate(attrs=invalid_payload)
 
         assert exc_info.value.detail["primary_field_values"] == (
-            "This field is required when x_axis is not 'date'."
+            "This field is required for headline data."
         )
 
     def test_to_models_builds_timeseries_plots_when_x_axis_is_date(
@@ -330,3 +356,59 @@ def test_to_models_builds_timeseries_plots_when_x_axis_is_date(
             assert plot.metric == static_fields["metric"]
             assert plot.date_from == static_fields["date_from"]
             assert plot.date_to == static_fields["date_to"]
+
+    def test_to_models_builds_headline_plots_with_primary_field_values(self):
+        """
+        Given a valid payload with a headline metric and primary field values
+        When `to_models()` is called from the serializer
+        Then one plot per segment is returned for each primary field value
+        """
+        # Given
+        fake_metric = FakeMetricFactory.build_example_metric(
+            metric_name=HEADLINE_METRIC,
+            metric_group_name="headline",
+        )
+        fake_topic = fake_metric.metric_group.topic
+        metric_manager = FakeMetricManager([fake_metric])
+        topic_manager = FakeTopicManager([fake_topic])
+        valid_payload = EXAMPLE_DUAL_CATEGORY_CHART_REQUEST_PAYLOAD.copy()
+        valid_payload["static_fields"] = valid_payload["static_fields"].copy()
+        valid_payload["static_fields"]["topic"] = fake_topic.name
+        valid_payload["static_fields"]["metric"] = HEADLINE_METRIC
+        valid_payload["x_axis"] = ChartAxisFields.sex.name
+        valid_payload["primary_field_values"] = ["m", "f"]
+
+        serializer_context = {
+            "topic_manager": topic_manager,
+            "metric_manager": metric_manager,
+        }
+        serializer = DualCategoryChartSerializer(
+            data=valid_payload,
+            context=serializer_context,
+        )
+        serializer.fields["static_fields"] = PlotSerializer(context=serializer_context)
+        serializer.is_valid(raise_exception=True)
+
+        # When
+        result: DualCategoryChartRequestParams = serializer.to_models(request=None)
+
+        # Then
+        segments = valid_payload["segments"]
+        secondary_category = valid_payload["secondary_category"]
+        x_axis = valid_payload["x_axis"]
+
+        assert result.primary_field_values == ["m", "f"]
+        assert len(result.plots) == len(segments) * len(
+            valid_payload["primary_field_values"]
+        )
+
+        for index, plot in enumerate(result.plots):
+            segment_index = index % len(segments)
+            primary_index = index // len(segments)
+            segment = segments[segment_index]
+            primary_field_value = valid_payload["primary_field_values"][primary_index]
+
+            assert plot.x_axis == x_axis
+            assert getattr(plot, x_axis) == primary_field_value
+            assert getattr(plot, secondary_category) == segment["secondary_field_value"]
+            assert plot.line_colour == segment["colour"]