Merge pull request #283 from American-Institutes-for-Research/HEA-1072/denormalize_annual_kcals_cost

Fix slow annual_kcals_cost - see HEA-1072

Author: rhunwicks

.github/workflows/01-build-then-test.yml

@@ -30,7 +30,7 @@ env:
 jobs:
   lint:
     runs-on: [ fewsnet ]
-    container: "python:3.11"
+    container: "python:3.12"
 
     steps:
       - uses: "actions/checkout@v4"

apps/baseline/apps.py

@@ -6,3 +6,6 @@ class BaselineConfig(AppConfig):
     default_auto_field = "django.db.models.BigAutoField"
     name = "baseline"
     verbose_name = _("baseline")
+
+    def ready(self):
+        import baseline.signals  # noqa: F401

apps/baseline/migrations/0034_livelihoodzonebaseline__annual_kcals_cost.py

@@ -0,0 +1,38 @@
+# Generated by Django 5.2.11 on 2026-05-03 14:30
+
+from django.db import migrations, models
+
+
+def populate_annual_kcals_cost(apps, schema_editor):
+    schema_editor.execute("""
+        UPDATE baseline_livelihoodzonebaseline
+        SET _annual_kcals_cost = get_annual_kcals_cost(id, 'P')
+        """)
+
+
+def clear_annual_kcals_cost(apps, schema_editor):
+    schema_editor.execute("""
+        UPDATE baseline_livelihoodzonebaseline
+        SET _annual_kcals_cost = NULL
+        """)
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("baseline", "0033_add_annual_kcals_cost_function"),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name="livelihoodzonebaseline",
+            name="_annual_kcals_cost",
+            field=models.FloatField(
+                blank=True,
+                help_text="Annual cost per person of 100% of recommended kcals in the baseline currency.",
+                null=True,
+                verbose_name="Annual kcals cost",
+            ),
+        ),
+        migrations.RunPython(populate_annual_kcals_cost, clear_annual_kcals_cost),
+    ]

apps/baseline/models.py

@@ -348,22 +348,23 @@ class Language(models.TextChoices):
         verbose_name=_("Currency"),
         help_text=_("Default currency for income or expenditure from Livelihood Activities within this Baseline."),
     )
+    # Derived field - always set by calculate_fields()
+    _annual_kcals_cost = models.FloatField(
+        blank=True,
+        null=True,
+        verbose_name=_("Annual kcals cost"),
+        help_text=_("Annual cost per person of 100% of recommended kcals in the baseline currency."),
+    )
     objects = LivelihoodZoneBaselineManager()
 
-    def _get_annual_kcals_cost(self):
+    def _get_annual_kcals_cost_sql(self):
         """
         Calculate the annual cost per person of 100% of recommended kcals.
 
-        This is required to calculate the combined food and income as kcals.
-
-        It is defined as the cost of providing 100% of required kcals for the P
-        Wealth Group divided by the average_household_size. The cost is
-        calculated as the cost of Basket 2 (Other food survival) plus the cost
-        of providing the remainder of kcals by purchasing the main staple. All
-        costs are in the currency for the BSS.
-
-        This calculation is based on the formulae in the Graphs worksheet.
+        This method uses a stored SQL function and is used for troubleshooting.
         """
+        if not self.pk:
+            return None
         return (
             type(self)
             .objects.filter(pk=self.pk)
@@ -377,10 +378,22 @@ def _get_annual_kcals_cost(self):
             .get()
         )
 
-    def _get_annual_kcals_cost_python(self):
+    def _get_annual_kcals_cost(self):
         """
-        A Python implementation of the annual_kcals_cost calculation, used for troubleshooting.
+        Calculate the annual cost per person of 100% of recommended kcals.
+
+        This is required to calculate the combined food and income as kcals.
+
+        It is defined as the cost of providing 100% of required kcals for the P
+        Wealth Group divided by the average_household_size. The cost is
+        calculated as the cost of Basket 2 (Other food survival) plus the cost
+        of providing the remainder of kcals by purchasing the main staple. All
+        costs are in the currency for the BSS.
+
+        This calculation is based on the formulae in the Graphs worksheet.
         """
+        if not self.pk:
+            return None
         poor_main_staple = LivelihoodProductCategory.objects.filter(
             basket=LivelihoodProductCategory.ProductBasket.MAIN_STAPLE,
             baseline_livelihood_activity__wealth_group__livelihood_zone_baseline=self,
@@ -400,6 +413,10 @@ def _get_annual_kcals_cost_python(self):
                 )
             )
         poor_main_staple = poor_main_staple[0]
+        poor_household_size = poor_main_staple.baseline_livelihood_activity.wealth_group.average_household_size
+        if not poor_household_size:
+            # Cannot calculate without household size
+            return None
         poor_other_food = LivelihoodProductCategory.objects.filter(
             basket=LivelihoodProductCategory.ProductBasket.SURVIVAL_OTHER_FOOD,
             baseline_livelihood_activity__wealth_group__livelihood_zone_baseline=self,
@@ -412,12 +429,11 @@ def _get_annual_kcals_cost_python(self):
                 F("baseline_livelihood_activity__expenditure") * F("percentage_allocation_to_basket")
             ),
         )
-        poor_household_size = poor_main_staple.baseline_livelihood_activity.wealth_group.average_household_size
         main_staple_kcals_per_unit = poor_main_staple.baseline_livelihood_activity.extra.get(
             "product__kcals_per_unit",
             poor_main_staple.baseline_livelihood_activity.livelihood_strategy.product.kcals_per_unit,
         )
-        main_staple_percentage_kcals_required = 1 - poor_other_food["total_percentage_kcals"]
+        main_staple_percentage_kcals_required = 1 - (poor_other_food["total_percentage_kcals"] or 0)
         main_staple_cost = (
             2100  # kcals per person per day
             * 365  # days per year
@@ -426,17 +442,29 @@ def _get_annual_kcals_cost_python(self):
             / main_staple_kcals_per_unit
             * poor_main_staple.baseline_livelihood_activity.price
         )
-        total_cost = main_staple_cost + poor_other_food["total_expenditure"]
+        total_cost = main_staple_cost + (poor_other_food["total_expenditure"] or 0)
         total_food_cost_per_person = total_cost / poor_household_size
         return total_food_cost_per_person
 
-    def get_annual_kcals_cost(self):
-        key = f"livelihood_zone_baseline~{self.pk}~annual_kcals_cost"
-        annual_kcals_cost = cache.get(key)
-        if not annual_kcals_cost:
-            annual_kcals_cost = self._get_annual_kcals_cost()
-            cache.set(key, annual_kcals_cost, 60 * 60 * 24)
-        return annual_kcals_cost
+    @property
+    def annual_kcals_cost(self):
+        return self._annual_kcals_cost
+
+    def calculate_fields(self):
+        self._annual_kcals_cost = self._get_annual_kcals_cost()
+
+    def save(self, *args, **kwargs):
+        self.calculate_fields()
+        self.full_clean(
+            exclude=[field.name for field in self._meta.fields if isinstance(field, models.ForeignKey)],
+            validate_unique=False,
+        )
+
+        # Make sure that _annual_kcals_cost is included in the update_fields if the update_fields argument is provided.
+        update_fields = kwargs.get("update_fields")
+        if update_fields is not None:
+            kwargs["update_fields"] = set(update_fields) | {"_annual_kcals_cost"}
+        super().save(*args, **kwargs)
 
     def natural_key(self):
         try:
@@ -734,7 +762,7 @@ class WealthGroup(common_models.Model):
 
     @cached_property
     def household_annual_kcals_cost(self):
-        annual_kcals_cost = self.livelihood_zone_baseline.get_annual_kcals_cost()
+        annual_kcals_cost = self.livelihood_zone_baseline.annual_kcals_cost
         if annual_kcals_cost and self.average_household_size:
             return annual_kcals_cost * self.average_household_size
 
@@ -759,7 +787,7 @@ def _get_survival_threshold_as_percentage_kcals(self):
         This basket is defined for the Poor Wealth Group, and other values use
         the same expenditure, scaled according to average household size.
         """
-        annual_kcals_cost = self.livelihood_zone_baseline.get_annual_kcals_cost()
+        annual_kcals_cost = self.livelihood_zone_baseline.annual_kcals_cost
         poor_survival_non_food = (
             LivelihoodProductCategory.objects.filter(
                 basket=LivelihoodProductCategory.ProductBasket.SURVIVAL_NON_FOOD,
@@ -804,7 +832,7 @@ def survival_threshold_as_percentage_kcals(self):
 
     @cached_property
     def survival_threshold_as_cash(self):
-        annual_kcals_cost = self.livelihood_zone_baseline.get_annual_kcals_cost()
+        annual_kcals_cost = self.livelihood_zone_baseline.annual_kcals_cost
         if annual_kcals_cost and self.survival_threshold_as_percentage_kcals and self.average_household_size:
             return self.survival_threshold_as_percentage_kcals * self.average_household_size * annual_kcals_cost
 
@@ -847,7 +875,7 @@ def _get_livelihoods_protection_threshold_as_percentage_kcals(self):
         Livelihoods Protection Basket then the amount is inherited from the Poor wealth group,
         rather than based on typical consumption for the Wealth Group in the Reference Year.
         """
-        annual_kcals_cost = self.livelihood_zone_baseline.get_annual_kcals_cost()
+        annual_kcals_cost = self.livelihood_zone_baseline.annual_kcals_cost
 
         livelihood_protection_qs = LivelihoodProductCategory.objects.filter(
             basket=LivelihoodProductCategory.ProductBasket.LIVELIHOODS_PROTECTION,
@@ -981,7 +1009,7 @@ def livelihoods_protection_threshold_as_percentage_kcals(self):
 
     @cached_property
     def livelihoods_protection_threshold_as_cash(self):
-        annual_kcals_cost = self.livelihood_zone_baseline.get_annual_kcals_cost()
+        annual_kcals_cost = self.livelihood_zone_baseline.annual_kcals_cost
         if (
             annual_kcals_cost
             and self.livelihoods_protection_threshold_as_percentage_kcals
@@ -1754,7 +1782,7 @@ def total_income_as_percentage_kcals(self):
         """
         The total food consumed and income received as percentage of required household kcals.
         """
-        annual_kcals_cost = self.livelihood_zone_baseline.get_annual_kcals_cost()
+        annual_kcals_cost = self.livelihood_zone_baseline.annual_kcals_cost
         percentage_kcals = self.percentage_kcals or 0
         income = self.income or 0
         if self.wealth_group.average_household_size:
@@ -1769,7 +1797,7 @@ def total_income_as_cash(self):
         """
         The total food consumed and income received as an amount in the BSS currency.
         """
-        annual_kcals_cost = self.livelihood_zone_baseline.get_annual_kcals_cost()
+        annual_kcals_cost = self.livelihood_zone_baseline.annual_kcals_cost
         percentage_kcals = self.percentage_kcals or 0
         income = self.income or 0
         if self.wealth_group.average_household_size:
@@ -1854,15 +1882,17 @@ def validate_expenditure(self):
 
         However, some LivelihoodActivity subclasses, such as FoodPurchase and
         OtherPurchase, involve spending money to acquire the item, in which
-        case we must validate that expenditure = quantity_produced * price
+        case we must validate that expenditure = quantity_purchased * price
         """
-        quantity_produced = self.quantity_produced or 0
+        quantity_produced_or_purchased = self.quantity_produced or self.quantity_purchased or 0
         price = self.price or 0
         expenditure = self.expenditure or 0
 
-        if self.expenditure and not math.isclose(expenditure, quantity_produced * price):
+        if self.expenditure and not math.isclose(expenditure, quantity_produced_or_purchased * price):
             raise ValidationError(
-                _("Expenditure for a Livelihood Activity must be quantity produced multiplied by price")
+                _(
+                    "Expenditure for a Livelihood Activity must be quantity produced or quantity purchased multiplied by price"
+                )
             )
 
     def validate_kcals_consumed(self):

apps/baseline/serializers.py

@@ -1,14 +1,12 @@
-from django.db.models import F, FloatField, Sum, Value
+from django.db.models import F, FloatField, Sum
 from django.db.models.functions import Coalesce
 from rest_framework import serializers
 from rest_framework_gis.serializers import GeoFeatureModelSerializer
 
 from common.fields import translation_fields
 from common.serializers import AggregatingSerializer
-from metadata.models import WealthGroupCategory
 
 from .models import (
-    AnnualKcalsCost,
     BaselineLivelihoodActivity,
     BaselineWealthGroup,
     BaselineWealthGroupCharacteristicValue,
@@ -77,6 +75,8 @@ class Meta:
 
 
 class LivelihoodZoneBaselineSerializer(serializers.ModelSerializer):
+    annual_kcals_cost = serializers.FloatField(read_only=True)
+
     class Meta:
         model = LivelihoodZoneBaseline
         fields = (
@@ -100,6 +100,7 @@ class Meta:
             "valid_to_date",
             "population_source",
             "population_estimate",
+            "annual_kcals_cost",
         )
 
     livelihood_zone_name = serializers.CharField(source="livelihood_zone.name", read_only=True)
@@ -113,6 +114,8 @@ def get_bss_language(self, obj):
 
 
 class LivelihoodZoneBaselineGeoSerializer(GeoFeatureModelSerializer):
+    annual_kcals_cost = serializers.FloatField(read_only=True)
+
     class Meta:
         model = LivelihoodZoneBaseline
         fields = (
@@ -137,6 +140,7 @@ class Meta:
             "valid_to_date",
             "population_source",
             "population_estimate",
+            "annual_kcals_cost",
         )
         geo_field = "geography"
         auto_bbox = True
@@ -1721,9 +1725,7 @@ class Meta:
                     Coalesce(F("income"), 0)
                     / (
                         F("wealth_group__average_household_size")
-                        * AnnualKcalsCost(
-                            F("wealth_group__livelihood_zone_baseline_id"), Value(WealthGroupCategory.POOR)
-                        )
+                        * F("wealth_group__livelihood_zone_baseline___annual_kcals_cost")
                     )
                 )
             ),
@@ -1734,7 +1736,7 @@ class Meta:
                 (
                     Coalesce(F("percentage_kcals"), 0)
                     * F("wealth_group__average_household_size")
-                    * AnnualKcalsCost(F("wealth_group__livelihood_zone_baseline_id"), Value(WealthGroupCategory.POOR))
+                    * F("wealth_group__livelihood_zone_baseline___annual_kcals_cost")
                 )
                 + Coalesce(F("income"), 0)
             ),