Merge pull request #229 from American-Institutes-for-Research/HEA-898/aide_alimentaire

Hea 898/aide alimentaire

Author: rhunwicks

apps/baseline/migrations/0025_alter_milkproduction_milking_animals.py

@@ -0,0 +1,18 @@
+# Generated by Django 5.2.10 on 2026-02-01 19:02
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("baseline", "0024_baselinewealthgroupcharacteristicvalue_and_more"),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name="milkproduction",
+            name="milking_animals",
+            field=models.PositiveSmallIntegerField(blank=True, null=True, verbose_name="Number of milking animals"),
+        ),
+    ]

apps/baseline/models.py

@@ -1012,8 +1012,12 @@ def clean(self):
         """
         if self.strategy_type in self.REQUIRES_PRODUCT and not self.product:
             raise ValidationError(_("A %s Livelihood Strategy must have a Product" % self.strategy_type))
-        if self.strategy_type in ["MilkProduction", "ButterProduction"] and not self.season:
+        if self.strategy_type in self.REQUIRES_SEASON and not self.season:
             raise ValidationError(_("A %s Livelihood Strategy must have a Season" % self.strategy_type))
+        # All strategies require either a product or an additional identifier to distinguish them
+        if not self.product and not self.additional_identifier:
+            raise ValidationError(_("A Livelihood Strategy must have either a Product or an Additional Identifier"))
+
         super().clean()
 
     def save(self, *args, **kwargs):
@@ -1239,6 +1243,18 @@ def validate_quantity_produced(self):
         """
         pass
 
+    def validate_quantity_purchased(self):
+        """
+        Validate the quantity_purchased.
+
+        In most LivelihoodActivity subclasses the quantity_purchased is not used
+        and this validation passes.
+
+        However, FoodPurchase has additional fields that allow the quantity_purchased
+        to be validated. This method is overwritten in that subclass.
+        """
+        pass
+
     def validate_quantity_consumed(self):
         # Default to 0 if any of the quantities are None
         quantity_produced = self.quantity_produced or 0
@@ -1347,6 +1363,7 @@ def clean(self):
         self.validate_livelihood_zone_baseline()
         self.validate_strategy_type()
         self.validate_quantity_produced()
+        self.validate_quantity_purchased()
         self.validate_quantity_consumed()
         self.validate_income()
         self.validate_expenditure()
@@ -1505,7 +1522,12 @@ class MilkType(models.TextChoices):
         WHOLE = "whole", _("whole")
 
     # Production calculation /validation is `lactation days * daily_production`
-    milking_animals = models.PositiveSmallIntegerField(verbose_name=_("Number of milking animals"))
+    # Although logically we can't have a MilkProduction without milking animals,
+    # the BSS may contain records with 0 production and a blank milking_animals
+    # field, particularly in Season 2 in a Baseline, or in a Response scenario
+    milking_animals = models.PositiveSmallIntegerField(
+        blank=True, null=True, verbose_name=_("Number of milking animals")
+    )
     lactation_days = models.PositiveSmallIntegerField(
         blank=True, null=True, verbose_name=_("Average number of days of lactation")
     )
@@ -1551,9 +1573,13 @@ def validate_quantity_produced(self):
 
     def clean(self):
         super().clean()
-        if self.milking_animals and not self.lactation_days:
+        # Note that we check that lactation_days and daily_production are not None
+        # because 0 is a valid value for both fields.  It is possible to have a
+        # non-zero milking_animals but zero lactation_days or daily_production,
+        # particularly in Season 2 in a Baseline, or in a Response scenario.
+        if self.milking_animals and self.lactation_days is None:
             raise ValidationError(_("Lactation days must be provided if there are milking animals"))
-        if self.milking_animals and not self.daily_production:
+        if self.milking_animals and self.daily_production is None:
             raise ValidationError(_("Daily production must be provided if there are milking animals"))
 
     class Meta:
@@ -1618,7 +1644,7 @@ class MeatProduction(LivelihoodActivity):
     carcass_weight = models.FloatField(verbose_name=_("Carcass weight per animal"))
 
     def validate_quantity_produced(self):
-        if self.quantity_produced != self.animals_slaughtered * self.carcass_weight:
+        if self.quantity_produced and self.quantity_produced != self.animals_slaughtered * self.carcass_weight:
             raise ValidationError(
                 _("Quantity Produced for a Meat Production must be animals slaughtered multiplied by carcass weight")
             )
@@ -1691,20 +1717,28 @@ class FoodPurchase(LivelihoodActivity):
         help_text=_("Number of times in a year that the purchase is made"),
     )
 
-    def validate_quantity_produced(self):
+    def validate_quantity_purchased(self):
         if (
-            self.quantity_produced is not None
+            self.quantity_purchased is not None
             and self.unit_multiple is not None
             and self.times_per_month is not None
             and self.months_per_year is not None
         ):
-            if self.quantity_produced != self.unit_multiple * self.times_per_month * self.months_per_year:
+            if self.quantity_purchased != self.unit_multiple * self.times_per_month * self.months_per_year:
                 raise ValidationError(
                     _(
-                        "Quantity produced for a Food Purchase must be purchase amount * purchases per month * months per year"  # NOQA: E501
+                        "Quantity purchased for a Food Purchase must be purchase amount * purchases per month * months per year"  # NOQA: E501
                     )
                 )
 
+    def validate_expenditure(self):
+        quantity_purchased = self.quantity_purchased or 0
+        price = self.price or 0
+        expenditure = self.expenditure or 0
+
+        if self.expenditure and expenditure != quantity_purchased * price:
+            raise ValidationError(_("Expenditure for a Food Purchase must be quantity purchased multiplied by price"))
+
     class Meta:
         verbose_name = LivelihoodStrategyType.FOOD_PURCHASE.label
         verbose_name_plural = _("Food Purchases")

apps/baseline/tests/test_models.py

@@ -135,33 +135,33 @@ def setUpTestData(cls):
             unit_multiple=None,
             times_per_month=None,
             months_per_year=None,
-            quantity_produced=None,
+            quantity_purchased=None,
         )
         cls.foodpurchase2 = FoodPurchase(
             unit_multiple=2,
             times_per_month=5,
             months_per_year=12,
-            quantity_produced=120,
+            quantity_purchased=120,
         )
         # Incorrect: 2 * 5 * 12 = 120
         cls.foodpurchase3 = FoodPurchase(
             unit_multiple=2,
             times_per_month=5,
             months_per_year=12,
-            quantity_produced=100,
+            quantity_purchased=100,
         )
 
-    def test_validate_quantity_produced(self):
+    def test_validate_quantity_purchased(self):
         """
-        Test validate_quantity_produced method
+        Test validate_quantity_purchased method
         """
         # Missing data should not raise ValidationError
-        self.foodpurchase1.validate_quantity_produced()
+        self.foodpurchase1.validate_quantity_purchased()
         # Expected consistant values, should not raise
-        self.foodpurchase2.validate_quantity_produced()
+        self.foodpurchase2.validate_quantity_purchased()
         # Incorrect: 2 * 5 * 12 = 120
         with conditional_logging():
-            self.assertRaises(ValidationError, self.foodpurchase3.validate_quantity_produced)
+            self.assertRaises(ValidationError, self.foodpurchase3.validate_quantity_purchased)
 
 
 class PaymentInKindTestCase(TestCase):

apps/common/lookups.py

@@ -75,13 +75,14 @@ class Lookup(ABC):
     # where an Excel date or other epoch-based date format might accidentally match a primary key
     match_pk: bool = True
 
-    # Queryset Filters
+    # Queryset Filters and Excludes
     filters: dict
+    excludes: dict
 
     # Related models to also instantiate
     related_models = []
 
-    def __init__(self, filters: dict = None, require_match=None):
+    def __init__(self, filters: dict = None, excludes: dict = None, require_match=None):
         # Make sure we don't have any fields in multiple categories
         assert len(set((*self.id_fields, *self.parent_fields, *self.lookup_fields))) == len(self.id_fields) + len(
             self.parent_fields
@@ -90,11 +91,17 @@ def __init__(self, filters: dict = None, require_match=None):
         self.composite_key = len(self.id_fields) > 1
 
         self.filters = filters or dict()
+        self.excludes = excludes or dict()
 
         # Override the require_match if necessary
         if require_match is not None:
             self.require_match = require_match
 
+        # Create instance level caches for key methods
+        self.get_lookup_df = functools.cache(self.get_lookup_df)
+        self.get = functools.cache(self.get)
+        self.get_instance = functools.cache(self.get_instance)
+
     def get_queryset_columns(self):
         return [*self.lookup_fields, *self.parent_fields, *self.id_fields]
 
@@ -108,18 +115,21 @@ def get_queryset(self):
         It uses values_list to avoid hydrating a Django model, given that the result will be used
         to instantiate a DataFrame.
         """
-        queryset = self.model.objects.filter(**self.filters).values_list(*self.get_queryset_columns())
+        queryset = (
+            self.model.objects.filter(**self.filters)
+            .exclude(**self.excludes)
+            .values_list(*self.get_queryset_columns())
+        )
         return queryset
 
-    @functools.cache
     def get_lookup_df(self):
         """
         Build a dataframe for a model that can be used to lookup the primary key.
 
         Create a dataframe that contains the primary key, and all the columns that
         can be used to lookup the primary key, e.g. the name, description, aliases, etc.
 
-        Use the queryset.iterator() to prevent Django from caching the queryset, and instead use functools.cache to
+        Use the queryset.iterator() to prevent Django from caching the queryset, because functools.cache is used to
         cache the dataframe returned from this function.
         """
         df = pd.DataFrame(list(self.get_queryset().iterator()), columns=self.get_queryset_columns())
@@ -322,14 +332,13 @@ def get_instances(self, df, column, related_models=None):
         df[column] = df[column].map(model_map)
         return df
 
-    @functools.cache
     def get(self, value: str, **parent_values) -> str | None:
         """
         Return the lookup value for a single string, or None if there is no match.
 
         Used to do a lookup for a single value instead of a dataframe.
 
-        Unlike `do_lookup()` this is a cached property to avoid repeated database queries.
+        Unlike `do_lookup()` this is a cached method to avoid repeated database queries.
         Note that this cache is per-instance of the Lookup class, so the class should be instantiated outside a loop
         performing repeated lookups:
 
@@ -347,14 +356,13 @@ def get(self, value: str, **parent_values) -> str | None:
         except ValueError:
             return None
 
-    @functools.cache
     def get_instance(self, value: str, **parent_values) -> Model | None:
         """
         Return the Django model instance for a single string, or None if there is no match.
 
         Used to do a lookup for a single value instead of a dataframe.
 
-        Unlike `do_lookup()` this is a cached property to avoid repeated database queries.
+        Unlike `do_lookup()` this is a cached method to avoid repeated database queries.
         Note that this cache is per-instance of the Lookup class, so the class should be instantiated outside a loop
         performing repeated lookups:
 
@@ -444,6 +452,14 @@ class ClassifiedProductLookup(Lookup):
         "hs2012",
     )
 
+    def get_queryset(self):
+        """
+        Exclude specific products that are unwanted duplicates.
+        """
+        # P23162 is Husked Rice, but we use "rice" as an alias for it. This
+        # conflicts with R0113: Rice, which we never use, so ignore it.
+        return super().get_queryset().exclude(cpc="R0113")
+
     def get_lookup_df(self):
         """
         Build a dataframe for a model that can be used to lookup the primary key.

apps/common/tests/test_lookups.py

@@ -40,6 +40,21 @@ def test_ignore_unwanted_parents(self):
         self.assertEqual(len(result_df), 1)
         self.assertEqual(result_df["cpc"][0], product.pk)
 
+    def test_excludes_r0113(self):
+        # Create the unwanted product R0113 with a matching common name
+        ClassifiedProductFactory(cpc="R0113", common_name_en="Rice", description_en="Rice")
+        # Create the preferred product that we want the lookup to return, and include 'rice' as an alias
+        preferred = ClassifiedProductFactory(cpc="P23162", common_name_en="Husked Rice", description_en="Husked Rice")
+        preferred.aliases = ["arroz", "rice", "riz"]
+        preferred.save()
+
+        df = pd.DataFrame({"product": ["rice"]})
+        result_df = ClassifiedProductLookup().do_lookup(df, "product", "cpc")
+        self.assertTrue("cpc" in result_df.columns)
+        self.assertEqual(len(result_df), 1)
+        # The lookup should return the preferred product, not the unwanted R0113
+        self.assertEqual(result_df["cpc"][0], preferred.pk)
+
         # The child record doesn't could have the search term in a the common name instead of the description,
         # and the child doesn't need to be first child (with a 0 suffix). The child could have its own children,
         # that don't match the search term.