Skip to content

Commit f3c3983

Browse files
rhunwicksrhunwicks
committed
Support Override and Regular Expression statuses for ActivityLabel - see DATA-4695
1 parent 1fab908 commit f3c3983

5 files changed

Lines changed: 132 additions & 14 deletions

File tree

‎apps/metadata/migrations/0010_alter_activitylabel_status_and_more.py‎

Lines changed: 54 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,54 @@
1+
# Generated by Django 5.1.12 on 2025-09-17 16:39
2+
3+
from django.db import migrations, models
4+
5+
6+
def forwards(apps, schema_editor):
7+
ActivityLabel = apps.get_model("metadata", "ActivityLabel")
8+
ActivityLabel.objects.filter(status__in=["Complete"]).update(status="Override")
9+
10+
11+
def backwards(apps, schema_editor):
12+
ActivityLabel = apps.get_model("metadata", "ActivityLabel")
13+
ActivityLabel.objects.filter(status__in=["Override"]).update(status="Complete")
14+
15+
16+
class Migration(migrations.Migration):
17+
18+
dependencies = [
19+
("metadata", "0009_livelihoodcategory_color"),
20+
]
21+
22+
operations = [
23+
migrations.AlterField(
24+
model_name="activitylabel",
25+
name="status",
26+
field=models.CharField(
27+
blank=True,
28+
choices=[
29+
("Regular Expression", "Processed by Regular Expression"),
30+
("Override", "Override automatically recognized metadata"),
31+
("Discussion", "Under Discussion"),
32+
("Correct BSS", "Correct the BSS"),
33+
],
34+
max_length=20,
35+
verbose_name="Status",
36+
),
37+
),
38+
migrations.AlterField(
39+
model_name="wealthcharacteristiclabel",
40+
name="status",
41+
field=models.CharField(
42+
blank=True,
43+
choices=[
44+
("Complete", "Complete"),
45+
("Discussion", "Under Discussion"),
46+
("Correct BSS", "Correct the BSS"),
47+
],
48+
max_length=20,
49+
verbose_name="Status",
50+
),
51+
),
52+
# Change existing "Complete" ActivityLabel statuses to "Override"
53+
migrations.RunPython(forwards, backwards),
54+
]

‎apps/metadata/models.py‎

Lines changed: 11 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -380,13 +380,6 @@ class Meta:
380380
verbose_name_plural = _("Seasons")
381381

382382

383-
# Defined outside ActivityLabel to make it easy to share between ActivityLabel and WealthCharacteristicLabel
384-
class LabelStatus(models.TextChoices):
385-
COMPLETE = "Complete", _("Complete")
386-
DISCUSSION = "Discussion", _("Under Discussion")
387-
CORRECT_BSS = "Correct BSS", _("Correct the BSS")
388-
389-
390383
class ActivityLabel(common_models.Model):
391384
"""
392385
A label from Column A of the 'Data', 'Data2' or 'Data3' worksheet in a BSS and associated attributes.
@@ -395,6 +388,12 @@ class ActivityLabel(common_models.Model):
395388
the LivelihoodStrategy and/or LivelihoodActivity for a given row in a BSS.
396389
"""
397390

391+
class LabelStatus(models.TextChoices):
392+
REGULAR_EXPRESSION = "Regular Expression", _("Processed by Regular Expression")
393+
OVERRIDE = "Override", _("Override automatically recognized metadata")
394+
DISCUSSION = "Discussion", _("Under Discussion")
395+
CORRECT_BSS = "Correct BSS", _("Correct the BSS")
396+
398397
class LivelihoodActivityType(models.TextChoices):
399398
LIVELIHOOD_ACTIVITY = "LivelihoodActivity", _("Livelihood Activity") # Labels from the 'Data' worksheet
400399
OTHER_CASH_INCOME = "OtherCashIncome", _("Other Cash Income") # Labels from the 'Data2' worksheet
@@ -476,6 +475,11 @@ class WealthCharacteristicLabel(common_models.Model):
476475
Used by the ingestion pipeline for WealthCharacteristicValue to determine the attributes for a given row in a BSS.
477476
"""
478477

478+
class LabelStatus(models.TextChoices):
479+
COMPLETE = "Complete", _("Complete")
480+
DISCUSSION = "Discussion", _("Under Discussion")
481+
CORRECT_BSS = "Correct BSS", _("Correct the BSS")
482+
479483
wealth_characteristic_label = common_models.NameField(
480484
max_length=200, unique=True, verbose_name=_("Wealth Characteristic Label")
481485
)

‎pipelines/assets/livelihood_activity.py‎

Lines changed: 5 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -86,7 +86,6 @@
8686
from metadata.lookups import SeasonNameLookup # NOQA: E402
8787
from metadata.models import ( # NOQA: E402
8888
ActivityLabel,
89-
LabelStatus,
9089
LivelihoodActivityScenario,
9190
LivelihoodStrategyType,
9291
)
@@ -206,7 +205,9 @@ def get_livelihood_activity_label_map(activity_type: str) -> dict[str, dict]:
206205
"""
207206
label_map = {
208207
instance["activity_label"].lower(): instance
209-
for instance in ActivityLabel.objects.filter(status=LabelStatus.COMPLETE, activity_type=activity_type).values(
208+
for instance in ActivityLabel.objects.filter(
209+
status=ActivityLabel.LabelStatus.OVERRIDE, activity_type=activity_type
210+
).values(
210211
"activity_label",
211212
"strategy_type",
212213
"is_start",
@@ -232,12 +233,12 @@ def get_label_attributes(label: str, activity_type: str) -> pd.Series:
232233
individually in the ActivityLabel model.
233234
234235
Before looking for a regex match, if the label has a corresponding instance in the ActivityLabel
235-
model with status=COMPLETE, then it returns the attributes from that instance. This allows us to
236+
model with status=OVERRIDE, then it returns the attributes from that instance. This allows us to
236237
support labels that are too complex to match with a regex. For example, the ButterProduction
237238
labels often contain the name of the milk that the butter is derived from, and so we need to
238239
return a CPC code that is different to the one matched by the product in the label. This also
239240
allows us to create new labels to support new BSSs without needing to update code. We use the
240-
ActivityLabel instances in before testing the regexes, so that we can override the regexes if
241+
ActivityLabel instances before testing the regexes, so that we can override the regexes if
241242
necessary, e.g. to ignore labels containing a product_id that doesn't match any of the
242243
ClassifiedProduct instances.
243244

‎pipelines/assets/wealth_characteristic.py‎

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -109,7 +109,7 @@
109109
WealthGroupCharacteristicValue,
110110
)
111111
from metadata.lookups import WealthGroupCategoryLookup # NOQA: E402
112-
from metadata.models import LabelStatus, WealthCharacteristicLabel # NOQA: E402
112+
from metadata.models import WealthCharacteristicLabel # NOQA: E402
113113

114114
# Indexes of header rows in the Data3 dataframe (wealth_group_category, district, village)
115115
HEADER_ROWS = [3, 4, 5]
@@ -187,7 +187,9 @@ def wealth_characteristic_instances(
187187
wealthgroupcategorylookup = WealthGroupCategoryLookup()
188188
label_map = {
189189
instance.pop("wealth_characteristic_label").lower(): instance
190-
for instance in WealthCharacteristicLabel.objects.filter(status=LabelStatus.COMPLETE).values(
190+
for instance in WealthCharacteristicLabel.objects.filter(
191+
status=WealthCharacteristicLabel.LabelStatus.COMPLETE
192+
).values(
191193
"wealth_characteristic_label",
192194
"wealth_characteristic_id",
193195
"product_id",

‎pipelines_tests/test_assets/test_livelihood_activity_assets.py‎

Lines changed: 58 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,10 @@
33

44
import pandas as pd
55
from django.test import TestCase
6-
from pipelines.assets.livelihood_activity import get_label_attributes
6+
from pipelines.assets.livelihood_activity import (
7+
get_label_attributes,
8+
get_livelihood_activity_label_map,
9+
)
710

811
from metadata.models import ActivityLabel
912

@@ -44,3 +47,57 @@ def test_livelihood_activity_regexes(self):
4447
any([bool(v) for k, v in unwanted_attributes.items()]),
4548
msg=f"Extra attributes {({k: v for k, v in unwanted_attributes.items() if v})} found for '{label}'",
4649
)
50+
51+
def test_activity_label_override(self):
52+
label = "riz - kg produits"
53+
expected_regex_attributes = {
54+
"activity_label": label,
55+
"is_start": True,
56+
"product_id": "riz",
57+
"unit_of_measure_id": "kg",
58+
"attribute": "quantity_produced",
59+
}
60+
# Test that the regular expression matches the label and returns the expected attributes
61+
regex_attributes = {k: v for k, v in get_label_attributes(label, LIVELIHOOD_ACTIVITY).items()}
62+
self.assertDictEqual(
63+
expected_regex_attributes,
64+
{k: v for k, v in regex_attributes.items() if k in expected_regex_attributes},
65+
)
66+
# Now create an ActivityLabel instance with status=OVERRIDE for the same label but different attributes
67+
expected_override_attributes = {
68+
"activity_label": label,
69+
"is_start": False,
70+
"product_id": "R01132",
71+
"season": "season 1",
72+
}
73+
ActivityLabel.objects.create(
74+
status=ActivityLabel.LabelStatus.OVERRIDE,
75+
activity_type=LIVELIHOOD_ACTIVITY,
76+
**expected_override_attributes,
77+
)
78+
# Clear the cache of label attributes
79+
get_label_attributes.cache_clear()
80+
get_livelihood_activity_label_map.cache_clear()
81+
# Test that the override attributes are returned instead of the regex attributes
82+
override_attributes = {k: v for k, v in get_label_attributes(label, LIVELIHOOD_ACTIVITY).items()}
83+
self.assertDictEqual(
84+
expected_override_attributes,
85+
{k: v for k, v in override_attributes.items() if k in expected_override_attributes},
86+
)
87+
# Test that additional attributes set in the regex instance are ignored when using the override
88+
self.assertEqual(None, override_attributes["unit_of_measure_id"])
89+
# Update the ActivityLabel instance to make it use the regex again
90+
ActivityLabel.objects.filter(activity_label=label, activity_type=LIVELIHOOD_ACTIVITY).update(
91+
status=ActivityLabel.LabelStatus.REGULAR_EXPRESSION
92+
)
93+
# Clear the cache of label attributes
94+
get_label_attributes.cache_clear()
95+
get_livelihood_activity_label_map.cache_clear()
96+
# Test that the regex attributes are returned again
97+
regex_attributes = {k: v for k, v in get_label_attributes(label, LIVELIHOOD_ACTIVITY).items()}
98+
self.assertDictEqual(
99+
expected_regex_attributes,
100+
{k: v for k, v in regex_attributes.items() if k in expected_regex_attributes},
101+
)
102+
# Test that additional attributes set in the ActivityLabel instance are ignored when using the regex
103+
self.assertEqual("", regex_attributes["season"])

0 commit comments

Comments
 (0)