Skip to content

Hea 688/ignore activities without wealth category #152

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
rhunwicks merged 3 commits into from
Sep 5, 2025
Merged

Hea 688/ignore activities without wealth category #152

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
8f3cf74
Ignore separator columns in Data, Data2, Data3 worksheets - see HEA-688
rhunwicks Sep 4, 2025
de17734
Minor linting improvements - see HEA-688
rhunwicks Sep 4, 2025
88e0dde
Refactor assertions to prevent linting warning - see HEA-688
rhunwicks Sep 5, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 3 additions & 2 deletions apps/common/lookups.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@

import pandas as pd
from django.contrib.auth.models import User
from django.db.models import Model

from .fields import translation_fields
from .models import (
Expand All @@ -31,7 +32,7 @@ class Lookup(ABC):

# The primary key fields for the model being retrieved, typically
# a single auto-incremented surrogate key named `id`
id_fields = None
id_fields = []

# The fields passed from the dataframe that are used to constrain the lookup,
# typically the fields that are foreign keys from the model being looked up to
Expand Down Expand Up @@ -346,7 +347,7 @@ def get(self, value: str, **parent_values) -> str | None:
return None

@functools.cache
def get_instance(self, value: str, **parent_values) -> str | None:
def get_instance(self, value: str, **parent_values) -> Model | None:
"""
Return the Django model instance for a single string, or None if there is no match.

Expand Down
52 changes: 37 additions & 15 deletions pipelines/assets/livelihood_activity.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -420,10 +420,14 @@ def get_instances_from_dataframe(
# to the list, provided that it has at least one Livelihood Activity where there is some income,
# expediture or consumption. This excludes empty activities that only contain attributes for,
# for example, 'type_of_milk_sold_or_other_uses'.
# Also ignore any livelihood activities that don't have a Wealth Category component to the Wealth Group
# natural key. These are from blank columns between Wealth Category groups in the BSS, which sometimes
# contain data where values or formulae have been copied across all the columns in a row.
non_empty_livelihood_activities = [
livelihood_activity
for livelihood_activity in livelihood_activities_for_strategy
if any(
if livelihood_activity["wealth_group"][2] # Make sure there is a Wealth Category

Copilot AI Sep 4, 2025

Copy link

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Using a magic number (index 2) to access the wealth category makes the code unclear. Consider adding a named constant or accessing by key name to make the intention explicit.

Copilot uses AI. Check for mistakes.
and any(
(
field in livelihood_activity
and (livelihood_activity[field] or livelihood_activity[field] == 0)
Expand All @@ -432,16 +436,27 @@ def get_instances_from_dataframe(
)
]

# Don't save Livelihood Strategies from the Data worksheet that are captured in more detail
# on the Data2 or Data3 worksheets. These strategies have entries in the Data sheet like 'Construction cash income -- see Data2'
if non_empty_livelihood_activities and re.match(
r"^.*(?:data ?[23]|prochaine feuille)$", livelihood_strategy["activity_label"], re.IGNORECASE
):
non_empty_livelihood_activities = []
# Don't save Livelihood Strategies from the Data worksheet that are captured in more detail on the
# Data2 or Data3 worksheets. These strategies have entries in the Data sheet like
# 'Construction cash income -- see Data2'
if non_empty_livelihood_activities:
# Assertion to prevent linting from complaining about possible None values
assert livelihood_strategy is not None, (
"Found Livelihood Activities from row %s, but there is no Livelihood Strategy defined." % row
)
if re.match(
r"^.*(?:data ?[23]|prochaine feuille)$", livelihood_strategy["activity_label"], re.IGNORECASE
):
non_empty_livelihood_activities = []

if non_empty_livelihood_activities:
# Finalize the livelihood strategy and activities, making various adjustments for quirks in the BSS

# Assertion to prevent linting from complaining about possible None values
assert livelihood_strategy is not None, (
"Found Livelihood Activities from row %s, but there is no Livelihood Strategy defined." % row
)

# Copy the product_id for MilkProduction and ButterProduction the previous livelihood strategy if
# necessary.
if (
Expand All @@ -468,15 +483,22 @@ def get_instances_from_dataframe(
country_id=livelihood_zone_baseline.livelihood_zone.country_id,
)
and previous_livelihood_activities_for_strategy
and "milking_animals" in previous_livelihood_strategy["attribute_rows"]
and "milking_animals" not in livelihood_strategy["attribute_rows"]
):
livelihood_strategy["attribute_rows"]["milking_animals"] = row
for i in range(len(previous_livelihood_activities_for_strategy)):
if "milking_animals" in previous_livelihood_activities_for_strategy[i]:
livelihood_activities_for_strategy[i]["milking_animals"] = (
previous_livelihood_activities_for_strategy[i]["milking_animals"]
)
# Assertion to prevent linting from complaining about possible None values
assert previous_livelihood_strategy is not None, (
"Found Previous Livelihood Activities from row %s, but there is no Previous Livelihood Strategy defined."
% row
)
if (
"milking_animals" in previous_livelihood_strategy["attribute_rows"]
and "milking_animals" not in livelihood_strategy["attribute_rows"]
):
livelihood_strategy["attribute_rows"]["milking_animals"] = row
for i in range(len(previous_livelihood_activities_for_strategy)):
if "milking_animals" in previous_livelihood_activities_for_strategy[i]:
livelihood_activities_for_strategy[i]["milking_animals"] = (
previous_livelihood_activities_for_strategy[i]["milking_animals"]
)

# Calculate kcals_consumed if the livelihood activity only contains the percentage_kcals.
# This is typical for ButterProduction and consumption of green crops.
Expand Down