Add miscellaneous income SOI targets (#882)

Author: MaxGhenis

changelog.d/add-misc-income-targets.added.md

@@ -0,0 +1 @@
+Added IRS SOI aggregate targets for positive miscellaneous income in calibration.

policyengine_us_data/calibration/target_config.yaml

@@ -253,6 +253,12 @@ include:
   - variable: tax_unit_count
     geo_level: national
     domain_variable: adjusted_gross_income,tax_exempt_interest_income
+  - variable: miscellaneous_income
+    geo_level: national
+    domain_variable: miscellaneous_income
+  - variable: tax_unit_count
+    geo_level: national
+    domain_variable: miscellaneous_income
   - variable: refundable_ctc
     geo_level: national
     domain_variable: refundable_ctc

policyengine_us_data/db/etl_irs_soi.py

@@ -177,6 +177,7 @@ def _skip_coarse_state_agi_person_count_target(geo_type: str, agi_stub: int) ->
     "charitable_deduction": "charitable_contributions_deductions",
     "dividend_income": "ordinary_dividends",
     "income_tax_before_credits": "income_tax_before_credits",
+    "miscellaneous_income": "other_income",
     "qualified_dividend_income": "qualified_dividends",
     "rental_income": "rent_and_royalty_net_income",
     "tax_exempt_interest_income": "exempt_interest",

policyengine_us_data/storage/calibration_targets/refresh_soi_table_targets.py

@@ -43,6 +43,7 @@
     "income_tax_before_credits": {True: ("ER",), False: ("ES",)},
     "ira_distributions": {True: ("AT",), False: ("AU",)},
     "ordinary_dividends": {True: ("X",), False: ("Y",)},
+    "other_income": {True: ("CN",), False: ("CO",)},
     "partnership_and_s_corp_income": {
         True: ("BP", "BT"),
         False: ("BQ", "BU"),

policyengine_us_data/storage/calibration_targets/soi_targets.csv

@@ -9778,6 +9778,12 @@ Year,SOI table,XLSX column,XLSX row,Variable,Filing status,AGI lower bound,AGI u
 2023,Table 1.4,V,27,exempt_interest,All,5000000.0,10000000.0,True,False,False,29763
 2023,Table 1.4,W,28,exempt_interest,All,10000000.0,inf,False,False,False,6779791000
 2023,Table 1.4,V,28,exempt_interest,All,10000000.0,inf,True,False,False,21005
+2021,Table 1.4,CC,9,other_income,All,-inf,inf,False,False,True,62702551000
+2021,Table 1.4,CB,9,other_income,All,-inf,inf,True,False,True,5930776
+2022,Table 1.4,CO,9,other_income,All,-inf,inf,False,False,True,62607506000
+2022,Table 1.4,CN,9,other_income,All,-inf,inf,True,False,True,7557464
+2023,Table 1.4,CO,9,other_income,All,-inf,inf,False,False,True,60816508000
+2023,Table 1.4,CN,9,other_income,All,-inf,inf,True,False,True,7468434
 2023,Table 2.1,CF,10,idpitgst,All,-inf,inf,False,False,True,218543083000
 2023,Table 2.1,CF,33,idpitgst,All,-inf,inf,False,True,False,214582294000
 2023,Table 2.1,CE,10,idpitgst,All,-inf,inf,True,False,True,14464822

policyengine_us_data/utils/soi.py

@@ -17,6 +17,7 @@
     # deductions, so use total interest deductions as the closest available
     # proxy.
     "mortgage_interest_deductions": "interest_deduction",
+    "other_income": "miscellaneous_income",
     "total_pension_income": "pension_income",
     "total_social_security": "social_security",
     "business_net_losses": "total_self_employment_income",
@@ -77,6 +78,7 @@ def pe(variable):
     df["ordinary_dividends"] = pe("non_qualified_dividend_income") + pe(
         "qualified_dividend_income"
     )
+    df["other_income"] = pe("miscellaneous_income") * (pe("miscellaneous_income") > 0)
     df["partnership_and_s_corp_income"] = pe("partnership_s_corp_income") * (
         pe("partnership_s_corp_income") > 0
     )
@@ -131,6 +133,7 @@ def puf_to_soi(puf, year):
     df["ira_distributions"] = puf.E01400
     df["count_of_exemptions"] = puf.XTOT
     df["ordinary_dividends"] = puf.E00600
+    df["other_income"] = puf.E01200 * (puf.E01200 > 0)
     df["partnership_and_s_corp_income"] = puf.E26270 * (puf.E26270 > 0)
     df["partnership_and_s_corp_losses"] = -puf.E26270 * (puf.E26270 < 0)
     df["total_pension_income"] = puf.E01500

tests/unit/calibration/test_target_config.py

@@ -284,6 +284,30 @@ def test_training_config_includes_national_charitable_deduction_targets(
             "domain_variable": "charitable_deduction",
         } in include_rules
 
+    def test_training_config_includes_national_miscellaneous_income_targets(
+        self,
+    ):
+        config = load_target_config(
+            str(
+                Path(__file__).resolve().parents[3]
+                / "policyengine_us_data"
+                / "calibration"
+                / "target_config.yaml"
+            )
+        )
+
+        include_rules = config["include"]
+        assert {
+            "variable": "miscellaneous_income",
+            "geo_level": "national",
+            "domain_variable": "miscellaneous_income",
+        } in include_rules
+        assert {
+            "variable": "tax_unit_count",
+            "geo_level": "national",
+            "domain_variable": "miscellaneous_income",
+        } in include_rules
+
     def test_training_config_includes_medicare_part_b_target(self):
         config = load_target_config(
             str(

tests/unit/test_etl_irs_soi_overlay.py

@@ -201,6 +201,10 @@ def test_workbook_domain_targets_include_charitable_deduction():
     )
 
 
+def test_workbook_domain_targets_include_miscellaneous_income():
+    assert WORKBOOK_NATIONAL_DOMAIN_TARGETS["miscellaneous_income"] == "other_income"
+
+
 def test_skip_coarse_state_agi_person_count_target_only_for_state_stub_9():
     assert _skip_coarse_state_agi_person_count_target("state", 9) is True
     assert _skip_coarse_state_agi_person_count_target("state", 8) is False

tests/unit/test_refresh_soi_table_targets.py

@@ -103,9 +103,11 @@ def test_build_target_year_rows_reads_standard_table_cells(monkeypatch):
 
 def test_build_target_year_rows_uses_semantic_table_1_4_columns(monkeypatch):
     module = load_module()
-    workbook = make_workbook(cols=80)
+    workbook = make_workbook(cols=100)
     row_index = 9  # Excel row 10
 
+    workbook.iat[row_index, module._column_index("CN")] = 9
+    workbook.iat[row_index, module._column_index("CO")] = 11
     workbook.iat[row_index, module._column_index("BP")] = 10
     workbook.iat[row_index, module._column_index("BQ")] = 30
     workbook.iat[row_index, module._column_index("BT")] = 20
@@ -117,6 +119,24 @@ def test_build_target_year_rows_uses_semantic_table_1_4_columns(monkeypatch):
 
     targets = pd.DataFrame(
         [
+            make_target_row(
+                **{
+                    "SOI table": "Table 1.4",
+                    "XLSX column": "CB",
+                    "XLSX row": 10,
+                    "Variable": "other_income",
+                    "Count": True,
+                }
+            ),
+            make_target_row(
+                **{
+                    "SOI table": "Table 1.4",
+                    "XLSX column": "CC",
+                    "XLSX row": 10,
+                    "Variable": "other_income",
+                    "Count": False,
+                }
+            ),
             make_target_row(
                 **{
                     "SOI table": "Table 1.4",
@@ -163,8 +183,17 @@ def test_build_target_year_rows_uses_semantic_table_1_4_columns(monkeypatch):
         targets, source_year=2021, target_year=2023
     )
 
-    assert refreshed["Value"].tolist() == [30.0, 70_000.0, 11.0, 15_000.0]
+    assert refreshed["Value"].tolist() == [
+        9.0,
+        11_000.0,
+        30.0,
+        70_000.0,
+        11.0,
+        15_000.0,
+    ]
     assert refreshed["XLSX column"].tolist() == [
+        "CN",
+        "CO",
         "BP+BT",
         "BQ+BU",
         "BR+BV",

tests/unit/test_soi_utils.py

@@ -84,9 +84,12 @@ def test_get_soi_includes_mortgage_interest_deduction_targets():
     soi_module = load_soi_module()
     soi = soi_module.get_soi(2024)
     mortgage_interest = soi[soi.Variable == "mortgage_interest_deductions"]
+    other_income = soi[soi.Variable == "other_income"]
 
     assert not mortgage_interest.empty
     assert mortgage_interest["Value"].gt(0).all()
+    assert not other_income.empty
+    assert other_income["Value"].gt(0).all()
 
 
 def test_pe_to_soi_combines_sstb_and_non_sstb_schedule_c(monkeypatch):
@@ -102,6 +105,7 @@ def calculate(self, variable, map_to=None):
             values = {
                 "self_employment_income": np.array([100.0, -10.0]),
                 "sstb_self_employment_income": np.array([50.0, -25.0]),
+                "miscellaneous_income": np.array([12.0, -5.0]),
                 "filing_status": np.array(["SINGLE", "SINGLE"]),
                 "tax_unit_weight": np.ones(n),
                 "household_id": np.arange(1, n + 1),
@@ -120,6 +124,7 @@ def calculate(self, variable, map_to=None):
     np.testing.assert_array_equal(
         soi["business_net_losses"].to_numpy(), np.array([0.0, 35.0])
     )
+    np.testing.assert_array_equal(soi["other_income"].to_numpy(), np.array([12.0, 0.0]))
 
 
 def test_get_soi_uses_best_available_year_per_variable(monkeypatch):