diff --git a/policyengine_uk_data/datasets/local_areas/local_authorities/loss.py b/policyengine_uk_data/datasets/local_areas/local_authorities/loss.py index fd5ed9440..d81104d8c 100644 --- a/policyengine_uk_data/datasets/local_areas/local_authorities/loss.py +++ b/policyengine_uk_data/datasets/local_areas/local_authorities/loss.py @@ -11,6 +11,9 @@ - ONS income: ONS small area income estimates - Tenure: English Housing Survey - Private rent: VOA/ONS private rental market statistics +- Council tax bands A-H: VOA Council Tax Stock of Properties (per LA) +- Council tax £ paid (net of CTR): MHCLG taxbase × Band D (England), + Welsh Government Council Tax Income (Wales) """ from policyengine_uk import Microsimulation @@ -252,6 +255,57 @@ def create_local_authority_target_matrix( national_rent * la_household_share, ) + # ── Council tax band counts (LA targets) ─────────────────────── + # Derived/proxy targets: per-LA VOA dwellings in each band A-H. + # Lineage drift vs the matrix-side household council_tax_band: + # VOA counts dwellings (incl. exempt / empty / second homes); + # matrix counts households. See la_council_tax.py for full + # caveat. Missing cells stay NaN and are masked out by the + # calibrator; this keeps the target direct instead of fabricating + # national-share fallbacks for Scotland or Northern Ireland. Band I + # is Wales-only and rarely populated, so it is intentionally + # excluded. + ct_path = STORAGE_FOLDER / "la_council_tax.csv" + if ct_path.exists(): + ct_data = pd.read_csv(ct_path) + ct_columns = ["code"] + [f"count_band_{b}" for b in "ABCDEFGH"] + if "total_council_tax_net" in ct_data.columns: + ct_columns.append("total_council_tax_net") + ct_merged = la_codes.merge(ct_data[ct_columns], on="code", how="left") + ct_band = sim.calculate("council_tax_band").values + for band in "ABCDEFGH": + col = f"voa/council_tax/{band}" + matrix[col] = (ct_band == band).astype(float) + csv_col = f"count_band_{band}" + has_count = ct_merged[csv_col].notna().values + direct = ct_merged[csv_col].values + y[col] = np.where( + has_count, + direct, + np.nan, + ) + + # ── Council tax £ paid, net of CTR (LA targets) ──────────── + # Derived/proxy target: y = MHCLG taxbase × Band D (E) or WG + # Council Tax Income (W). Matrix col is FRS-reported + # council_tax_less_benefit (gross − reported CTB). Same + # intent (household council tax paid net of CTR), different + # construction paths — see la_council_tax.py for the lineage + # caveat flagged in review by @MaxGhenis. Both sides are net + # of CTR, per Max's 28 Apr standup decision on FRS alignment. + # Missing cells remain NaN and are masked out by the calibrator. + if "total_council_tax_net" in ct_merged.columns: + matrix["housing/council_tax_net"] = sim.calculate( + "council_tax_less_benefit" + ).values + has_ct_net = ct_merged["total_council_tax_net"].notna().values + direct_net = ct_merged["total_council_tax_net"].values + y["housing/council_tax_net"] = np.where( + has_ct_net, + direct_net, + np.nan, + ) + # ── Country mask ─────────────────────────────────────────────── country_mask = create_country_mask( household_countries=sim.calculate("country").values, diff --git a/policyengine_uk_data/storage/la_council_tax.csv b/policyengine_uk_data/storage/la_council_tax.csv new file mode 100644 index 000000000..a4142e223 --- /dev/null +++ b/policyengine_uk_data/storage/la_council_tax.csv @@ -0,0 +1,361 @@ +code,name,country,band_d_amount,count_band_A,count_band_B,count_band_C,count_band_D,count_band_E,count_band_F,count_band_G,count_band_H,count_band_I,total_dwellings,has_council_tax,total_council_tax_net +E06000001,Hartlepool,ENGLAND,2560.2,24090.0,7830.0,6780.0,3630.0,2070.0,930.0,700.0,110.0,,46140.0,True,69738772.716 +E06000002,Middlesbrough,ENGLAND,2549.16,33080.0,11550.0,11610.0,5620.0,2810.0,1040.0,600.0,50.0,,66360.0,True,96111361.854 +E06000003,Redcar and Cleveland,ENGLAND,2544.48,26840.0,13800.0,14820.0,6090.0,3610.0,1070.0,420.0,30.0,,66660.0,True,108610441.7904 +E06000004,Stockton-on-Tees,ENGLAND,2589.83,34900.0,17560.0,16890.0,10400.0,6430.0,3000.0,1610.0,130.0,,90910.0,True,163386702.9723 +E06000005,Darlington,ENGLAND,2493.83,23160.0,11180.0,7800.0,5950.0,3590.0,1400.0,680.0,50.0,,53820.0,True,89851198.602 +E06000006,Halton,ENGLAND,2366.61,27340.0,12760.0,8370.0,5230.0,3890.0,1270.0,360.0,40.0,,59260.0,True,90641943.9813 +E06000007,Warrington,ENGLAND,2447.61,27780.0,21410.0,19980.0,12400.0,7960.0,4900.0,2730.0,230.0,,97390.0,True,179465496.1275 +E06000008,Blackburn with Darwen,ENGLAND,2455.27,35860.0,9950.0,9010.0,4960.0,2780.0,840.0,580.0,70.0,,64050.0,True,97480357.175 +E06000009,Blackpool,ENGLAND,2513.22,32210.0,21330.0,11480.0,4860.0,1950.0,580.0,250.0,30.0,,72700.0,True,105132843.1146 +E06000010,"Kingston upon Hull, City of",ENGLAND,2295.04,82540.0,24430.0,11440.0,4990.0,1410.0,320.0,60.0,40.0,,125230.0,True,158193802.34239998 +E06000011,East Riding of Yorkshire,ENGLAND,2466.08,40330.0,38430.0,33140.0,25660.0,17380.0,7740.0,3440.0,300.0,,166420.0,True,322923928.2 +E06000012,North East Lincolnshire,ENGLAND,2483.85,38690.0,18400.0,9390.0,5330.0,2520.0,980.0,510.0,60.0,,75880.0,True,119402941.722 +E06000013,North Lincolnshire,ENGLAND,2348.46,35920.0,16440.0,11580.0,7850.0,4020.0,1720.0,550.0,30.0,,78110.0,True,124935934.9014 +E06000014,York,ENGLAND,2287.15,13150.0,25560.0,27300.0,13880.0,8030.0,3830.0,1930.0,140.0,,93800.0,True,161307680.6415 +E06000015,Derby,ENGLAND,2306.0,58500.0,22270.0,17230.0,8900.0,4850.0,2340.0,710.0,50.0,,114860.0,True,172765773.66 +E06000016,Leicester,ENGLAND,2528.75,86990.0,28270.0,16960.0,7340.0,3570.0,1590.0,620.0,60.0,,145410.0,True,216985639.7625 +E06000017,Rutland,ENGLAND,2737.58,1750.0,4890.0,3340.0,2570.0,2400.0,1690.0,1360.0,170.0,,18160.0,True,44849143.4966 +E06000018,Nottingham,ENGLAND,2755.39,92180.0,25740.0,17180.0,7790.0,2660.0,1120.0,760.0,120.0,,147540.0,True,204079861.5691 +E06000019,"Herefordshire, County of",ENGLAND,2574.37,14040.0,20630.0,17710.0,13820.0,12660.0,7680.0,3800.0,190.0,,90520.0,True,191881285.7988 +E06000020,Telford and Wrekin,ENGLAND,2256.26,26960.0,22860.0,14900.0,10050.0,5870.0,2570.0,1210.0,60.0,,84480.0,True,136176527.1748 +E06000021,Stoke-on-Trent,ENGLAND,2183.12,70450.0,25560.0,15880.0,5100.0,1980.0,560.0,200.0,40.0,,119780.0,True,153262730.4136 +E06000022,Bath and North East Somerset,ENGLAND,2383.42,9370.0,19630.0,21290.0,14500.0,10460.0,5980.0,5210.0,460.0,,86890.0,True,170676301.01860002 +E06000023,"Bristol, City of",ENGLAND,2713.68,56890.0,76330.0,41320.0,19900.0,10120.0,4940.0,2900.0,360.0,,212750.0,True,385352247.8376 +E06000024,North Somerset,ENGLAND,2491.22,13940.0,22140.0,23810.0,18060.0,13390.0,6880.0,3470.0,290.0,,101970.0,True,208481532.3544 +E06000025,South Gloucestershire,ENGLAND,2550.59,14480.0,37160.0,30510.0,24250.0,13700.0,6630.0,2210.0,200.0,,129130.0,True,266248336.30640003 +E06000026,Plymouth,ENGLAND,2441.85,48160.0,33380.0,23670.0,10310.0,5250.0,1890.0,620.0,60.0,,123360.0,True,191876739.2955 +E06000027,Torbay,ENGLAND,2470.25,13790.0,17850.0,17180.0,10750.0,5450.0,2420.0,1290.0,140.0,,68860.0,True,128340628.3275 +E06000030,Swindon,ENGLAND,2438.1,15190.0,28070.0,25270.0,18530.0,9640.0,4020.0,1500.0,70.0,,102290.0,True,192953720.862 +E06000031,Peterborough,ENGLAND,2293.47,35930.0,22730.0,15460.0,9010.0,4930.0,2030.0,1070.0,80.0,,91240.0,True,146052091.4337 +E06000032,Luton,ENGLAND,2439.93,21520.0,28100.0,22860.0,8000.0,3520.0,1130.0,290.0,30.0,,85450.0,True,145845376.1913 +E06000033,Southend-on-Sea,ENGLAND,2265.17,17170.0,16180.0,24770.0,12900.0,6820.0,3710.0,1640.0,130.0,,83310.0,True,142569029.64220002 +E06000034,Thurrock,ENGLAND,2254.68,7540.0,14110.0,27720.0,12800.0,4860.0,2320.0,890.0,60.0,,70300.0,True,123409210.8492 +E06000035,Medway,ENGLAND,2328.55,11470.0,38870.0,35530.0,19470.0,10230.0,4420.0,1560.0,60.0,,121600.0,True,218665514.865 +E06000036,Bracknell Forest,ENGLAND,2264.8,1650.0,5660.0,19880.0,10500.0,8670.0,5450.0,2690.0,310.0,,54820.0,True,113227158.584 +E06000037,West Berkshire,ENGLAND,2505.37,2750.0,7290.0,20500.0,17840.0,10950.0,7120.0,4880.0,780.0,,72100.0,True,170564587.45200002 +E06000038,Reading,ENGLAND,2612.77,7270.0,14990.0,31530.0,12220.0,5920.0,3390.0,1910.0,100.0,,77330.0,True,160555291.3094 +E06000039,Slough,ENGLAND,2414.44,1560.0,10950.0,24390.0,13670.0,4700.0,1730.0,360.0,10.0,,57370.0,True,108753476.0536 +E06000040,Windsor and Maidenhead,ENGLAND,1952.62,2140.0,4110.0,10700.0,16670.0,13590.0,8410.0,9970.0,2020.0,,67610.0,True,141816135.0368 +E06000041,Wokingham,ENGLAND,2497.7,2030.0,4250.0,12130.0,21120.0,17040.0,11160.0,6830.0,570.0,,75140.0,True,194973858.872 +E06000042,Milton Keynes,ENGLAND,2372.21,18060.0,35810.0,32280.0,16440.0,13300.0,7180.0,3240.0,180.0,,126480.0,True,241552441.9611 +E06000043,Brighton and Hove,ENGLAND,2580.54,30090.0,30490.0,35410.0,20470.0,11720.0,4830.0,2840.0,220.0,,136070.0,True,250160876.49659997 +E06000044,Portsmouth,ENGLAND,2291.71,27460.0,31800.0,22440.0,6280.0,3790.0,1660.0,660.0,60.0,,94150.0,True,141561630.9178 +E06000045,Southampton,ENGLAND,2381.48,36920.0,35880.0,23760.0,9790.0,3200.0,1420.0,450.0,30.0,,111450.0,True,166766709.22 +E06000046,Isle of Wight,ENGLAND,2625.79,10650.0,18690.0,18000.0,13460.0,7340.0,3290.0,1570.0,150.0,,73150.0,True,156531403.0753 +E06000047,County Durham,ENGLAND,2622.15,144190.0,36650.0,32590.0,23630.0,11500.0,4490.0,2310.0,280.0,,255650.0,True,398097461.3715 +E06000049,Cheshire East,ENGLAND,2454.87,31710.0,39250.0,37540.0,28220.0,23520.0,15410.0,13130.0,2010.0,,190790.0,True,405983135.6229 +E06000050,Cheshire West and Chester,ENGLAND,2517.33,35320.0,39140.0,33010.0,22530.0,17600.0,10540.0,8110.0,640.0,,166880.0,True,334162542.90389997 +E06000051,Shropshire,ENGLAND,2528.18,27440.0,38890.0,31760.0,22070.0,17630.0,9370.0,4830.0,360.0,,152340.0,True,311591738.141 +E06000052,Cornwall,ENGLAND,2590.93,66210.0,73220.0,61520.0,45030.0,26830.0,9650.0,4430.0,450.0,,287330.0,True,580602125.5231999 +E06000053,Isles of Scilly,ENGLAND,2047.96,10.0,40.0,110.0,270.0,330.0,270.0,130.0,10.0,,1170.0,True,2924282.0840000003 +E06000054,Wiltshire,ENGLAND,2571.78,27030.0,42950.0,58820.0,41200.0,31680.0,18490.0,11570.0,1370.0,,233110.0,True,509263386.9618 +E06000055,Bedford,ENGLAND,2471.74,10800.0,19080.0,20170.0,12480.0,8960.0,5790.0,3290.0,250.0,,80820.0,True,161728840.13579997 +E06000056,Central Bedfordshire,ENGLAND,2516.16,11290.0,25640.0,36860.0,26700.0,17800.0,10200.0,5700.0,410.0,,134600.0,True,287374256.8704 +E06000057,Northumberland,ENGLAND,2596.94,70860.0,26550.0,21800.0,18410.0,12210.0,7600.0,4550.0,590.0,,162560.0,True,307605231.7234 +E06000058,"Bournemouth, Christchurch and Poole",ENGLAND,2435.58,26730.0,34610.0,54390.0,36070.0,21890.0,9450.0,5820.0,1310.0,,190260.0,True,374759089.9416 +E06000059,Dorset,ENGLAND,2765.02,20840.0,31970.0,43430.0,37400.0,28890.0,16160.0,8230.0,740.0,,187640.0,True,455833178.642 +E06000060,Buckinghamshire,ENGLAND,2526.58,6970.0,26310.0,54560.0,47150.0,36730.0,29170.0,31530.0,6160.0,,238580.0,True,600363690.9438 +E06000061,North Northamptonshire,ENGLAND,2423.96,48480.0,42960.0,29500.0,18750.0,12750.0,5530.0,3060.0,280.0,,161300.0,True,290406600.0528 +E06000062,West Northamptonshire,ENGLAND,2486.66,38700.0,41080.0,46110.0,24480.0,17970.0,10680.0,6520.0,490.0,,186010.0,True,367682918.7856 +E06000063,Cumberland,ENGLAND,2511.32,64010.0,25940.0,20470.0,15630.0,8470.0,2980.0,980.0,80.0,,138560.0,True,236997160.946 +E06000064,Westmorland and Furness,ENGLAND,2507.99,29300.0,24520.0,23280.0,17770.0,12750.0,6480.0,3440.0,340.0,,117860.0,True,242993207.1637 +E06000065,North Yorkshire,ENGLAND,2544.34,48920.0,65620.0,70150.0,46130.0,37890.0,22340.0,14070.0,1310.0,,306440.0,True,660473592.9662 +E06000066,Somerset,ENGLAND,2560.59,43360.0,72220.0,59900.0,40390.0,31050.0,16940.0,7820.0,520.0,,272210.0,True,564425860.5495 +E07000008,Cambridge,ENGLAND,2467.02,4420.0,11090.0,20890.0,10610.0,6060.0,3990.0,3270.0,510.0,,60840.0,True,116051433.2028 +E07000009,East Cambridgeshire,ENGLAND,2485.18,5000.0,11670.0,8580.0,7610.0,5110.0,2460.0,820.0,90.0,,41350.0,True,84112209.39359999 +E07000010,Fenland,ENGLAND,2537.74,17450.0,13190.0,9030.0,4960.0,2520.0,710.0,190.0,30.0,,48060.0,True,83011632.47899999 +E07000011,Huntingdonshire,ENGLAND,2556.4,12600.0,21820.0,19420.0,12970.0,10360.0,4490.0,2010.0,180.0,,83840.0,True,170498919.052 +E07000012,South Cambridgeshire,ENGLAND,2536.25,2780.0,8620.0,22500.0,13830.0,12340.0,8670.0,4900.0,420.0,,74060.0,True,176533018.1875 +E07000032,Amber Valley,ENGLAND,2407.69,22980.0,12960.0,11020.0,7050.0,3630.0,1940.0,1550.0,140.0,,61270.0,True,103047976.3088 +E07000033,Bolsover,ENGLAND,2559.41,22440.0,6910.0,4880.0,3040.0,1150.0,350.0,150.0,20.0,,38940.0,True,61046074.74419999 +E07000034,Chesterfield,ENGLAND,2339.49,27100.0,10630.0,6530.0,4070.0,2150.0,620.0,220.0,30.0,,51350.0,True,72837728.4498 +E07000035,Derbyshire Dales,ENGLAND,2444.7,3680.0,7630.0,7720.0,5850.0,5290.0,3300.0,2170.0,140.0,,35770.0,True,77618344.90799999 +E07000036,Erewash,ENGLAND,2360.64,22010.0,14290.0,7900.0,5080.0,2430.0,920.0,530.0,40.0,,53190.0,True,83145730.2816 +E07000037,High Peak,ENGLAND,2384.66,8670.0,13380.0,9510.0,5120.0,4100.0,2200.0,860.0,50.0,,43890.0,True,78740328.56319998 +E07000038,North East Derbyshire,ENGLAND,2466.57,19260.0,10030.0,8200.0,5700.0,3300.0,1630.0,900.0,70.0,,49080.0,True,84402991.51889999 +E07000039,South Derbyshire,ENGLAND,2339.61,11910.0,11780.0,9770.0,8370.0,5700.0,2430.0,1020.0,100.0,,51070.0,True,92340288.9864 +E07000040,East Devon,ENGLAND,2595.46,6860.0,14150.0,17830.0,13670.0,11400.0,6700.0,4270.0,210.0,,75100.0,True,172658590.1726 +E07000041,Exeter,ENGLAND,2495.36,13480.0,15450.0,14560.0,10030.0,4570.0,2220.0,1020.0,50.0,,61380.0,True,102276147.5008 +E07000042,Mid Devon,ENGLAND,2656.41,6550.0,9380.0,7290.0,6700.0,5010.0,2640.0,980.0,60.0,,38610.0,True,83891659.18439999 +E07000043,North Devon,ENGLAND,2642.21,10450.0,11830.0,10620.0,8670.0,5100.0,2040.0,640.0,50.0,,49400.0,True,101676124.8487 +E07000044,South Hams,ENGLAND,2615.22,5360.0,9100.0,9570.0,8500.0,7590.0,4140.0,3200.0,380.0,,47840.0,True,121130556.9588 +E07000045,Teignbridge,ENGLAND,2643.12,9080.0,14740.0,14520.0,12280.0,8610.0,4110.0,2060.0,120.0,,65510.0,True,140888947.77359998 +E07000046,Torridge,ENGLAND,2601.42,8460.0,7500.0,7340.0,6150.0,3520.0,1210.0,390.0,30.0,,34600.0,True,70389872.715 +E07000047,West Devon,ENGLAND,2705.33,3640.0,6760.0,5760.0,4560.0,3710.0,1920.0,1050.0,80.0,,27470.0,True,61999400.275 +E07000061,Eastbourne,ENGLAND,2654.28,8870.0,13370.0,10980.0,8760.0,4600.0,2070.0,1130.0,90.0,,49870.0,True,100026807.228 +E07000062,Hastings,ENGLAND,2676.58,14990.0,12520.0,7810.0,5800.0,2350.0,860.0,200.0,40.0,,44570.0,True,78580133.0378 +E07000063,Lewes,ENGLAND,2756.17,4510.0,6410.0,13850.0,10360.0,6260.0,3300.0,2420.0,250.0,,47360.0,True,109470855.4599 +E07000064,Rother,ENGLAND,2700.95,5050.0,7420.0,10230.0,9430.0,7690.0,4110.0,2770.0,280.0,,46970.0,True,108819546.892 +E07000065,Wealden,ENGLAND,2728.43,4360.0,8470.0,17900.0,16070.0,12010.0,8130.0,7340.0,950.0,,75240.0,True,192940927.45 +E07000066,Basildon,ENGLAND,2327.24,9100.0,16310.0,24750.0,14900.0,7930.0,4900.0,2080.0,200.0,,80160.0,True,147426464.96799996 +E07000067,Braintree,ENGLAND,2280.34,6240.0,17650.0,20390.0,10900.0,8240.0,4990.0,2500.0,230.0,,71150.0,True,136107360.4854 +E07000068,Brentwood,ENGLAND,2258.14,690.0,3080.0,6950.0,8900.0,6470.0,4620.0,4060.0,660.0,,35420.0,True,79803886.9956 +E07000069,Castle Point,ENGLAND,2318.63,3370.0,6440.0,13880.0,8710.0,4480.0,1920.0,770.0,70.0,,39630.0,True,74080576.29450001 +E07000070,Chelmsford,ENGLAND,2300.18,5180.0,10960.0,23980.0,18540.0,11460.0,6610.0,4590.0,450.0,,81760.0,True,167657590.00199997 +E07000071,Colchester,ENGLAND,2282.85,9600.0,23120.0,21170.0,16230.0,9320.0,4310.0,2600.0,170.0,,86520.0,True,154434870.98549998 +E07000072,Epping Forest,ENGLAND,2277.21,2120.0,5220.0,12060.0,14210.0,10280.0,7130.0,6320.0,1260.0,,58610.0,True,130820750.1822 +E07000073,Harlow,ENGLAND,2298.69,2620.0,8360.0,19630.0,5240.0,3070.0,1180.0,450.0,20.0,,40560.0,True,70293043.71090001 +E07000074,Maldon,ENGLAND,2328.37,2480.0,3980.0,8640.0,5690.0,4710.0,3180.0,1700.0,190.0,,30560.0,True,64400222.8441 +E07000075,Rochford,ENGLAND,2363.48,1580.0,4080.0,12110.0,10890.0,5350.0,2800.0,1450.0,80.0,,38340.0,True,80682376.7428 +E07000076,Tendring,ENGLAND,2269.63,13840.0,18040.0,22070.0,12380.0,6120.0,2160.0,980.0,90.0,,75690.0,True,127770023.7539 +E07000077,Uttlesford,ENGLAND,2333.82,1430.0,4180.0,8770.0,8080.0,7610.0,5310.0,4960.0,460.0,,40790.0,True,95153738.8794 +E07000078,Cheltenham,ENGLAND,2369.52,9990.0,13530.0,14150.0,9450.0,5270.0,2800.0,2210.0,150.0,,57560.0,True,107492824.5912 +E07000079,Cotswold,ENGLAND,2391.65,3710.0,5430.0,11550.0,7770.0,6740.0,5310.0,4950.0,780.0,,46250.0,True,108783482.755 +E07000080,Forest of Dean,ENGLAND,2443.42,7210.0,10410.0,9230.0,6150.0,4670.0,2190.0,1040.0,80.0,,40980.0,True,77993453.2818 +E07000081,Gloucester,ENGLAND,2355.84,17700.0,16980.0,14340.0,6110.0,3880.0,930.0,180.0,10.0,,60130.0,True,96160677.12 +E07000082,Stroud,ENGLAND,2491.36,7630.0,13050.0,13150.0,8640.0,7320.0,4230.0,2660.0,260.0,,56940.0,True,120515229.9472 +E07000083,Tewkesbury,ENGLAND,2340.53,7060.0,7130.0,13060.0,6790.0,6070.0,3780.0,2130.0,210.0,,46230.0,True,91803099.7013 +E07000084,Basingstoke and Deane,ENGLAND,2255.21,2620.0,13300.0,28100.0,15490.0,11350.0,6790.0,3570.0,460.0,,81690.0,True,161429600.6554 +E07000085,East Hampshire,ENGLAND,2343.92,3080.0,6360.0,13430.0,11590.0,9270.0,6950.0,5160.0,670.0,,56500.0,True,125280437.9112 +E07000086,Eastleigh,ENGLAND,2342.21,4710.0,12600.0,19810.0,11260.0,8190.0,3510.0,1210.0,30.0,,61320.0,True,120453442.6446 +E07000087,Fareham,ENGLAND,2270.55,3640.0,7390.0,15710.0,10880.0,8130.0,3720.0,1530.0,140.0,,51150.0,True,102211759.96500002 +E07000088,Gosport,ENGLAND,2344.29,6170.0,13380.0,9230.0,5170.0,2010.0,1570.0,330.0,20.0,,37880.0,True,65026548.9783 +E07000089,Hart,ENGLAND,2400.02,780.0,2600.0,10120.0,9720.0,8280.0,7220.0,4050.0,260.0,,43030.0,True,104670992.251 +E07000090,Havant,ENGLAND,2320.28,8610.0,14860.0,13770.0,10550.0,5950.0,2590.0,990.0,50.0,,57380.0,True,100856260.4384 +E07000091,New Forest,ENGLAND,2419.76,7330.0,12250.0,18260.0,19460.0,13700.0,7220.0,4700.0,620.0,,83540.0,True,182225326.0744 +E07000092,Rushmoor,ENGLAND,2320.28,1500.0,9090.0,16950.0,9310.0,4250.0,1320.0,330.0,20.0,,42770.0,True,77926023.73 +E07000093,Test Valley,ENGLAND,2305.52,3120.0,10040.0,15600.0,11420.0,8800.0,5320.0,3950.0,560.0,,58810.0,True,122507609.308 +E07000094,Winchester,ENGLAND,2359.76,3050.0,7530.0,13750.0,11310.0,9190.0,7180.0,6020.0,750.0,,58770.0,True,130385919.4664 +E07000095,Broxbourne,ENGLAND,2306.43,590.0,3820.0,9960.0,14530.0,7880.0,3010.0,2350.0,230.0,,42360.0,True,86002461.67589998 +E07000096,Dacorum,ENGLAND,2408.51,1240.0,8300.0,20760.0,16240.0,9420.0,5860.0,5260.0,850.0,,67930.0,True,148105614.2813 +E07000098,Hertsmere,ENGLAND,2405.83,820.0,3090.0,7450.0,14980.0,8990.0,4640.0,4750.0,1230.0,,45940.0,True,105962184.0536 +E07000099,North Hertfordshire,ENGLAND,2448.52,3450.0,9440.0,20320.0,10450.0,7490.0,4900.0,3630.0,360.0,,60020.0,True,126916316.3428 +E07000102,Three Rivers,ENGLAND,2422.31,890.0,2430.0,6980.0,10030.0,7560.0,4470.0,5170.0,1630.0,,39150.0,True,95732065.0638 +E07000103,Watford,ENGLAND,2446.7,470.0,4490.0,16000.0,14260.0,3890.0,2220.0,1930.0,80.0,,43350.0,True,88923036.06899999 +E07000105,Ashford,ENGLAND,2410.16,4380.0,13910.0,13710.0,10220.0,7120.0,6220.0,3570.0,220.0,,59350.0,True,122977522.2408 +E07000106,Canterbury,ENGLAND,2419.33,7760.0,14100.0,21250.0,14160.0,7950.0,4210.0,2320.0,120.0,,71860.0,True,136703588.4309 +E07000107,Dartford,ENGLAND,2375.15,1750.0,6830.0,15370.0,14590.0,7750.0,3030.0,1120.0,70.0,,50510.0,True,102951399.283 +E07000108,Dover,ENGLAND,2461.74,7300.0,17350.0,14600.0,7750.0,4650.0,2580.0,1590.0,80.0,,55890.0,True,105381919.746 +E07000109,Gravesham,ENGLAND,2408.24,3680.0,7050.0,15260.0,10970.0,4780.0,2130.0,1100.0,110.0,,45070.0,True,87394547.95199999 +E07000110,Maidstone,ENGLAND,2502.59,4520.0,9570.0,21210.0,20600.0,10980.0,6610.0,4720.0,390.0,,78600.0,True,174634859.47349998 +E07000111,Sevenoaks,ENGLAND,2522.36,1910.0,3480.0,11660.0,12160.0,7730.0,6170.0,7990.0,1550.0,,52660.0,True,135170775.26360002 +E07000112,Folkestone and Hythe,ENGLAND,2538.75,7190.0,13030.0,14680.0,8720.0,5280.0,2790.0,1980.0,100.0,,53780.0,True,108702648.8625 +E07000113,Swale,ENGLAND,2405.65,10400.0,16590.0,17920.0,12190.0,6130.0,2820.0,1310.0,120.0,,67470.0,True,124193966.6175 +E07000114,Thanet,ENGLAND,2490.9,16400.0,20220.0,18390.0,8250.0,4210.0,1660.0,840.0,40.0,,70010.0,True,122454935.628 +E07000115,Tonbridge and Malling,ENGLAND,2471.83,1770.0,4080.0,16220.0,14410.0,9650.0,5570.0,5100.0,470.0,,57270.0,True,135852345.3209 +E07000116,Tunbridge Wells,ENGLAND,2446.15,3500.0,5440.0,14220.0,11090.0,7180.0,5240.0,5620.0,620.0,,52900.0,True,121677542.9905 +E07000117,Burnley,ENGLAND,2549.42,25800.0,5850.0,6530.0,3080.0,1350.0,340.0,130.0,20.0,,43090.0,True,63026276.850200005 +E07000118,Chorley,ENGLAND,2430.23,15200.0,12200.0,9810.0,7020.0,5490.0,2390.0,1050.0,70.0,,53230.0,True,95874955.1254 +E07000119,Fylde,ENGLAND,2483.94,7180.0,7030.0,9690.0,7560.0,5420.0,2960.0,1690.0,140.0,,41680.0,True,82953511.2036 +E07000120,Hyndburn,ENGLAND,2465.98,21980.0,5870.0,5900.0,2870.0,870.0,280.0,170.0,10.0,,37960.0,True,56535230.0986 +E07000121,Lancaster,ENGLAND,2503.19,23600.0,16510.0,12830.0,6740.0,4270.0,2130.0,870.0,80.0,,67020.0,True,111255931.6554 +E07000122,Pendle,ENGLAND,2640.18,25170.0,4850.0,4640.0,3500.0,1890.0,1030.0,550.0,40.0,,41660.0,True,68878098.31379999 +E07000123,Preston,ENGLAND,2575.74,31000.0,13610.0,11260.0,7490.0,3750.0,2130.0,1080.0,70.0,,70390.0,True,120344265.1854 +E07000124,Ribble Valley,ENGLAND,2386.5,3900.0,5620.0,5810.0,5100.0,4310.0,2680.0,2050.0,240.0,,29700.0,True,62155509.495000005 +E07000125,Rossendale,ENGLAND,2519.97,16480.0,5250.0,4430.0,3460.0,2040.0,720.0,470.0,40.0,,32890.0,True,55365000.885 +E07000126,South Ribble,ENGLAND,2442.66,10130.0,13620.0,13130.0,8660.0,4510.0,1770.0,610.0,30.0,,52470.0,True,95763190.6902 +E07000127,West Lancashire,ENGLAND,2456.23,14980.0,9760.0,10530.0,7650.0,5430.0,2690.0,1700.0,110.0,,52850.0,True,99299213.7627 +E07000128,Wyre,ENGLAND,2460.53,12060.0,12580.0,13420.0,7900.0,6050.0,2760.0,1150.0,80.0,,55980.0,True,101098232.0347 +E07000129,Blaby,ENGLAND,2490.22,5040.0,15700.0,10280.0,6940.0,4640.0,1660.0,550.0,40.0,,44840.0,True,87035255.8826 +E07000130,Charnwood,ENGLAND,2405.64,13340.0,22020.0,19430.0,11510.0,7600.0,3430.0,2070.0,230.0,,79620.0,True,146830955.77319998 +E07000131,Harborough,ENGLAND,2389.87,4750.0,9770.0,8640.0,7110.0,7400.0,4120.0,2840.0,260.0,,44900.0,True,96766553.261 +E07000132,Hinckley and Bosworth,ENGLAND,2382.62,8930.0,16590.0,11540.0,7590.0,4630.0,2350.0,1160.0,70.0,,52840.0,True,97034200.9008 +E07000133,Melton,ENGLAND,2429.67,3700.0,7790.0,4100.0,4050.0,2690.0,1640.0,1050.0,90.0,,25110.0,True,50451320.0556 +E07000134,North West Leicestershire,ENGLAND,2418.2,10270.0,14790.0,8280.0,7300.0,5420.0,2050.0,990.0,60.0,,49150.0,True,92237450.964 +E07000135,Oadby and Wigston,ENGLAND,2406.85,4150.0,6340.0,7340.0,3290.0,2200.0,650.0,490.0,90.0,,24560.0,True,45705768.6095 +E07000136,Boston,ENGLAND,2308.73,15550.0,6610.0,6240.0,2390.0,880.0,240.0,80.0,20.0,,31990.0,True,47770163.303 +E07000137,East Lindsey,ENGLAND,2275.88,28110.0,15500.0,16290.0,6970.0,3790.0,1490.0,610.0,60.0,,72820.0,True,113440237.2128 +E07000138,Lincoln,ENGLAND,2323.08,28930.0,9140.0,4950.0,2570.0,1440.0,530.0,140.0,40.0,,47750.0,True,61694105.2524 +E07000139,North Kesteven,ENGLAND,2337.81,14220.0,14110.0,13430.0,7480.0,3840.0,1680.0,430.0,60.0,,55250.0,True,95308586.1792 +E07000140,South Holland,ENGLAND,2278.2,16760.0,9810.0,10700.0,4790.0,2150.0,450.0,120.0,20.0,,44790.0,True,71420499.24599999 +E07000141,South Kesteven,ENGLAND,2260.0,20100.0,15530.0,11790.0,9650.0,6080.0,3130.0,1130.0,110.0,,67510.0,True,114197302.8 +E07000142,West Lindsey,ENGLAND,2354.25,17270.0,8670.0,8200.0,6290.0,3970.0,1690.0,550.0,70.0,,46710.0,True,79041283.4475 +E07000143,Breckland,ENGLAND,2443.67,16260.0,18480.0,14810.0,8550.0,5040.0,1910.0,830.0,60.0,,65950.0,True,117759650.8889 +E07000144,Broadland,ENGLAND,2437.78,5060.0,16080.0,21850.0,10660.0,5650.0,2370.0,880.0,100.0,,62650.0,True,122031390.72980002 +E07000145,Great Yarmouth,ENGLAND,2414.41,20910.0,12800.0,9080.0,4480.0,2020.0,640.0,260.0,20.0,,50210.0,True,78466876.354 +E07000146,King's Lynn and West Norfolk,ENGLAND,2430.13,24670.0,17970.0,13850.0,10010.0,5410.0,2790.0,1220.0,120.0,,76030.0,True,139444747.608 +E07000147,North Norfolk,ENGLAND,2459.93,12190.0,14640.0,11880.0,9220.0,5130.0,2460.0,1100.0,90.0,,56700.0,True,115525372.7991 +E07000148,Norwich,ENGLAND,2503.44,28590.0,23800.0,9090.0,3760.0,2360.0,930.0,610.0,70.0,,69220.0,True,104319095.832 +E07000149,South Norfolk,ENGLAND,2482.87,7340.0,18430.0,16780.0,11990.0,7950.0,3400.0,1580.0,120.0,,67590.0,True,135243344.1359 +E07000170,Ashfield,ENGLAND,2608.76,30290.0,11920.0,9110.0,4460.0,1560.0,520.0,130.0,30.0,,58020.0,True,93062738.9692 +E07000171,Bassetlaw,ENGLAND,2644.95,28230.0,9040.0,7100.0,7070.0,3530.0,1720.0,740.0,60.0,,57490.0,True,104384221.326 +E07000172,Broxtowe,ENGLAND,2618.21,16870.0,13610.0,11200.0,6230.0,2840.0,840.0,510.0,30.0,,52130.0,True,94838792.4596 +E07000173,Gedling,ENGLAND,2609.55,14900.0,15650.0,10760.0,7070.0,4390.0,1500.0,950.0,100.0,,55320.0,True,105315566.5125 +E07000174,Mansfield,ENGLAND,2600.38,27850.0,9960.0,7030.0,4250.0,1770.0,470.0,210.0,30.0,,51550.0,True,82840253.6524 +E07000175,Newark and Sherwood,ENGLAND,2682.1,24260.0,8980.0,9640.0,6640.0,4570.0,2810.0,1530.0,140.0,,58560.0,True,115832603.688 +E07000176,Rushcliffe,ENGLAND,2643.27,6420.0,11430.0,12090.0,10200.0,7720.0,4640.0,2720.0,150.0,,55380.0,True,126555776.2623 +E07000177,Cherwell,ENGLAND,2582.96,6230.0,16680.0,19780.0,12700.0,9260.0,4490.0,2890.0,270.0,,72300.0,True,157880562.9144 +E07000178,Oxford,ENGLAND,2678.4,2970.0,9890.0,19810.0,16320.0,7670.0,3140.0,3370.0,610.0,,63780.0,True,133562326.464 +E07000179,South Oxfordshire,ENGLAND,2597.57,2130.0,6070.0,18530.0,15260.0,10980.0,6810.0,6420.0,990.0,,67190.0,True,169008476.30990002 +E07000180,Vale of White Horse,ENGLAND,2577.44,1910.0,6870.0,19390.0,14150.0,10450.0,6620.0,4870.0,490.0,,64750.0,True,156141676.0416 +E07000181,West Oxfordshire,ENGLAND,2566.95,1690.0,6220.0,17990.0,11950.0,7990.0,4640.0,3100.0,420.0,,53990.0,True,128058641.1165 +E07000192,Cannock Chase,ENGLAND,2375.83,14120.0,14510.0,8860.0,5680.0,2000.0,660.0,280.0,20.0,,46120.0,True,73182739.00659999 +E07000193,East Staffordshire,ENGLAND,2351.34,18370.0,12310.0,9590.0,6790.0,5160.0,2660.0,1300.0,100.0,,56270.0,True,98871566.2002 +E07000194,Lichfield,ENGLAND,2351.9,6010.0,11020.0,12460.0,7530.0,5140.0,4030.0,2730.0,430.0,,49330.0,True,99099705.438 +E07000195,Newcastle-under-Lyme,ENGLAND,2334.25,24990.0,10910.0,11780.0,4990.0,2910.0,1860.0,1020.0,50.0,,58520.0,True,92254905.4325 +E07000196,South Staffordshire,ENGLAND,2313.67,6900.0,10910.0,12080.0,7570.0,5600.0,3610.0,2490.0,220.0,,49370.0,True,94538916.1434 +E07000197,Stafford,ENGLAND,2302.68,12420.0,14480.0,14660.0,10060.0,7020.0,3780.0,1660.0,110.0,,64170.0,True,117457312.0128 +E07000198,Staffordshire Moorlands,ENGLAND,2338.21,9660.0,10860.0,10880.0,6360.0,4560.0,2020.0,810.0,40.0,,45180.0,True,81123728.30800001 +E07000199,Tamworth,ENGLAND,2300.76,9530.0,12410.0,6180.0,4020.0,2210.0,600.0,130.0,10.0,,35070.0,True,55627499.18880001 +E07000200,Babergh,ENGLAND,2341.55,5220.0,12430.0,9280.0,7820.0,4880.0,2650.0,1800.0,200.0,,44280.0,True,86388232.49550001 +E07000202,Ipswich,ENGLAND,2468.25,19420.0,23220.0,11540.0,4510.0,2370.0,960.0,380.0,20.0,,62410.0,True,100245258.675 +E07000203,Mid Suffolk,ENGLAND,2316.88,5950.0,13510.0,11200.0,8080.0,6420.0,3390.0,1740.0,110.0,,50400.0,True,98656457.408 +E07000207,Elmbridge,ENGLAND,2557.75,460.0,1890.0,8180.0,14150.0,11310.0,8140.0,11460.0,4550.0,,60130.0,True,174567000.205 +E07000208,Epsom and Ewell,ENGLAND,2530.84,160.0,1270.0,5610.0,9290.0,7810.0,4830.0,4180.0,150.0,,33310.0,True,86398878.87280001 +E07000209,Guildford,ENGLAND,2547.3,2290.0,3550.0,12790.0,16940.0,10640.0,6920.0,7590.0,1860.0,,62580.0,True,159479498.871 +E07000210,Mole Valley,ENGLAND,2519.86,1690.0,2740.0,4520.0,8420.0,7240.0,6310.0,7830.0,1080.0,,39830.0,True,107809539.0484 +E07000211,Reigate and Banstead,ENGLAND,2566.61,1230.0,4270.0,13130.0,18260.0,11660.0,7700.0,7540.0,1140.0,,64930.0,True,167523096.6898 +E07000212,Runnymede,ENGLAND,2492.99,2500.0,1610.0,8040.0,11690.0,6950.0,4300.0,3230.0,1140.0,,39460.0,True,92856672.7589 +E07000213,Spelthorne,ENGLAND,2526.44,460.0,1790.0,10140.0,15180.0,10110.0,4730.0,2200.0,120.0,,44730.0,True,106445081.7136 +E07000214,Surrey Heath,ENGLAND,2585.16,740.0,2650.0,6380.0,10230.0,6900.0,5940.0,5330.0,540.0,,38700.0,True,105937943.7816 +E07000215,Tandridge,ENGLAND,2594.85,940.0,2230.0,5440.0,9130.0,7870.0,4940.0,6400.0,1290.0,,38240.0,True,103822024.38 +E07000216,Waverley,ENGLAND,2604.87,1030.0,3480.0,10920.0,13590.0,10110.0,7380.0,8910.0,2180.0,,57590.0,True,157838242.44239998 +E07000217,Woking,ENGLAND,2598.04,340.0,3620.0,11320.0,12600.0,6360.0,4240.0,5260.0,830.0,,44560.0,True,114327659.514 +E07000218,North Warwickshire,ENGLAND,2532.1,6750.0,7510.0,6520.0,4060.0,2610.0,1420.0,790.0,80.0,,29740.0,True,56553111.487 +E07000219,Nuneaton and Bedworth,ENGLAND,2502.16,20960.0,13710.0,14700.0,7910.0,3180.0,870.0,200.0,20.0,,61530.0,True,105203242.1352 +E07000220,Rugby,ENGLAND,2483.05,9090.0,12130.0,12530.0,7400.0,5280.0,3530.0,1910.0,110.0,,51990.0,True,104257558.485 +E07000221,Stratford-on-Avon,ENGLAND,2484.17,3850.0,8630.0,18950.0,11350.0,10780.0,6850.0,5980.0,1000.0,,67380.0,True,156157534.5785 +E07000222,Warwick,ENGLAND,2461.55,5040.0,12630.0,18940.0,13730.0,8520.0,5700.0,4720.0,460.0,,69750.0,True,147803375.902 +E07000223,Adur,ENGLAND,2548.28,2780.0,5110.0,11580.0,6340.0,2050.0,760.0,340.0,10.0,,28980.0,True,56716252.510400005 +E07000224,Arun,ENGLAND,2487.24,8520.0,12990.0,21180.0,16680.0,10930.0,6350.0,2800.0,310.0,,79760.0,True,168637981.04999998 +E07000225,Chichester,ENGLAND,2469.68,3350.0,6540.0,15290.0,13040.0,9560.0,6440.0,6020.0,1360.0,,61610.0,True,147627418.8024 +E07000226,Crawley,ENGLAND,2418.46,1300.0,7670.0,22570.0,9100.0,3820.0,2350.0,470.0,10.0,,47290.0,True,89533759.2908 +E07000227,Horsham,ENGLAND,2441.01,2440.0,6790.0,14020.0,14410.0,11640.0,8560.0,8080.0,840.0,,66760.0,True,161787091.53750002 +E07000228,Mid Sussex,ENGLAND,2474.26,2340.0,7790.0,15320.0,17830.0,12370.0,9200.0,5050.0,450.0,,70340.0,True,167375845.5958 +E07000229,Worthing,ENGLAND,2456.13,8370.0,11640.0,13190.0,9760.0,5610.0,2450.0,930.0,30.0,,51980.0,True,99423479.24490002 +E07000234,Bromsgrove,ENGLAND,2478.65,3950.0,7460.0,9350.0,8060.0,7220.0,3850.0,2920.0,380.0,,43190.0,True,95860549.425 +E07000235,Malvern Hills,ENGLAND,2450.92,4100.0,7690.0,8820.0,5980.0,5370.0,4150.0,2550.0,130.0,,38780.0,True,83479413.6048 +E07000236,Redditch,ENGLAND,2462.95,8040.0,12220.0,7970.0,4490.0,3450.0,1290.0,480.0,20.0,,37960.0,True,66457090.224 +E07000237,Worcester,ENGLAND,2405.81,9010.0,15430.0,12030.0,5670.0,3590.0,1620.0,450.0,10.0,,47800.0,True,82468785.04810001 +E07000238,Wychavon,ENGLAND,2373.19,7180.0,12070.0,14580.0,8550.0,7890.0,6900.0,4850.0,230.0,,62250.0,True,131136427.7655 +E07000239,Wyre Forest,ENGLAND,2512.47,11730.0,11960.0,12080.0,6540.0,3610.0,1800.0,1300.0,140.0,,49150.0,True,88924076.83229998 +E07000240,St Albans,ENGLAND,2419.22,880.0,3220.0,10530.0,16530.0,13100.0,9530.0,8270.0,1410.0,,63480.0,True,159353973.01559997 +E07000241,Welwyn Hatfield,ENGLAND,2452.26,1250.0,5850.0,16360.0,12550.0,5590.0,4500.0,3960.0,750.0,,50790.0,True,110078150.397 +E07000242,East Hertfordshire,ENGLAND,2454.78,1000.0,6460.0,16520.0,17010.0,12070.0,8140.0,5950.0,870.0,,68010.0,True,163143549.60119998 +E07000243,Stevenage,ENGLAND,2391.97,1700.0,6890.0,21760.0,3380.0,3280.0,980.0,430.0,20.0,,38440.0,True,69980502.8671 +E07000244,East Suffolk,ENGLAND,2334.8,28230.0,31230.0,23810.0,18810.0,11760.0,5650.0,3070.0,250.0,,122810.0,True,226694674.284 +E07000245,West Suffolk,ENGLAND,2351.47,12730.0,29030.0,17370.0,12390.0,7410.0,3050.0,2100.0,200.0,,84280.0,True,145044054.46629998 +E08000001,Bolton,ENGLAND,2399.74,65640.0,22820.0,18990.0,11000.0,5820.0,2370.0,1930.0,250.0,,128800.0,True,200612072.67079997 +E08000002,Bury,ENGLAND,2555.15,30710.0,18680.0,17540.0,9380.0,5700.0,1900.0,1320.0,180.0,,85410.0,True,151795814.6185 +E08000003,Manchester,ENGLAND,2312.04,137810.0,43100.0,38170.0,21260.0,7640.0,3400.0,1270.0,150.0,,252810.0,True,329432337.26280004 +E08000004,Oldham,ENGLAND,2602.23,51140.0,17700.0,16730.0,7310.0,3540.0,1630.0,940.0,80.0,,99070.0,True,159794278.8741 +E08000005,Rochdale,ENGLAND,2600.83,51910.0,17400.0,13280.0,8260.0,4580.0,1710.0,900.0,50.0,,98100.0,True,159790833.872 +E08000006,Salford,ENGLAND,2594.45,63640.0,31220.0,22180.0,12350.0,3910.0,1490.0,880.0,110.0,,135780.0,True,217839621.46499997 +E08000007,Stockport,ENGLAND,2618.9,31820.0,29160.0,28940.0,19920.0,13320.0,6570.0,3620.0,230.0,,133580.0,True,262258400.663 +E08000008,Tameside,ENGLAND,2447.21,53040.0,19680.0,20250.0,7290.0,4000.0,950.0,450.0,40.0,,105700.0,True,163602522.5507 +E08000009,Trafford,ENGLAND,2291.7,20050.0,23020.0,27720.0,15590.0,7940.0,4690.0,4230.0,1070.0,,104300.0,True,190331322.582 +E08000010,Wigan,ENGLAND,2152.68,68940.0,34800.0,25810.0,13480.0,7730.0,2030.0,650.0,60.0,,153490.0,True,214035504.5928 +E08000011,Knowsley,ENGLAND,2488.25,38180.0,15340.0,10660.0,5000.0,2110.0,330.0,130.0,20.0,,71770.0,True,102122010.025 +E08000012,Liverpool,ENGLAND,2673.59,143680.0,44920.0,29870.0,14000.0,5140.0,2360.0,1760.0,150.0,,241880.0,True,341610797.0288 +E08000013,St. Helens,ENGLAND,2403.38,37850.0,19360.0,15770.0,7030.0,3810.0,1690.0,600.0,40.0,,86150.0,True,135803491.6098 +E08000014,Sefton,ENGLAND,2583.22,39990.0,28250.0,31380.0,15840.0,8930.0,3990.0,2740.0,290.0,,131400.0,True,236006001.5674 +E08000015,Wirral,ENGLAND,2500.59,61080.0,33140.0,27740.0,13690.0,8580.0,4390.0,3220.0,290.0,,152120.0,True,252960384.5652 +E08000016,Barnsley,ENGLAND,2325.96,65310.0,19340.0,14570.0,9790.0,4190.0,1630.0,710.0,60.0,,115610.0,True,164690528.184 +E08000017,Doncaster,ENGLAND,2167.75,82760.0,27200.0,16270.0,10220.0,4900.0,2280.0,980.0,140.0,,144760.0,True,193081752.63 +E08000018,Rotherham,ENGLAND,2381.53,64280.0,24200.0,16130.0,9790.0,5070.0,2010.0,810.0,70.0,,122350.0,True,180542241.30550003 +E08000019,Sheffield,ENGLAND,2510.16,152470.0,42460.0,33260.0,16910.0,9780.0,4420.0,2890.0,220.0,,262400.0,True,393731372.7144 +E08000021,Newcastle upon Tyne,ENGLAND,2542.19,77750.0,21680.0,20520.0,10160.0,5940.0,2640.0,1930.0,140.0,,140760.0,True,202683393.8353 +E08000022,North Tyneside,ENGLAND,2461.77,50580.0,16240.0,20200.0,8490.0,4540.0,1590.0,380.0,40.0,,102050.0,True,161795549.77019998 +E08000023,South Tyneside,ENGLAND,2434.43,45760.0,10470.0,8880.0,4940.0,1850.0,730.0,340.0,50.0,,73020.0,True,97994108.9063 +E08000024,Sunderland,ENGLAND,2197.14,80320.0,20110.0,18370.0,10090.0,3780.0,1230.0,700.0,60.0,,134650.0,True,174110755.3878 +E08000025,Birmingham,ENGLAND,2362.9,164600.0,132630.0,85230.0,44530.0,23130.0,9140.0,5960.0,930.0,,466150.0,True,667795638.413 +E08000026,Coventry,ENGLAND,2516.72,63820.0,44330.0,25210.0,10750.0,5230.0,2610.0,1490.0,190.0,,153630.0,True,240351793.43999997 +E08000027,Dudley,ENGLAND,2144.84,43610.0,40300.0,31110.0,16400.0,7150.0,2530.0,980.0,140.0,,142230.0,True,214959553.9248 +E08000028,Sandwell,ENGLAND,2244.46,59230.0,44650.0,21640.0,7680.0,2990.0,550.0,70.0,40.0,,136860.0,True,186768586.649 +E08000029,Solihull,ENGLAND,2197.26,14590.0,12510.0,23390.0,17700.0,12200.0,9260.0,5820.0,490.0,,95950.0,True,178805570.0886 +E08000030,Walsall,ENGLAND,2627.48,52370.0,27810.0,18930.0,10490.0,5680.0,2420.0,860.0,60.0,,118620.0,True,201080913.026 +E08000031,Wolverhampton,ENGLAND,2538.99,57530.0,25570.0,17660.0,7370.0,3150.0,1730.0,1000.0,130.0,,114130.0,True,176264505.8892 +E08000032,Bradford,ENGLAND,2360.73,92730.0,46940.0,41630.0,19000.0,13330.0,6260.0,3820.0,330.0,,224030.0,True,360921787.7391 +E08000033,Calderdale,ENGLAND,2420.15,44900.0,18530.0,16060.0,7730.0,5630.0,3150.0,1380.0,60.0,,97430.0,True,159935951.571 +E08000034,Kirklees,ENGLAND,2441.07,86120.0,35310.0,33310.0,17860.0,12720.0,5720.0,2330.0,170.0,,193530.0,True,322588621.035 +E08000035,Leeds,ENGLAND,2283.73,143230.0,80140.0,73400.0,38150.0,22930.0,10740.0,7490.0,760.0,,376840.0,True,573341150.031 +E08000036,Wakefield,ENGLAND,2296.89,79970.0,31330.0,24780.0,16330.0,8860.0,2600.0,1230.0,100.0,,165190.0,True,246450485.8683 +E08000037,Gateshead,ENGLAND,2715.81,56090.0,13230.0,15950.0,6070.0,2780.0,900.0,390.0,50.0,,95480.0,True,151587307.6041 +E09000001,City of London,ENGLAND,1329.56,,350.0,690.0,950.0,2890.0,1390.0,1500.0,310.0,,8080.0,True,15244455.7524 +E09000002,Barking and Dagenham,ENGLAND,2198.51,6030.0,12420.0,50340.0,10770.0,1800.0,390.0,50.0,10.0,,81800.0,True,129472518.3653 +E09000003,Barnet,ENGLAND,2132.6,3420.0,9570.0,32530.0,39640.0,33920.0,20370.0,16820.0,4350.0,,160620.0,True,339539499.162 +E09000004,Bexley,ENGLAND,2366.36,4350.0,11380.0,31230.0,28140.0,19550.0,5290.0,1750.0,50.0,,101750.0,True,206446665.3644 +E09000005,Brent,ENGLAND,2235.27,6090.0,14040.0,41900.0,39340.0,23430.0,6590.0,3460.0,250.0,,135110.0,True,251602952.3661 +E09000006,Bromley,ENGLAND,2140.04,2100.0,10590.0,30460.0,36460.0,29550.0,18580.0,14120.0,1750.0,,143600.0,True,304871061.41800004 +E09000007,Camden,ENGLAND,2207.55,3380.0,12270.0,20780.0,26560.0,19820.0,12950.0,13140.0,4940.0,,113830.0,True,220511198.178 +E09000008,Croydon,ENGLAND,2599.91,4450.0,24400.0,52850.0,42340.0,24140.0,11860.0,7520.0,660.0,,168220.0,True,368973637.3935 +E09000009,Ealing,ENGLAND,2138.53,5200.0,14970.0,36980.0,48680.0,25200.0,10520.0,7200.0,1060.0,,149800.0,True,274136385.72010005 +E09000010,Enfield,ENGLAND,2267.67,5420.0,12400.0,35190.0,37170.0,21550.0,9590.0,6050.0,1020.0,,128400.0,True,250144478.0601 +E09000011,Greenwich,ENGLAND,2107.86,10790.0,22200.0,45460.0,27100.0,12850.0,4010.0,2280.0,360.0,,125040.0,True,204263058.6012 +E09000012,Hackney,ENGLAND,2060.3,8170.0,31670.0,35990.0,25200.0,13500.0,4950.0,1320.0,50.0,,120850.0,True,174558361.219 +E09000013,Hammersmith and Fulham,ENGLAND,1519.51,4120.0,6630.0,14510.0,25920.0,17400.0,11330.0,12270.0,2940.0,,95120.0,True,141718057.4413 +E09000014,Haringey,ENGLAND,2313.78,7700.0,19400.0,36160.0,28640.0,11700.0,5540.0,4800.0,750.0,,114700.0,True,202145101.8972 +E09000015,Harrow,ENGLAND,2511.07,840.0,4470.0,22800.0,30020.0,22620.0,8550.0,6220.0,1320.0,,96860.0,True,232407940.58450004 +E09000016,Havering,ENGLAND,2424.66,5300.0,11080.0,29790.0,36200.0,16040.0,6770.0,3140.0,380.0,,108680.0,True,228206356.3206 +E09000017,Hillingdon,ENGLAND,2045.46,1270.0,6550.0,28300.0,47450.0,18810.0,10050.0,5340.0,490.0,,118260.0,True,217533341.451 +E09000018,Hounslow,ENGLAND,2185.56,2560.0,9560.0,30620.0,40720.0,16350.0,6370.0,4120.0,1030.0,,111320.0,True,208959599.4408 +E09000019,Islington,ENGLAND,2108.15,4660.0,6180.0,29760.0,33560.0,19520.0,10380.0,7300.0,960.0,,112320.0,True,181931110.361 +E09000020,Kensington and Chelsea,ENGLAND,1666.65,1460.0,3310.0,9370.0,14110.0,13410.0,12210.0,19930.0,15480.0,,89280.0,True,171780148.848 +E09000021,Kingston upon Thames,ENGLAND,2609.2,630.0,3420.0,16020.0,21010.0,15050.0,8770.0,4460.0,1080.0,,70440.0,True,178293341.644 +E09000022,Lambeth,ENGLAND,2047.11,5020.0,33450.0,42330.0,33160.0,16550.0,10820.0,5960.0,1180.0,,148470.0,True,245763600.6423 +E09000023,Lewisham,ENGLAND,2237.33,9280.0,36440.0,48040.0,27670.0,8060.0,2820.0,1360.0,190.0,,133860.0,True,218508185.6243 +E09000024,Merton,ENGLAND,2146.76,1190.0,8590.0,24140.0,28360.0,13540.0,5940.0,4180.0,1850.0,,87780.0,True,175588523.8184 +E09000025,Newham,ENGLAND,1944.23,5570.0,34340.0,56120.0,26900.0,7220.0,2210.0,240.0,40.0,,132640.0,True,188163726.4957 +E09000026,Redbridge,ENGLAND,2294.58,2790.0,14100.0,27920.0,32140.0,19750.0,7820.0,3320.0,220.0,,108060.0,True,220890933.1662 +E09000027,Richmond upon Thames,ENGLAND,2486.1,600.0,2180.0,13480.0,20700.0,19770.0,12160.0,13290.0,3640.0,,85810.0,True,227971988.904 +E09000028,Southwark,ENGLAND,1967.26,13220.0,38050.0,35710.0,28440.0,21540.0,7920.0,4590.0,830.0,,150300.0,True,227093195.8654 +E09000029,Sutton,ENGLAND,2378.64,990.0,8050.0,28300.0,24800.0,12950.0,7110.0,3780.0,290.0,,86270.0,True,183237343.08 +E09000030,Tower Hamlets,ENGLAND,1837.78,4500.0,25040.0,40570.0,32250.0,27010.0,14280.0,5570.0,780.0,,150000.0,True,222040965.5338 +E09000031,Waltham Forest,ENGLAND,2386.96,5230.0,32380.0,39150.0,23440.0,8390.0,1850.0,440.0,30.0,,110900.0,True,201084766.7584 +E09000032,Wandsworth,ENGLAND,1028.21,6560.0,13280.0,37340.0,35080.0,26990.0,20450.0,15230.0,3040.0,,157970.0,True,156911704.16070002 +E09000033,Westminster,ENGLAND,1049.55,1660.0,6460.0,15690.0,23130.0,24460.0,19180.0,24500.0,17190.0,,132280.0,True,170747374.67849997 +N09000001,Antrim and Newtownabbey,NORTHERN_IRELAND,,,,,,,,,,,,False, +N09000002,"Armagh City, Banbridge and Craigavon",NORTHERN_IRELAND,,,,,,,,,,,,False, +N09000003,Belfast,NORTHERN_IRELAND,,,,,,,,,,,,False, +N09000004,Causeway Coast and Glens,NORTHERN_IRELAND,,,,,,,,,,,,False, +N09000005,Derry City and Strabane,NORTHERN_IRELAND,,,,,,,,,,,,False, +N09000006,Fermanagh and Omagh,NORTHERN_IRELAND,,,,,,,,,,,,False, +N09000007,Lisburn and Castlereagh,NORTHERN_IRELAND,,,,,,,,,,,,False, +N09000008,Mid and East Antrim,NORTHERN_IRELAND,,,,,,,,,,,,False, +N09000009,Mid Ulster,NORTHERN_IRELAND,,,,,,,,,,,,False, +N09000010,"Newry, Mourne and Down",NORTHERN_IRELAND,,,,,,,,,,,,False, +S12000005,Clackmannanshire,SCOTLAND,1594.38,,,,,,,,,,,True, +S12000006,Dumfries and Galloway,SCOTLAND,1454.98,,,,,,,,,,,True, +S12000008,East Ayrshire,SCOTLAND,1606.44,,,,,,,,,,,True, +S12000010,East Lothian,SCOTLAND,1579.18,,,,,,,,,,,True, +S12000011,East Renfrewshire,SCOTLAND,1528.44,,,,,,,,,,,True, +S12000013,Na h-Eileanan Siar,SCOTLAND,1387.56,,,,,,,,,,,True, +S12000014,Falkirk,SCOTLAND,1576.77,,,,,,,,,,,True, +S12000017,Highland,SCOTLAND,1527.09,,,,,,,,,,,True, +S12000018,Inverclyde,SCOTLAND,1551.3,,,,,,,,,,,True, +S12000019,Midlothian,SCOTLAND,1666.2,,,,,,,,,,,True, +S12000020,Moray,SCOTLAND,1573.76,,,,,,,,,,,True, +S12000021,North Ayrshire,SCOTLAND,1553.77,,,,,,,,,,,True, +S12000023,Orkney Islands,SCOTLAND,1574.6,,,,,,,,,,,True, +S12000026,Scottish Borders,SCOTLAND,1491.72,,,,,,,,,,,True, +S12000027,Shetland Islands,SCOTLAND,1386.67,,,,,,,,,,,True, +S12000028,South Ayrshire,SCOTLAND,1569.41,,,,,,,,,,,True, +S12000029,South Lanarkshire,SCOTLAND,1378.85,,,,,,,,,,,True, +S12000030,Stirling,SCOTLAND,1611.78,,,,,,,,,,,True, +S12000033,Aberdeen City,SCOTLAND,1636.27,,,,,,,,,,,True, +S12000034,Aberdeenshire,SCOTLAND,1532.76,,,,,,,,,,,True, +S12000035,Argyll and Bute,SCOTLAND,1625.64,,,,,,,,,,,True, +S12000036,City of Edinburgh,SCOTLAND,1563.51,,,,,,,,,,,True, +S12000038,Renfrewshire,SCOTLAND,1572.61,,,,,,,,,,,True, +S12000039,West Dunbartonshire,SCOTLAND,1559.86,,,,,,,,,,,True, +S12000040,West Lothian,SCOTLAND,1515.45,,,,,,,,,,,True, +S12000041,Angus,SCOTLAND,1461.52,,,,,,,,,,,True, +S12000042,Dundee City,SCOTLAND,1605.34,,,,,,,,,,,True, +S12000045,East Dunbartonshire,SCOTLAND,1599.7,,,,,,,,,,,True, +S12000047,Fife,SCOTLAND,1498.76,,,,,,,,,,,True, +S12000048,Perth and Kinross,SCOTLAND,1537.04,,,,,,,,,,,True, +S12000049,Glasgow City,SCOTLAND,1611.0,,,,,,,,,,,True, +S12000050,North Lanarkshire,SCOTLAND,1452.86,,,,,,,,,,,True, +W06000001,Isle of Anglesey,WALES,2260.73,4960.0,6930.0,7210.0,7400.0,5540.0,2770.0,1080.0,170.0,60.0,36120.0,True,63114620.0 +W06000002,Gwynedd,WALES,2468.77,9310.0,15800.0,12700.0,10720.0,8300.0,4030.0,1280.0,210.0,90.0,62440.0,True,119406313.0 +W06000003,Conwy,WALES,2472.82,5460.0,8340.0,15610.0,11970.0,9300.0,5120.0,1940.0,430.0,160.0,58340.0,True,110610412.0 +W06000004,Denbighshire,WALES,2339.03,4160.0,7420.0,14970.0,8050.0,5660.0,3920.0,2060.0,320.0,170.0,46720.0,True,82023146.0 +W06000005,Flintshire,WALES,2376.17,4520.0,9640.0,20990.0,13400.0,11140.0,7830.0,3230.0,600.0,220.0,71570.0,True,131972546.99999999 +W06000006,Wrexham,WALES,2308.88,4510.0,12770.0,17250.0,10350.0,8270.0,5190.0,2570.0,720.0,290.0,61910.0,True,104887499.37199569 +W06000008,Ceredigion,WALES,2419.45,1780.0,5000.0,7730.0,7570.0,9180.0,3890.0,980.0,110.0,20.0,36260.0,True,70082599.0 +W06000009,Pembrokeshire,WALES,2166.15,6730.0,9460.0,14520.0,11520.0,12550.0,6070.0,2120.0,340.0,100.0,63400.0,True,111301517.0 +W06000010,Carmarthenshire,WALES,2351.2,9370.0,24550.0,18930.0,14700.0,13440.0,6700.0,2260.0,300.0,70.0,90310.0,True,151662184.0 +W06000011,Swansea,WALES,2238.29,17730.0,28650.0,25020.0,17330.0,13200.0,8180.0,3880.0,1200.0,540.0,115730.0,True,177267464.0 +W06000012,Neath Port Talbot,WALES,2541.39,13830.0,27500.0,11720.0,7430.0,4580.0,1530.0,540.0,110.0,20.0,67260.0,True,106279092.92000002 +W06000013,Bridgend,WALES,2477.99,10600.0,15370.0,14840.0,11050.0,8020.0,4480.0,1510.0,300.0,110.0,66280.0,True,115651162.0 +W06000014,Vale of Glamorgan,WALES,2231.54,1510.0,6680.0,14520.0,11910.0,10790.0,7740.0,5920.0,2270.0,1060.0,62400.0,True,118046914.0 +W06000015,Cardiff,WALES,2013.18,5320.0,20420.0,34920.0,37370.0,31540.0,22250.0,10590.0,2850.0,1480.0,166750.0,True,250934240.0 +W06000016,Rhondda Cynon Taf,WALES,2289.43,47080.0,25500.0,17510.0,9660.0,7140.0,3520.0,1190.0,200.0,70.0,111860.0,True,151783228.38 +W06000018,Caerphilly,WALES,2082.88,15550.0,26890.0,18930.0,9540.0,6840.0,2430.0,840.0,90.0,80.0,81190.0,True,105324903.21000004 +W06000019,Blaenau Gwent,WALES,2531.26,19200.0,8160.0,2730.0,1810.0,900.0,350.0,60.0,,20.0,33230.0,True,45129431.0 +W06000020,Torfaen,WALES,2155.68,6300.0,13120.0,12050.0,4350.0,4330.0,2410.0,690.0,70.0,30.0,43340.0,True,60708353.0 +W06000021,Monmouthshire,WALES,2416.5,520.0,3450.0,7140.0,9460.0,7660.0,8100.0,5520.0,1790.0,670.0,44300.0,True,98574346.0 +W06000022,Newport,WALES,2088.48,6910.0,15400.0,18470.0,12990.0,8560.0,6090.0,2710.0,560.0,190.0,71880.0,True,107214282.0 +W06000023,Powys,WALES,2351.72,6090.0,9440.0,13330.0,10630.0,12660.0,9740.0,4180.0,580.0,210.0,66870.0,True,129025924.0 +W06000024,Merthyr Tydfil,WALES,2593.6,14150.0,6680.0,2270.0,2210.0,1560.0,580.0,160.0,,10.0,27620.0,True,41504016.0 diff --git a/policyengine_uk_data/targets/compute/council_tax.py b/policyengine_uk_data/targets/compute/council_tax.py index 2c538f253..0c52856d7 100644 --- a/policyengine_uk_data/targets/compute/council_tax.py +++ b/policyengine_uk_data/targets/compute/council_tax.py @@ -19,9 +19,18 @@ def compute_council_tax_band(target, ctx) -> np.ndarray: def compute_obr_council_tax(target, ctx) -> np.ndarray: - """Compute OBR council tax receipts, optionally by country.""" + """Compute OBR council tax receipts, optionally by country. + + OBR Table 4.1 reports "Total net council tax receipts" — net of + council tax reduction (CTR) support. The matching household-level + signal is therefore ``council_tax_less_benefit`` (= gross council + tax less the CTR award), not ``council_tax`` (which is the gross + liability before CTR). Using the gross variable here would + systematically push weights down to fit a net target, leaking + bias into adjacent national calibrations. + """ name = target.name - ct = ctx.pe("council_tax") + ct = ctx.pe("council_tax_less_benefit") if name == "obr/council_tax": return ct diff --git a/policyengine_uk_data/targets/sources/la_council_tax.py b/policyengine_uk_data/targets/sources/la_council_tax.py new file mode 100644 index 000000000..9aac581c0 --- /dev/null +++ b/policyengine_uk_data/targets/sources/la_council_tax.py @@ -0,0 +1,240 @@ +"""Local-authority council tax calibration targets (derived proxies). + +Produces three kinds of LA-level calibration target from public data: + +- ``ons/council_tax_band_d/{code}``: the average Band D council tax + (inclusive of all precepts) each household pays in billing authority + ``code``. Sourced from MHCLG, Welsh Government and Scottish + Government annual publications. +- ``voa/council_tax/{code}/{band}``: the number of dwellings in band + ``A``–``H`` (England) or ``A``–``I`` (Wales) for billing authority + ``code``. Sourced from the VOA *Council Tax: Stock of Properties* + summary tables. +- ``housing/council_tax_net/{code}``: net council tax requirement per + LA (net of CTR support). England derived from MHCLG taxbase × Band D; + Wales sourced directly from WG Council Tax Income (Table 3). + +Data for all 360 LAs in ``local_authorities_2021.csv`` is joined from +the committed canonical file ``storage/la_council_tax.csv``. Rows where +a source did not provide a value are omitted so calibrators cleanly +skip them. + +Lineage caveats (flagged in PR review by @MaxGhenis): + +- ``voa/council_tax/{A..H}`` is a **derived proxy**, not a direct + match for the matrix-side household ``council_tax_band``: + * Target counts VOA dwellings; matrix counts policyengine-uk + households. A household ≠ a dwelling in general. + * VOA stock includes exempt, empty, and second-home dwellings, + which contribute zero to the matrix-side sum (no household lives + in them in the FRS). + * VOA covers England and Wales only. Scotland and NI cells are + masked out of the loss matrix unless a direct source is available. + * Banding ratios differ: Scotland diverged from the standard + 6/9–18/9 E&W ratios after the 2017 reform; Wales has Band I, + England does not. + +- ``housing/council_tax_net`` is a **derived proxy**: + * Target value (England) is MHCLG ``taxbase × Band D``, where + taxbase is Band D equivalent dwellings adjusted for ~7 + discount, premium, and exemption classes (single-person, + disabled relief, second-home, empty-home premium, family + annexe, etc.). Wales uses WG-published net council tax income + direct. + * Matrix col is FRS-reported ``council_tax_less_benefit`` + (household-reported gross less reported CTB). + * Same intent (what households pay net of CTR), different + construction paths and underlying microdata sources. + +Known coverage gaps: + +- Northern Ireland is excluded because its domestic rates system is + distinct from council tax. ``loss.py`` masks NI cells rather than + fabricating a fallback. +- Band-count rows for Scottish LAs are absent because the VOA summary + tables do not cover Scotland; Scottish Assessors publishes per-LA + chargeable-dwellings data separately and is a follow-up. +- Band I only exists in Wales (introduced in the 2005 Welsh revaluation); + English rows leave it null. +- City of London has Band A suppressed by VOA for disclosure control; + its other bands are populated. + +Sources: +- MHCLG *Council Tax levels set by local authorities in England 2026-27* + https://www.gov.uk/government/statistics/council-tax-levels-set-by-local-authorities-in-england-2026-to-2027 +- MHCLG *Council Taxbase 2025 in England* (Table 1.35 taxbase after CTR) + https://www.gov.uk/government/statistics/council-taxbase-2025-in-england +- Welsh Government *Council Tax levels: April 2026 to March 2027* + https://www.gov.wales/council-tax-levels-april-2026-march-2027-html +- Scottish Government *Council Tax Assumptions 2025* (CT by Band, 2025-26) + https://www.gov.scot/publications/council-tax-datasets/ +- VOA *Council Tax: Stock of Properties, 2025* + https://www.gov.uk/government/statistics/council-tax-stock-of-properties-2025 +""" + +from __future__ import annotations + +from functools import lru_cache + +import pandas as pd + +from policyengine_uk_data.targets.schema import ( + GeographicLevel, + Target, + Unit, +) +from policyengine_uk_data.targets.sources._common import STORAGE + + +_CSV_NAME = "la_council_tax.csv" + +# Latest fiscal years covered by each source. The LA Band D amounts are +# structurally single-year snapshots; callers that need longer time +# series should uprate via the existing council-tax uprating index. +_YEAR_BAND_D_ENGLAND = 2026 +_YEAR_BAND_D_WALES = 2026 +_YEAR_BAND_D_SCOTLAND = 2025 +_YEAR_BAND_COUNT = 2025 + +_BAND_COUNT_COLUMNS = {band: f"count_band_{band}" for band in "ABCDEFGHI"} + +_ENGLAND_REF = ( + "https://www.gov.uk/government/statistics/" + "council-tax-levels-set-by-local-authorities-in-england-2026-to-2027" +) +_WALES_REF = "https://www.gov.wales/council-tax-levels-april-2026-march-2027-html" +_SCOTLAND_REF = "https://www.gov.scot/publications/council-tax-datasets/" +_VOA_REF = ( + "https://www.gov.uk/government/statistics/council-tax-stock-of-properties-2025" +) +# Net council tax requirement per LA. England derived from MHCLG +# Council Taxbase 2025 Table 1.35 ("Tax base after allowance for council +# tax support") × LA Band D amount. Wales sourced directly from the +# Welsh Government Table 3 "Council tax income (£m)" — already net. +_NET_CT_REF_ENG = ( + "https://www.gov.uk/government/statistics/council-taxbase-2025-in-england" +) +_NET_CT_REF_WAL = _WALES_REF + + +@lru_cache(maxsize=1) +def _load_table() -> pd.DataFrame | None: + """Return the committed LA council-tax table, or ``None`` if missing.""" + csv_path = STORAGE / _CSV_NAME + if not csv_path.exists(): + return None + return pd.read_csv(csv_path) + + +def load_la_net_council_tax() -> pd.DataFrame: + """Load per-LA net council tax requirement (£, after CTR support). + + Returns a DataFrame with columns ``code, total_council_tax_net`` + for LAs where a directly-observed net figure is available + (England + Wales). Scotland and NI are absent; loss-matrix callers + should mask those cells rather than fabricating fallback values. + """ + df = _load_table() + if df is None or df.empty: + return pd.DataFrame(columns=["code", "total_council_tax_net"]) + if "total_council_tax_net" not in df.columns: + return pd.DataFrame(columns=["code", "total_council_tax_net"]) + return df.loc[ + df["total_council_tax_net"].notna(), + ["code", "total_council_tax_net"], + ].reset_index(drop=True) + + +def _year_for_band_d(country: str) -> int: + if country == "WALES": + return _YEAR_BAND_D_WALES + if country == "SCOTLAND": + return _YEAR_BAND_D_SCOTLAND + return _YEAR_BAND_D_ENGLAND + + +def _ref_for_band_d(country: str) -> str: + if country == "WALES": + return _WALES_REF + if country == "SCOTLAND": + return _SCOTLAND_REF + return _ENGLAND_REF + + +def get_targets() -> list[Target]: + """Emit LA-level Band D amount + band-count targets.""" + df = _load_table() + if df is None or df.empty: + return [] + + targets: list[Target] = [] + + # Band D amount targets — one per LA with a reported value. + for _, row in df.iterrows(): + amount = row.get("band_d_amount") + if pd.isna(amount): + continue + code = str(row["code"]) + country = str(row["country"]) + targets.append( + Target( + name=f"ons/council_tax_band_d/{code}", + variable="council_tax_band_d_amount", + source="ons", + unit=Unit.GBP, + geographic_level=GeographicLevel.LOCAL_AUTHORITY, + geo_code=code, + geo_name=str(row["name"]), + values={_year_for_band_d(country): float(amount)}, + reference_url=_ref_for_band_d(country), + ) + ) + + # Band count targets — one per (LA, band) where VOA has a value. + for _, row in df.iterrows(): + code = str(row["code"]) + name = str(row["name"]) + for band, col in _BAND_COUNT_COLUMNS.items(): + count = row.get(col) + if pd.isna(count): + continue + targets.append( + Target( + name=f"voa/council_tax/{code}/{band}", + variable="council_tax_band", + source="voa", + unit=Unit.COUNT, + geographic_level=GeographicLevel.LOCAL_AUTHORITY, + geo_code=code, + geo_name=name, + values={_YEAR_BAND_COUNT: float(count)}, + is_count=True, + reference_url=_VOA_REF, + ) + ) + + # Net council tax £ targets — one per LA with an observed value. + # Mirrors the FRS net-of-CTR amount; pairs with the band targets + # above to cover both FRS council-tax data points. + if "total_council_tax_net" in df.columns: + for _, row in df.iterrows(): + net = row.get("total_council_tax_net") + if pd.isna(net): + continue + country = str(row["country"]) + ref = _NET_CT_REF_WAL if country == "WALES" else _NET_CT_REF_ENG + targets.append( + Target( + name=f"housing/council_tax_net/{row['code']}", + variable="council_tax_less_benefit", + source="mhclg" if country == "ENGLAND" else "stats_wales", + unit=Unit.GBP, + geographic_level=GeographicLevel.LOCAL_AUTHORITY, + geo_code=str(row["code"]), + geo_name=str(row["name"]), + values={_YEAR_BAND_D_ENGLAND: float(net)}, + reference_url=ref, + ) + ) + + return targets diff --git a/policyengine_uk_data/tests/test_calibrate_save.py b/policyengine_uk_data/tests/test_calibrate_save.py index 20eadb8e5..6b1c04334 100644 --- a/policyengine_uk_data/tests/test_calibrate_save.py +++ b/policyengine_uk_data/tests/test_calibrate_save.py @@ -120,3 +120,42 @@ def test_calibrate_local_areas_saves_weights_in_nonverbose_branch( # Verify the saved weights have the area_count x n_households shape # produced by the calibrator. assert weights.shape == (2, 4) + + +def test_calibrate_local_areas_masks_nan_local_targets(tmp_path, monkeypatch): + """Sparse local targets should be allowed. + + Local-authority sources are not available for every area/metric pair. + A NaN target means "do not train on this cell", not "propagate NaN + through the loss". + """ + + import h5py + + from policyengine_uk_data.utils import calibrate as calibrate_module + from policyengine_uk_data.utils.calibrate import calibrate_local_areas + + monkeypatch.setattr(calibrate_module, "STORAGE_FOLDER", tmp_path) + + matrix_fn, national_matrix_fn = _make_toy_inputs(n_households=4, area_count=2) + + def sparse_matrix_fn(dataset): + matrix, local_targets, country_mask = matrix_fn(dataset) + local_targets.iloc[1, 0] = np.nan + return matrix, local_targets, country_mask + + weight_file = "toy_sparse_weights.h5" + calibrate_local_areas( + dataset=_StubDataset(np.array([1.0, 1.0, 1.0, 1.0])), + matrix_fn=sparse_matrix_fn, + national_matrix_fn=national_matrix_fn, + area_count=2, + weight_file=weight_file, + dataset_key="2025", + epochs=5, + verbose=False, + ) + + with h5py.File(tmp_path / weight_file, "r") as f: + weights = f["2025"][:] + assert np.isfinite(weights).all() diff --git a/policyengine_uk_data/tests/test_la_council_tax_targets.py b/policyengine_uk_data/tests/test_la_council_tax_targets.py new file mode 100644 index 000000000..7cdc59e7b --- /dev/null +++ b/policyengine_uk_data/tests/test_la_council_tax_targets.py @@ -0,0 +1,292 @@ +"""Tests for LA-level council-tax targets. + +Covers both the canonical CSV (``storage/la_council_tax.csv``) and the +``get_targets`` module output. Includes bound checks to guard against +the kind of outlier that slipped into #371 (Isles of Scilly household +count inflated 2000x by a silently leaked national-fallback). +""" + +from __future__ import annotations + +import pandas as pd +import pytest + +from policyengine_uk_data.targets.schema import GeographicLevel, Unit +from policyengine_uk_data.targets.sources._common import STORAGE +from policyengine_uk_data.targets.sources.la_council_tax import ( + _BAND_COUNT_COLUMNS, + get_targets, +) + + +_CSV_NAME = "la_council_tax.csv" +_REFERENCE_LIST = "local_authorities_2021.csv" + + +@pytest.fixture(scope="module") +def la_ct_df() -> pd.DataFrame: + path = STORAGE / _CSV_NAME + if not path.exists(): + pytest.skip(f"{path} not present in this checkout") + return pd.read_csv(path) + + +@pytest.fixture(scope="module") +def la_reference() -> pd.DataFrame: + return pd.read_csv(STORAGE / _REFERENCE_LIST) + + +# -- CSV structure --------------------------------------------------------- + + +def test_csv_row_count_matches_local_authorities(la_ct_df, la_reference): + assert len(la_ct_df) == len(la_reference), ( + f"la_council_tax.csv has {len(la_ct_df)} rows but " + f"local_authorities_2021.csv has {len(la_reference)}." + ) + + +def test_csv_has_every_expected_column(la_ct_df): + expected = { + "code", + "name", + "country", + "band_d_amount", + "has_council_tax", + "count_band_A", + "count_band_B", + "count_band_C", + "count_band_D", + "count_band_E", + "count_band_F", + "count_band_G", + "count_band_H", + "count_band_I", + "total_dwellings", + } + assert expected.issubset(la_ct_df.columns), ( + f"Missing columns: {expected - set(la_ct_df.columns)}" + ) + + +def test_country_column_covers_four_uk_countries(la_ct_df): + assert set(la_ct_df["country"].unique()) == { + "ENGLAND", + "WALES", + "SCOTLAND", + "NORTHERN_IRELAND", + } + + +def test_every_code_matches_reference(la_ct_df, la_reference): + assert set(la_ct_df["code"]) == set(la_reference["code"]), ( + "LA codes in la_council_tax.csv differ from the reference list" + ) + + +# -- Value plausibility (the #371 lesson) ---------------------------------- + + +def test_band_d_amount_within_plausible_range(la_ct_df): + """Real UK Band D ranges from ~£1,000 (Westminster / Wandsworth) to + ~£3,000 (some parts of England). A 1000× outlier like the #371 IoS + households bug would blow through these bounds instantly. + """ + ok = la_ct_df.dropna(subset=["band_d_amount"]) + assert ok["band_d_amount"].between(900, 3_500).all(), ( + "Band D amount outliers: " + f"{ok.loc[~ok['band_d_amount'].between(900, 3_500), ['code', 'name', 'band_d_amount']].to_dict('records')}" + ) + + +def test_total_dwellings_within_plausible_range(la_ct_df): + """Smallest UK billing authority (Isles of Scilly) has ~1,000 dwellings; + largest (Birmingham) has ~470,000. Anything outside [200, 800_000] + is a data-pipeline bug. + """ + ok = la_ct_df.dropna(subset=["total_dwellings"]) + assert ok["total_dwellings"].between(200, 800_000).all(), ( + "Total-dwelling outliers: " + f"{ok.loc[~ok['total_dwellings'].between(200, 800_000), ['code', 'name', 'total_dwellings']].to_dict('records')}" + ) + + +def test_isles_of_scilly_dwellings_are_thousands_not_millions(la_ct_df): + """Explicit regression test for the #371 Isles of Scilly bug. + + IoS had 2,492,115 households in #371's la_land_values.csv because a + national-total fallback leaked into one row. Real IoS has ~1,100 + dwellings. + """ + ios = la_ct_df[la_ct_df["code"] == "E06000053"] + assert len(ios) == 1 + total = float(ios["total_dwellings"].iloc[0]) + assert 500 <= total <= 5_000, ( + f"IoS total_dwellings = {total:.0f}; real figure is ~1,100" + ) + + +def test_band_counts_sum_to_total(la_ct_df): + """Σ(count_band_A..I) should equal the VOA-sourced total_dwellings + for every row with band data. VOA rounds per-band counts to the + nearest 10 while the total is independently rounded, so up to + 20 dwellings of slack is expected. + """ + have_bands = la_ct_df.dropna(subset=["total_dwellings"]).copy() + band_cols = list(_BAND_COUNT_COLUMNS.values()) + have_bands["sum_bands"] = have_bands[band_cols].fillna(0).sum(axis=1) + diff = (have_bands["total_dwellings"] - have_bands["sum_bands"]).abs() + assert diff.max() <= 20, ( + f"Band totals disagree by up to {int(diff.max())}; worst rows: " + f"{have_bands.loc[diff == diff.max(), ['code', 'name', 'total_dwellings', 'sum_bands']].head(3).to_dict('records')}" + ) + + +# -- Country-level coverage expectations ----------------------------------- + + +def test_all_english_las_have_band_d(la_ct_df): + """Every English LA should have a Band D figure from DLUHC.""" + eng = la_ct_df[la_ct_df["country"] == "ENGLAND"] + missing = eng[eng["band_d_amount"].isna()][["code", "name"]].to_dict("records") + assert not missing, f"English LAs missing Band D: {missing}" + + +def test_all_welsh_las_have_band_d(la_ct_df): + wa = la_ct_df[la_ct_df["country"] == "WALES"] + missing = wa[wa["band_d_amount"].isna()][["code", "name"]].to_dict("records") + assert not missing, f"Welsh LAs missing Band D: {missing}" + + +def test_all_scottish_las_have_band_d(la_ct_df): + sc = la_ct_df[la_ct_df["country"] == "SCOTLAND"] + missing = sc[sc["band_d_amount"].isna()][["code", "name"]].to_dict("records") + assert not missing, f"Scottish LAs missing Band D: {missing}" + + +def test_northern_ireland_has_no_council_tax(la_ct_df): + """NI uses domestic rates, not council tax. The CSV must reflect that.""" + ni = la_ct_df[la_ct_df["country"] == "NORTHERN_IRELAND"] + assert (ni["has_council_tax"] == False).all() + assert ni["band_d_amount"].isna().all() + + +# -- Spot-check published values ------------------------------------------- + + +def test_wandsworth_and_westminster_are_lowest_in_england(la_ct_df): + """Well-known political fact: Wandsworth and Westminster have the + lowest Band D in the UK. Catches data-join mistakes that swap rows. + """ + eng = ( + la_ct_df[la_ct_df["country"] == "ENGLAND"] + .dropna(subset=["band_d_amount"]) + .sort_values("band_d_amount") + ) + lowest_two = set(eng.head(2)["code"].tolist()) + assert "E09000032" in lowest_two, ( + "Wandsworth (E09000032) should be in the bottom two" + ) + assert "E09000033" in lowest_two, ( + "Westminster (E09000033) should be in the bottom two" + ) + + +def test_welsh_las_have_band_i(la_ct_df): + """Wales has council tax bands A–I (2005 revaluation); every Welsh + LA must carry a non-null count_band_I. Regression for the Band I + drop that slipped into an earlier revision of this PR. + """ + welsh = la_ct_df[la_ct_df["country"] == "WALES"] + missing = welsh[welsh["count_band_I"].isna()][["code", "name"]].to_dict("records") + assert not missing, f"Welsh LAs missing Band I: {missing}" + + +def test_english_las_have_no_band_i(la_ct_df): + """England has 8 council tax bands (A–H). count_band_I must be null + for every English row so we don't accidentally inject made-up counts. + """ + eng = la_ct_df[la_ct_df["country"] == "ENGLAND"] + populated = eng[eng["count_band_I"].notna()][["code", "name"]].to_dict("records") + assert not populated, f"English LAs must not have Band I: {populated}" + + +def test_cardiff_band_i_matches_published_figure(la_ct_df): + """Cardiff is the largest Welsh LA and has the highest Band I count + (~1,480 per VOA 2025). Specific spot-check against the published value + so a truncated or swapped-row source is caught immediately. + """ + cardiff = la_ct_df[la_ct_df["code"] == "W06000015"] + assert len(cardiff) == 1 + band_i = float(cardiff["count_band_I"].iloc[0]) + assert 1_400 <= band_i <= 1_600, ( + f"Cardiff count_band_I = {band_i:.0f}; VOA 2025 publishes 1,480" + ) + + +def test_scottish_band_d_is_lower_than_english_on_average(la_ct_df): + """Scotland's Band D is typically ~£1,500, well below England ~£2,400.""" + en_mean = la_ct_df[la_ct_df["country"] == "ENGLAND"]["band_d_amount"].mean() + sc_mean = la_ct_df[la_ct_df["country"] == "SCOTLAND"]["band_d_amount"].mean() + assert sc_mean < en_mean - 500 + + +# -- get_targets() output -------------------------------------------------- + + +def test_get_targets_runs_without_network(): + targets = get_targets() + assert targets, "Expected get_targets() to return a non-empty list" + + +def test_band_d_target_count_matches_csv(la_ct_df): + targets = get_targets() + band_d_targets = [t for t in targets if "council_tax_band_d/" in t.name] + expected = int(la_ct_df["band_d_amount"].notna().sum()) + assert len(band_d_targets) == expected + + +def test_band_count_target_count_matches_csv(la_ct_df): + targets = get_targets() + bc_targets = [t for t in targets if t.name.startswith("voa/council_tax/")] + expected = int(la_ct_df[list(_BAND_COUNT_COLUMNS.values())].notna().sum().sum()) + assert len(bc_targets) == expected + + +def test_every_target_carries_local_authority_geo_level(): + for target in get_targets(): + assert target.geographic_level == GeographicLevel.LOCAL_AUTHORITY + assert target.geo_code is not None + + +def test_band_d_targets_use_gbp_unit(): + for target in get_targets(): + if "council_tax_band_d/" in target.name: + assert target.unit == Unit.GBP + + +def test_band_count_targets_use_count_unit_and_is_count_flag(): + for target in get_targets(): + if target.name.startswith("voa/council_tax/"): + assert target.unit == Unit.COUNT + assert target.is_count is True + + +def test_every_target_has_at_least_one_value_year(): + for target in get_targets(): + assert target.values, f"Target {target.name} has no values" + + +def test_every_band_count_target_value_within_sensible_range(): + """No single band within one LA should exceed the largest LA's total + dwelling stock (Birmingham ≈ 470k). This catches a fallback-leak + where a national total leaked into a row, à la #371. + """ + for target in get_targets(): + if not target.name.startswith("voa/council_tax/"): + continue + for year, value in target.values.items(): + assert 0 <= value <= 500_000, ( + f"{target.name} has band count {value} in {year} — " + "out of plausible [0, 500k] range" + ) diff --git a/policyengine_uk_data/tests/test_la_loss_council_tax.py b/policyengine_uk_data/tests/test_la_loss_council_tax.py new file mode 100644 index 000000000..c33949e06 --- /dev/null +++ b/policyengine_uk_data/tests/test_la_loss_council_tax.py @@ -0,0 +1,417 @@ +"""Tests for the LA-level council-tax band-count columns wired into the +local-authority calibration loss matrix. + +Layered like test_la_loss_land_value.py: + +1. Light checks against la_council_tax.csv — exercise the data shape + the loss-matrix code relies on without needing a Microsimulation. +2. Full ``create_local_authority_target_matrix`` build, gated on the + enhanced FRS fixture so CI environments without the dataset skip + gracefully. + +Note: only bands A-H are wired. Band I is Wales-only and mostly null, +and the Band D ``£`` amount is a per-rate quantity that does not fit +the linear matrix-times-weights aggregation — both are deliberately +out of scope for this PR. +""" + +import numpy as np +import pandas as pd +import pytest + +from policyengine_uk_data.storage import STORAGE_FOLDER + + +CT_DATA = pd.read_csv(STORAGE_FOLDER / "la_council_tax.csv") +LA_CODES = pd.read_csv(STORAGE_FOLDER / "local_authorities_2021.csv") +WIRED_BANDS = list("ABCDEFGH") + + +# ── Layer 1: CSV shape and joinability ─────────────────────────────── + + +def test_csv_has_a_row_for_every_la_code(): + """The CSV must cover every LA in local_authorities_2021.csv so the + left-merge inside loss.py never produces NaN-only rows in the + has_count branch.""" + missing = set(LA_CODES["code"]) - set(CT_DATA["code"]) + assert not missing, ( + f"LA codes missing from la_council_tax.csv: {sorted(missing)[:5]}" + ) + + +def test_band_count_columns_exist_for_every_wired_band(): + """Every wired band needs a count_band_{X} column, otherwise the loss + matrix loop will KeyError on missing CSV columns.""" + for band in WIRED_BANDS: + assert f"count_band_{band}" in CT_DATA.columns + + +def test_england_and_wales_have_band_a_to_h_populated(): + """E/W rows should have non-null counts for A-H. If the CSV regresses + to NaN there, the loss matrix will silently fall back to the + national-share estimate and the calibrator loses its real signal.""" + ew = CT_DATA[CT_DATA["country"].isin(["ENGLAND", "WALES"])] + for band in WIRED_BANDS: + non_null = ew[f"count_band_{band}"].notna().sum() + # City of London suppresses Band A; allow up to 5 missing per band. + assert non_null >= len(ew) - 5, ( + f"Band {band}: only {non_null}/{len(ew)} E/W LAs have a count" + ) + + +def test_scotland_band_counts_are_null_as_documented(): + """Scotland VOA band counts are absent — they should consistently be + NaN so the loss matrix routes them through the fallback.""" + scotland = CT_DATA[CT_DATA["country"] == "SCOTLAND"] + for band in WIRED_BANDS: + assert scotland[f"count_band_{band}"].isna().all(), ( + f"Band {band}: Scotland rows unexpectedly populated" + ) + + +def test_ni_council_tax_disabled(): + """NI uses domestic rates, not council tax. has_council_tax must be + False for every NI row, otherwise downstream code may try to read + targets that do not exist.""" + ni = CT_DATA[CT_DATA["country"] == "NORTHERN_IRELAND"] + assert not ni.empty + assert (ni["has_council_tax"] == False).all() # noqa: E712 + + +# ── Layer 2: full LA loss matrix build ─────────────────────────────── + + +def test_la_loss_matrix_includes_all_wired_band_columns(enhanced_frs): + """matrix and y must expose voa/council_tax/{A..H} for the calibrator + to train on the new targets.""" + from policyengine_uk_data.datasets.local_areas.local_authorities.loss import ( + create_local_authority_target_matrix, + ) + + matrix, y, _ = create_local_authority_target_matrix( + enhanced_frs, time_period=enhanced_frs.time_period + ) + for band in WIRED_BANDS: + col = f"voa/council_tax/{band}" + assert col in matrix.columns, f"missing matrix column {col}" + assert col in y.columns, f"missing y column {col}" + + +def test_la_loss_band_y_vectors_length_360(enhanced_frs): + from policyengine_uk_data.datasets.local_areas.local_authorities.loss import ( + create_local_authority_target_matrix, + ) + + _, y, _ = create_local_authority_target_matrix( + enhanced_frs, time_period=enhanced_frs.time_period + ) + for band in WIRED_BANDS: + assert len(y[f"voa/council_tax/{band}"]) == 360 + + +def test_la_loss_band_y_direct_cells_finite(enhanced_frs): + """Direct band-count cells are finite and non-negative. + + Missing-source cells stay NaN so the calibrator can mask them out + instead of training on fabricated national-share fallbacks. + """ + from policyengine_uk_data.datasets.local_areas.local_authorities.loss import ( + create_local_authority_target_matrix, + ) + + _, y, _ = create_local_authority_target_matrix( + enhanced_frs, time_period=enhanced_frs.time_period + ) + + ni_indices = LA_CODES.index[ + LA_CODES["code"].isin( + CT_DATA.loc[CT_DATA["country"] == "NORTHERN_IRELAND", "code"] + ) + ] + for band in WIRED_BANDS: + col = f"voa/council_tax/{band}" + direct = y[col].dropna() + assert np.isfinite(direct).all(), f"{col}: inf in direct y cells" + assert (direct >= 0).all(), f"{col}: negative direct y values" + assert y[col].iloc[ni_indices].isna().all(), ( + f"{col}: NI cells should be masked, not zero-filled" + ) + + +def test_la_loss_band_matrix_columns_are_indicators(enhanced_frs): + """matrix entries are 0 or 1 (the household either is or isn't in band X).""" + from policyengine_uk_data.datasets.local_areas.local_authorities.loss import ( + create_local_authority_target_matrix, + ) + + matrix, _, _ = create_local_authority_target_matrix( + enhanced_frs, time_period=enhanced_frs.time_period + ) + for band in WIRED_BANDS: + col = f"voa/council_tax/{band}" + unique = set(np.unique(matrix[col].values)) + assert unique <= {0.0, 1.0}, f"{col}: non-indicator values {unique}" + + +def test_la_loss_band_matrix_rows_sum_to_at_most_one(enhanced_frs): + """Each household sits in at most one band. Summing the wired band + columns across each household should be 0 or 1.""" + from policyengine_uk_data.datasets.local_areas.local_authorities.loss import ( + create_local_authority_target_matrix, + ) + + matrix, _, _ = create_local_authority_target_matrix( + enhanced_frs, time_period=enhanced_frs.time_period + ) + band_sum = sum(matrix[f"voa/council_tax/{b}"] for b in WIRED_BANDS) + assert ((band_sum == 0) | (band_sum == 1)).all() + + +def test_la_loss_band_y_matches_csv_for_english_la(enhanced_frs): + """For an English LA with VOA data, y[band] must be the CSV value + verbatim — not the national-share fallback.""" + from policyengine_uk_data.datasets.local_areas.local_authorities.loss import ( + create_local_authority_target_matrix, + ) + + _, y, _ = create_local_authority_target_matrix( + enhanced_frs, time_period=enhanced_frs.time_period + ) + + # Hartlepool — first LA in local_authorities_2021.csv, VOA-covered. + target_code = "E06000001" + la_index = LA_CODES.index[LA_CODES["code"] == target_code][0] + ct_row = CT_DATA[CT_DATA["code"] == target_code].iloc[0] + for band in WIRED_BANDS: + expected = float(ct_row[f"count_band_{band}"]) + actual = float(y[f"voa/council_tax/{band}"].iloc[la_index]) + assert actual == pytest.approx(expected, rel=0, abs=0.5), ( + f"{target_code} band {band}: y={actual}, CSV={expected}" + ) + + +def test_la_loss_band_y_masks_scotland_without_direct_source(enhanced_frs): + """Scottish LAs have no VOA band counts; y must be NaN so the + calibrator excludes those cells instead of inventing a fallback.""" + from policyengine_uk_data.datasets.local_areas.local_authorities.loss import ( + create_local_authority_target_matrix, + ) + + _, y, _ = create_local_authority_target_matrix( + enhanced_frs, time_period=enhanced_frs.time_period + ) + + scotland_la = CT_DATA[CT_DATA["country"] == "SCOTLAND"]["code"].iloc[0] + la_index = LA_CODES.index[LA_CODES["code"] == scotland_la][0] + for band in WIRED_BANDS: + val = y[f"voa/council_tax/{band}"].iloc[la_index] + assert pd.isna(val), ( + f"Scotland LA {scotland_la} band {band}: expected masked NaN, got {val}" + ) + + +def test_la_loss_band_y_masks_ni(enhanced_frs): + """NI LAs have no council tax (domestic rates instead). Band targets + must be masked. A zero target is still a training constraint and can + be impossible if the matrix-side band variable is non-zero.""" + from policyengine_uk_data.datasets.local_areas.local_authorities.loss import ( + create_local_authority_target_matrix, + ) + + _, y, _ = create_local_authority_target_matrix( + enhanced_frs, time_period=enhanced_frs.time_period + ) + + ni_codes = CT_DATA[CT_DATA["country"] == "NORTHERN_IRELAND"]["code"] + for ni_la in ni_codes: + la_index = LA_CODES.index[LA_CODES["code"] == ni_la][0] + for band in WIRED_BANDS: + val = y[f"voa/council_tax/{band}"].iloc[la_index] + assert pd.isna(val), ( + f"NI LA {ni_la} band {band}: expected masked NaN, got {val}" + ) + + +# ── Council tax £ paid (net of CTR) ───────────────────────────────── + + +def test_csv_has_net_council_tax_column(): + """The CSV must expose total_council_tax_net so loss.py can wire it.""" + assert "total_council_tax_net" in CT_DATA.columns + + +def test_net_council_tax_covers_england_and_wales(): + """Direct-formula values are produced for England (MHCLG taxbase × Band D) + and Wales (Welsh Government Council Tax Income). Scotland and NI are + masked in loss.py unless direct values are added.""" + cov = ( + CT_DATA.assign(has_net=CT_DATA["total_council_tax_net"].notna()) + .groupby("country")["has_net"] + .agg(["sum", "count"]) + ) + assert cov.loc["ENGLAND", "sum"] == cov.loc["ENGLAND", "count"] + assert cov.loc["WALES", "sum"] == cov.loc["WALES", "count"] + assert cov.loc["SCOTLAND", "sum"] == 0 + assert cov.loc["NORTHERN_IRELAND", "sum"] == 0 + + +def test_net_council_tax_value_range(): + """Per-LA net council tax should be in £2m–£1.5bn. Lower bound is + set by Isles of Scilly (~£3m on ~1,100 households); upper bound by + Birmingham (~£700m). A 1000x outlier — like the IoS fallback leak + pre-review on #371 — must be caught by bounds, not spotted by eye.""" + covered = CT_DATA["total_council_tax_net"].dropna() + out_of_range = covered[(covered < 2e6) | (covered > 1.5e9)] + assert out_of_range.empty, ( + f"Net CT outside [£2m, £1.5bn]: {out_of_range.tolist()[:3]}" + ) + + +def test_england_net_total_in_range_of_mhclg_summary(): + """Sum of England LA net targets should be within ~5% of MHCLG's + published England Council Tax Requirement (~£45.86bn for 2026-27; + we compute from 2025 taxbase × 2026-27 Band D so a small year-mismatch + gap is expected).""" + eng_total = CT_DATA.loc[ + CT_DATA["country"] == "ENGLAND", "total_council_tax_net" + ].sum() + assert 4.3e10 < eng_total < 5.0e10, ( + f"England net total £{eng_total / 1e9:.2f}bn outside [£43bn, £50bn]" + ) + + +def test_la_loss_matrix_includes_council_tax_net(enhanced_frs): + """matrix and y must expose housing/council_tax_net so the + calibrator trains on the net £-amount target alongside band counts.""" + from policyengine_uk_data.datasets.local_areas.local_authorities.loss import ( + create_local_authority_target_matrix, + ) + + matrix, y, _ = create_local_authority_target_matrix( + enhanced_frs, time_period=enhanced_frs.time_period + ) + assert "housing/council_tax_net" in matrix.columns + assert "housing/council_tax_net" in y.columns + + +def test_la_loss_council_tax_net_matrix_uses_net_variable(enhanced_frs): + """Matrix col must be council_tax_less_benefit (net of CTR), so both + sides of the calibration constraint are net per the 28 Apr standup + decision on FRS-net-of-CTR alignment.""" + from policyengine_uk import Microsimulation + from policyengine_uk_data.datasets.local_areas.local_authorities.loss import ( + create_local_authority_target_matrix, + ) + + matrix, _, _ = create_local_authority_target_matrix( + enhanced_frs, time_period=enhanced_frs.time_period + ) + + sim = Microsimulation(dataset=enhanced_frs) + sim.default_calculation_period = enhanced_frs.time_period + expected = sim.calculate("council_tax_less_benefit").values + np.testing.assert_array_equal(matrix["housing/council_tax_net"].values, expected) + + +def test_la_loss_council_tax_net_y_direct_cells_finite(enhanced_frs): + """Direct net-council-tax cells are finite and non-negative. + + Missing-source cells stay NaN so the calibrator excludes them from + training instead of using national-share fallbacks or hard zeroes. + """ + from policyengine_uk_data.datasets.local_areas.local_authorities.loss import ( + create_local_authority_target_matrix, + ) + + _, y, _ = create_local_authority_target_matrix( + enhanced_frs, time_period=enhanced_frs.time_period + ) + col = y["housing/council_tax_net"] + direct = col.dropna() + assert np.isfinite(direct).all() + assert (direct >= 0).all() + + ni_indices = LA_CODES.index[ + LA_CODES["code"].isin( + CT_DATA.loc[CT_DATA["country"] == "NORTHERN_IRELAND", "code"] + ) + ] + assert col.iloc[ni_indices].isna().all(), ( + "NI LAs should be masked for housing/council_tax_net " + "(NI uses domestic rates, not council tax)" + ) + assert (direct > 0).all(), "direct housing/council_tax_net cells should be positive" + + +def test_la_loss_council_tax_net_y_matches_csv_for_english_la(enhanced_frs): + """For a covered (English) LA, y must equal the CSV value verbatim + rather than the national-share fallback.""" + from policyengine_uk_data.datasets.local_areas.local_authorities.loss import ( + create_local_authority_target_matrix, + ) + + _, y, _ = create_local_authority_target_matrix( + enhanced_frs, time_period=enhanced_frs.time_period + ) + + target_code = "E06000001" # Hartlepool + la_index = LA_CODES.index[LA_CODES["code"] == target_code][0] + expected = float( + CT_DATA.loc[CT_DATA["code"] == target_code, "total_council_tax_net"].iloc[0] + ) + actual = float(y["housing/council_tax_net"].iloc[la_index]) + assert actual == pytest.approx(expected, rel=1e-6) + + +def test_la_loss_council_tax_net_y_masks_scotland_without_direct_source(enhanced_frs): + """Scottish LAs have no published net CT in the CSV; the target + should be NaN until a direct source is wired in.""" + from policyengine_uk_data.datasets.local_areas.local_authorities.loss import ( + create_local_authority_target_matrix, + ) + + _, y, _ = create_local_authority_target_matrix( + enhanced_frs, time_period=enhanced_frs.time_period + ) + scotland_code = CT_DATA[CT_DATA["country"] == "SCOTLAND"]["code"].iloc[0] + la_index = LA_CODES.index[LA_CODES["code"] == scotland_code][0] + assert pd.isna(y["housing/council_tax_net"].iloc[la_index]) + + +def test_la_loss_english_council_tax_net_in_reach_of_initial_weights( + enhanced_frs, +): + """Sum of English LA net council-tax targets should be in the same + order of magnitude (0.3x–3x) as the implied initial weighted English + council_tax_less_benefit total — so the calibrator can reach the + target via reweighting rather than 100x weight inflation.""" + from policyengine_uk import Microsimulation + from policyengine_uk_data.datasets.local_areas.local_authorities.loss import ( + create_local_authority_target_matrix, + ) + + _, y, _ = create_local_authority_target_matrix( + enhanced_frs, time_period=enhanced_frs.time_period + ) + + sim = Microsimulation(dataset=enhanced_frs) + weights = sim.calculate("household_weight", 2025).values + ct_net = sim.calculate("council_tax_less_benefit", enhanced_frs.time_period).values + country = sim.calculate("country", enhanced_frs.time_period).values + england_initial = ( + weights[country == "ENGLAND"] * ct_net[country == "ENGLAND"] + ).sum() + + english_indices = [ + i for i, c in enumerate(LA_CODES["code"].values) if c.startswith("E0") + ] + english_target_sum = y["housing/council_tax_net"].iloc[english_indices].sum() + + if england_initial > 0: + ratio = english_target_sum / england_initial + assert 0.3 < ratio < 3.0, ( + f"England target sum (£{english_target_sum / 1e9:.1f}bn) / " + f"initial weighted England net CT (£{england_initial / 1e9:.1f}bn) " + f"= {ratio:.2f}; calibration target may be hard to reach" + ) diff --git a/policyengine_uk_data/tests/test_obr_council_tax.py b/policyengine_uk_data/tests/test_obr_council_tax.py new file mode 100644 index 000000000..74b054fc3 --- /dev/null +++ b/policyengine_uk_data/tests/test_obr_council_tax.py @@ -0,0 +1,86 @@ +"""Tests for the national OBR council tax compute function. + +OBR EFO Table 4.1 reports "Total net council tax receipts" — net of +council tax reduction (CTR). The matching household-level signal is +``council_tax_less_benefit`` (= gross council tax less the CTR +award), not ``council_tax`` (which is the gross liability). + +These tests pin the matrix column to the net variable so a future +edit cannot silently regress the gross/net mismatch. +""" + +from types import SimpleNamespace + +import numpy as np + + +def _dummy_ctx(council_tax_less_benefit, country): + """Return a context object compatible with compute_obr_council_tax.""" + + class _Ctx: + pass + + ctx = _Ctx() + ctx.country = np.array(country) + + def pe(variable): + if variable == "council_tax_less_benefit": + return np.array(council_tax_less_benefit) + raise AssertionError(f"unexpected pe call: {variable}") + + ctx.pe = pe + return ctx + + +def test_compute_uses_net_variable_not_gross(): + """matrix col for obr/council_tax must be council_tax_less_benefit + so it matches the OBR net target value. Calibrating gross against + a net target systematically pushes weights down to fit.""" + from policyengine_uk_data.targets.compute.council_tax import ( + compute_obr_council_tax, + ) + + ctx = _dummy_ctx( + council_tax_less_benefit=[1000.0, 1500.0, 0.0, 800.0], + country=["ENGLAND", "ENGLAND", "SCOTLAND", "WALES"], + ) + + out = compute_obr_council_tax(SimpleNamespace(name="obr/council_tax"), ctx) + np.testing.assert_array_equal(out, [1000.0, 1500.0, 0.0, 800.0]) + + +def test_compute_country_masks_apply_after_net_extraction(): + """Country variants must zero out non-matching households on the + net variable, not on a gross variable.""" + from policyengine_uk_data.targets.compute.council_tax import ( + compute_obr_council_tax, + ) + + ctx = _dummy_ctx( + council_tax_less_benefit=[1000.0, 1500.0, 600.0, 800.0], + country=["ENGLAND", "ENGLAND", "SCOTLAND", "WALES"], + ) + + eng = compute_obr_council_tax(SimpleNamespace(name="obr/council_tax_england"), ctx) + sco = compute_obr_council_tax(SimpleNamespace(name="obr/council_tax_scotland"), ctx) + wal = compute_obr_council_tax(SimpleNamespace(name="obr/council_tax_wales"), ctx) + + np.testing.assert_array_equal(eng, [1000.0, 1500.0, 0.0, 0.0]) + np.testing.assert_array_equal(sco, [0.0, 0.0, 600.0, 0.0]) + np.testing.assert_array_equal(wal, [0.0, 0.0, 0.0, 800.0]) + + +def test_compute_does_not_call_gross_variable(): + """If a future refactor reintroduces ctx.pe('council_tax'), the + DummyCtx will raise AssertionError. Pins the net-only contract.""" + from policyengine_uk_data.targets.compute.council_tax import ( + compute_obr_council_tax, + ) + + ctx = _dummy_ctx( + council_tax_less_benefit=[100.0, 200.0], + country=["ENGLAND", "WALES"], + ) + + # No exception means only council_tax_less_benefit was queried. + compute_obr_council_tax(SimpleNamespace(name="obr/council_tax"), ctx) diff --git a/policyengine_uk_data/utils/calibrate.py b/policyengine_uk_data/utils/calibrate.py index 3efd944f2..3e144126b 100644 --- a/policyengine_uk_data/utils/calibrate.py +++ b/policyengine_uk_data/utils/calibrate.py @@ -156,7 +156,10 @@ def track_stage(stage_name: str): # Set up validation targets if specified validation_targets_local = ( - matrix.columns.isin(excluded_training_targets) + torch.tensor( + matrix.columns.isin(excluded_training_targets), + dtype=torch.bool, + ) if hasattr(matrix, "columns") else None ) @@ -172,7 +175,9 @@ def track_stage(stage_name: str): matrix.values if hasattr(matrix, "values") else matrix, dtype=torch.float32, ) - y = torch.tensor(y.values if hasattr(y, "values") else y, dtype=torch.float32) + y_values = y.values if hasattr(y, "values") else y + local_target_available = torch.tensor(np.isfinite(y_values), dtype=torch.bool) + y = torch.tensor(np.nan_to_num(y_values, nan=0.0), dtype=torch.float32) matrix_national = torch.tensor( m_national.values if hasattr(m_national, "values") else m_national, dtype=torch.float32, @@ -190,15 +195,18 @@ def sre(x, y): def loss(w, validation: bool = False): pred_local = (w.unsqueeze(-1) * metrics.unsqueeze(0)).sum(dim=1) + local_mask = local_target_available if dropout_targets and validation_targets_local is not None: if validation: - mask = validation_targets_local + column_mask = validation_targets_local else: - mask = ~validation_targets_local - pred_local = pred_local[:, mask] - mse_local = torch.mean(sre(pred_local, y[:, mask])) + column_mask = ~validation_targets_local + local_mask = local_mask & column_mask.unsqueeze(0) + + if local_mask.any(): + mse_local = torch.mean(sre(pred_local[local_mask], y[local_mask])) else: - mse_local = torch.mean(sre(pred_local, y)) + mse_local = pred_local.sum() * 0 pred_national = (w.sum(axis=0) * matrix_national.T).sum(axis=1) if dropout_targets and validation_targets_national is not None: @@ -220,8 +228,10 @@ def pct_close(w, t=0.1, local=True, national=True): if local: pred_local = (w.unsqueeze(-1) * metrics.unsqueeze(0)).sum(dim=1) - e_local = torch.sum(torch.abs((pred_local / (1 + y) - 1)) < t).item() - c_local = pred_local.shape[0] * pred_local.shape[1] + e_local = torch.sum( + (torch.abs((pred_local / (1 + y) - 1)) < t) & local_target_available + ).item() + c_local = torch.sum(local_target_available).item() numerator += e_local denominator += c_local