Skip to content

Commit 465dbcb

Browse files
anth-volkclaude
anth-volk
and
claude
committed
Add UK pipeline diagrams with cross-country comparison
- Add PolicyEngine UK pipeline (10 stages: Overview + Stage 0-8) - Add country selector dropdown (US / UK) in sidebar - Add uk_specific node type (teal) for UK-only processing - Add absent node type (gray dashed) for cross-referencing - Show UK-only steps as absent in US stages (Stage 1, 4) - Show US-only steps as absent in UK stages - Update routes from / to /us and /uk prefixes - Rename app title to "PolicyEngine data pipelines" Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
1 parent fc01dfb commit 465dbcb

16 files changed

Lines changed: 2324 additions & 32 deletions

File tree

‎docs/pipeline-diagrams/index.html‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,7 @@
44
<meta charset="UTF-8" />
55
<link rel="icon" type="image/svg+xml" href="/favicon.svg" />
66
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
7-
<title>pipeline-diagrams</title>
7+
<title>PolicyEngine data pipelines</title>
88
</head>
99
<body>
1010
<div id="root"></div>

‎docs/pipeline-diagrams/src/App.jsx‎

Lines changed: 32 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
import { Routes, Route } from "react-router-dom";
1+
import { Routes, Route, Navigate } from "react-router-dom";
22
import Sidebar from "./components/Sidebar";
33
import Overview from "./stages/Overview";
44
import Stage0 from "./stages/Stage0";
@@ -10,23 +10,44 @@ import Stage5 from "./stages/Stage5";
1010
import Stage6 from "./stages/Stage6";
1111
import Stage7 from "./stages/Stage7";
1212
import Stage8 from "./stages/Stage8";
13+
import UKOverview from "./stages/uk/Overview";
14+
import UKStage0 from "./stages/uk/Stage0";
15+
import UKStage1 from "./stages/uk/Stage1";
16+
import UKStage2 from "./stages/uk/Stage2";
17+
import UKStage3 from "./stages/uk/Stage3";
18+
import UKStage4 from "./stages/uk/Stage4";
19+
import UKStage5 from "./stages/uk/Stage5";
20+
import UKStage6 from "./stages/uk/Stage6";
21+
import UKStage7 from "./stages/uk/Stage7";
22+
import UKStage8 from "./stages/uk/Stage8";
1323

1424
export default function App() {
1525
return (
1626
<div className="flex h-screen w-screen overflow-hidden">
1727
<Sidebar />
1828
<div className="flex-1 overflow-hidden">
1929
<Routes>
20-
<Route path="/" element={<Overview />} />
21-
<Route path="/stage/0" element={<Stage0 />} />
22-
<Route path="/stage/1" element={<Stage1 />} />
23-
<Route path="/stage/2" element={<Stage2 />} />
24-
<Route path="/stage/3" element={<Stage3 />} />
25-
<Route path="/stage/4" element={<Stage4 />} />
26-
<Route path="/stage/5" element={<Stage5 />} />
27-
<Route path="/stage/6" element={<Stage6 />} />
28-
<Route path="/stage/7" element={<Stage7 />} />
29-
<Route path="/stage/8" element={<Stage8 />} />
30+
<Route path="/" element={<Navigate to="/us" replace />} />
31+
<Route path="/us" element={<Overview />} />
32+
<Route path="/us/stage/0" element={<Stage0 />} />
33+
<Route path="/us/stage/1" element={<Stage1 />} />
34+
<Route path="/us/stage/2" element={<Stage2 />} />
35+
<Route path="/us/stage/3" element={<Stage3 />} />
36+
<Route path="/us/stage/4" element={<Stage4 />} />
37+
<Route path="/us/stage/5" element={<Stage5 />} />
38+
<Route path="/us/stage/6" element={<Stage6 />} />
39+
<Route path="/us/stage/7" element={<Stage7 />} />
40+
<Route path="/us/stage/8" element={<Stage8 />} />
41+
<Route path="/uk" element={<UKOverview />} />
42+
<Route path="/uk/stage/0" element={<UKStage0 />} />
43+
<Route path="/uk/stage/1" element={<UKStage1 />} />
44+
<Route path="/uk/stage/2" element={<UKStage2 />} />
45+
<Route path="/uk/stage/3" element={<UKStage3 />} />
46+
<Route path="/uk/stage/4" element={<UKStage4 />} />
47+
<Route path="/uk/stage/5" element={<UKStage5 />} />
48+
<Route path="/uk/stage/6" element={<UKStage6 />} />
49+
<Route path="/uk/stage/7" element={<UKStage7 />} />
50+
<Route path="/uk/stage/8" element={<UKStage8 />} />
3051
</Routes>
3152
</div>
3253
</div>

‎docs/pipeline-diagrams/src/components/Sidebar.jsx‎

Lines changed: 51 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -1,17 +1,30 @@
1-
import { NavLink } from "react-router-dom";
1+
import { NavLink, useNavigate, useLocation } from "react-router-dom";
22
import { EDGE_LEGEND } from "../constants/edgeStyles";
33

4-
const stages = [
5-
{ path: "/", label: "Overview", desc: "Cross-stage data flow" },
6-
{ path: "/stage/0", label: "Stage 0", desc: "Raw Data Download" },
7-
{ path: "/stage/1", label: "Stage 1", desc: "Base Dataset Construction" },
8-
{ path: "/stage/2", label: "Stage 2", desc: "Extended CPS (PUF Clone)" },
9-
{ path: "/stage/3", label: "Stage 3", desc: "Stratified CPS" },
10-
{ path: "/stage/4", label: "Stage 4", desc: "Source Imputation" },
11-
{ path: "/stage/5", label: "Stage 5", desc: "Matrix Build" },
12-
{ path: "/stage/6", label: "Stage 6", desc: "Weight Fitting (L0)" },
13-
{ path: "/stage/7", label: "Stage 7", desc: "Local Area H5 Build" },
14-
{ path: "/stage/8", label: "Stage 8", desc: "Validation & Promotion" },
4+
const usStages = [
5+
{ path: "/us", label: "Overview", desc: "Cross-stage data flow" },
6+
{ path: "/us/stage/0", label: "Stage 0", desc: "Raw Data Download" },
7+
{ path: "/us/stage/1", label: "Stage 1", desc: "Base Dataset Construction" },
8+
{ path: "/us/stage/2", label: "Stage 2", desc: "Extended CPS (PUF Clone)" },
9+
{ path: "/us/stage/3", label: "Stage 3", desc: "Stratified CPS" },
10+
{ path: "/us/stage/4", label: "Stage 4", desc: "Source Imputation" },
11+
{ path: "/us/stage/5", label: "Stage 5", desc: "Matrix Build" },
12+
{ path: "/us/stage/6", label: "Stage 6", desc: "Weight Fitting (L0)" },
13+
{ path: "/us/stage/7", label: "Stage 7", desc: "Local Area H5 Build" },
14+
{ path: "/us/stage/8", label: "Stage 8", desc: "Validation & Promotion" },
15+
];
16+
17+
const ukStages = [
18+
{ path: "/uk", label: "Overview", desc: "Cross-stage data flow" },
19+
{ path: "/uk/stage/0", label: "Stage 0", desc: "Raw Data Download" },
20+
{ path: "/uk/stage/1", label: "Stage 1", desc: "Base Dataset Construction" },
21+
{ path: "/uk/stage/2", label: "Stage 2", desc: "Income Enhancement (SPI)" },
22+
{ path: "/uk/stage/3", label: "Stage 3", desc: "Stratification", absent: true },
23+
{ path: "/uk/stage/4", label: "Stage 4", desc: "Source Imputation" },
24+
{ path: "/uk/stage/5", label: "Stage 5", desc: "Matrix Build" },
25+
{ path: "/uk/stage/6", label: "Stage 6", desc: "Weight Fitting (Torch)" },
26+
{ path: "/uk/stage/7", label: "Stage 7", desc: "Local Area Calibration" },
27+
{ path: "/uk/stage/8", label: "Stage 8", desc: "Validation & Output" },
1528
];
1629

1730
const nodeLegend = [
@@ -22,36 +35,54 @@ const nodeLegend = [
2235
{ color: "bg-red-200 border-red-300 border-dashed", label: "Missing" },
2336
{ color: "bg-yellow-200 border-yellow-400", label: "External" },
2437
{ color: "bg-pink-200 border-pink-400", label: "US-Specific" },
38+
{ color: "bg-teal-200 border-teal-400", label: "UK-Specific" },
39+
{ color: "bg-gray-100 border-gray-300 border-dashed", label: "Absent" },
2540
];
2641

2742
export default function Sidebar() {
43+
const navigate = useNavigate();
44+
const location = useLocation();
45+
const isUK = location.pathname.startsWith("/uk");
46+
const country = isUK ? "uk" : "us";
47+
const stages = isUK ? ukStages : usStages;
48+
2849
return (
2950
<aside className="w-[260px] bg-gray-50 border-r border-gray-200 flex flex-col h-full overflow-y-auto shrink-0">
3051
<div className="px-4 py-4 border-b border-gray-200">
3152
<h1 className="text-base font-bold text-gray-900 m-0">
32-
PE-US-Data Pipeline
53+
PolicyEngine data pipelines
3354
</h1>
34-
<p className="text-[11px] text-gray-500 mt-0.5">
35-
Interactive stage diagrams
36-
</p>
55+
<select
56+
className="mt-2 w-full text-xs border border-gray-300 rounded px-2 py-1.5 bg-white text-gray-700 focus:outline-none focus:ring-1 focus:ring-blue-400"
57+
value={country}
58+
onChange={(e) => navigate(e.target.value === "uk" ? "/uk" : "/us")}
59+
>
60+
<option value="us">PolicyEngine US</option>
61+
<option value="uk">PolicyEngine UK</option>
62+
</select>
3763
</div>
3864

3965
<nav className="flex-1 px-2 py-2 space-y-0.5">
4066
{stages.map((s) => (
4167
<NavLink
4268
key={s.path}
4369
to={s.path}
44-
end={s.path === "/"}
70+
end={s.path === "/us" || s.path === "/uk"}
4571
className={({ isActive }) =>
4672
`block px-3 py-2 rounded text-xs transition-colors ${
4773
isActive
4874
? "bg-blue-100 text-blue-900 font-semibold"
49-
: "text-gray-700 hover:bg-gray-100"
75+
: s.absent
76+
? "text-gray-400 hover:bg-gray-100"
77+
: "text-gray-700 hover:bg-gray-100"
5078
}`
5179
}
5280
>
53-
<div className="font-medium text-[13px]">{s.label}</div>
54-
<div className="text-[10px] text-gray-500 mt-0.5">
81+
<div className={`font-medium text-[13px] ${s.absent ? "text-gray-400" : ""}`}>
82+
{s.label}
83+
{s.absent && " (absent)"}
84+
</div>
85+
<div className={`text-[10px] mt-0.5 ${s.absent ? "text-gray-300" : "text-gray-500"}`}>
5586
{s.desc}
5687
</div>
5788
</NavLink>

‎docs/pipeline-diagrams/src/components/nodes/CustomNodes.jsx‎

Lines changed: 31 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -140,6 +140,35 @@ export function USSpecificNode({ data, selected }) {
140140
);
141141
}
142142

143+
export function UKSpecificNode({ data, selected }) {
144+
return (
145+
<NodeShell bg="bg-teal-50" border="border-teal-400" selected={selected}>
146+
<div className="font-bold text-teal-800 text-[11px] uppercase tracking-wide mb-0.5">
147+
UK-Specific
148+
</div>
149+
<NodeBody data={data} />
150+
</NodeShell>
151+
);
152+
}
153+
154+
export function AbsentNode({ data, selected }) {
155+
return (
156+
<NodeShell bg="bg-gray-50" border="border-gray-300" dashed selected={selected}>
157+
<div className="font-bold text-gray-400 text-[11px] uppercase tracking-wide mb-0.5">
158+
{data.absentFrom === "uk" ? "US only" : "UK only"}
159+
</div>
160+
<div className="font-semibold text-gray-400 text-sm leading-tight">
161+
{data.label}
162+
</div>
163+
{data.subtitle && (
164+
<div className="text-gray-400 text-[10px] mt-0.5 italic">
165+
{data.subtitle}
166+
</div>
167+
)}
168+
</NodeShell>
169+
);
170+
}
171+
143172
export function DataTableNode({ data }) {
144173
return (
145174
<div className="bg-white rounded-lg shadow-md border-2 border-gray-300 p-3 min-w-[400px] max-w-[600px]">
@@ -187,5 +216,7 @@ export const nodeTypes = {
187216
missing: MissingNode,
188217
external: ExternalNode,
189218
us_specific: USSpecificNode,
219+
uk_specific: UKSpecificNode,
220+
absent: AbsentNode,
190221
data_table: DataTableNode,
191222
};

‎docs/pipeline-diagrams/src/stages/Stage1.jsx‎

Lines changed: 31 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -331,6 +331,37 @@ const nodes = [
331331
subtitle: "Source provenance per variable",
332332
},
333333
},
334+
335+
// =====================
336+
// UK-ONLY absent nodes
337+
// =====================
338+
{
339+
id: "absent_benefits",
340+
type: "absent",
341+
data: {
342+
label: "UK benefit variables (20 types)",
343+
subtitle: "UK parses UC, PIP, DLA, JSA, ESA, etc.",
344+
absentFrom: "us",
345+
},
346+
},
347+
{
348+
id: "absent_brma",
349+
type: "absent",
350+
data: {
351+
label: "BRMA assignment",
352+
subtitle: "UK assigns Broad Rental Market Areas by region",
353+
absentFrom: "us",
354+
},
355+
},
356+
{
357+
id: "absent_council_tax",
358+
type: "absent",
359+
data: {
360+
label: "Council tax imputation",
361+
subtitle: "UK imputes council tax from FRS bands",
362+
absentFrom: "us",
363+
},
364+
},
334365
];
335366

336367
const edges = [

‎docs/pipeline-diagrams/src/stages/Stage4.jsx‎

Lines changed: 58 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -218,6 +218,64 @@ const nodes = [
218218
details: ["US Census SIPP survey format"],
219219
},
220220
},
221+
222+
// =============================================
223+
// UK-ONLY absent nodes
224+
// =============================================
225+
{
226+
id: "absent_consumption",
227+
type: "absent",
228+
data: {
229+
label: "LCFS consumption imputation",
230+
subtitle: "UK imputes 15 categories + NEED energy calibration",
231+
absentFrom: "us",
232+
},
233+
},
234+
{
235+
id: "absent_vat",
236+
type: "absent",
237+
data: {
238+
label: "VAT expenditure rate imputation",
239+
subtitle: "UK imputes from ETB survey",
240+
absentFrom: "us",
241+
},
242+
},
243+
{
244+
id: "absent_services",
245+
type: "absent",
246+
data: {
247+
label: "Public services imputation",
248+
subtitle: "UK imputes NHS, education, transport from ETB",
249+
absentFrom: "us",
250+
},
251+
},
252+
{
253+
id: "absent_cg",
254+
type: "absent",
255+
data: {
256+
label: "Capital gains optimization",
257+
subtitle: "UK uses Torch optimizer + Advani-Summers distribution",
258+
absentFrom: "us",
259+
},
260+
},
261+
{
262+
id: "absent_ss",
263+
type: "absent",
264+
data: {
265+
label: "Salary sacrifice imputation",
266+
subtitle: "UK 2-stage QRF + headcount targeting (5.4M)",
267+
absentFrom: "us",
268+
},
269+
},
270+
{
271+
id: "absent_student_loan",
272+
type: "absent",
273+
data: {
274+
label: "Student loan plan assignment",
275+
subtitle: "UK assigns Plan 1/2/4/5 by age",
276+
absentFrom: "us",
277+
},
278+
},
221279
];
222280

223281
const edges = [

0 commit comments

Comments
 (0)