Skip to content

Commit f8626c0

Browse files
anth-volkclaude
anth-volk
and
claude
committed
Add under-construction nodes for 6 in-flight PRs
New "under construction" node type (amber dashed) for showing pipeline changes that are actively being developed: US: - PR #611: Pipeline orchestrator in Overview (Modal hardening) - PR #540: Category takeup rerandomization in Stage 2, extracted puf_impute.py + source_impute.py modules in Stage 4 - PR #618: CMS marketplace data + plan selection in Stage 5 UK: - PR #291: New Stage 9 — OA calibration pipeline (6 phases) - PR #296: New Stage 10 — Adversarial weight regularisation - PR #279: Modal GPU calibration nodes in Stages 6, 7, Overview Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
1 parent 591b54a commit f8626c0

12 files changed

Lines changed: 601 additions & 10 deletions

File tree

‎docs/pipeline-diagrams/src/App.jsx‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,8 @@ import UKStage5 from "./stages/uk/Stage5";
2020
import UKStage6 from "./stages/uk/Stage6";
2121
import UKStage7 from "./stages/uk/Stage7";
2222
import UKStage8 from "./stages/uk/Stage8";
23+
import UKStage9 from "./stages/uk/Stage9";
24+
import UKStage10 from "./stages/uk/Stage10";
2325

2426
export default function App() {
2527
return (
@@ -48,6 +50,8 @@ export default function App() {
4850
<Route path="/uk/stage/6" element={<UKStage6 />} />
4951
<Route path="/uk/stage/7" element={<UKStage7 />} />
5052
<Route path="/uk/stage/8" element={<UKStage8 />} />
53+
<Route path="/uk/stage/9" element={<UKStage9 />} />
54+
<Route path="/uk/stage/10" element={<UKStage10 />} />
5155
</Routes>
5256
</div>
5357
</div>

‎docs/pipeline-diagrams/src/components/Sidebar.jsx‎

Lines changed: 9 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -25,6 +25,8 @@ const ukStages = [
2525
{ path: "/uk/stage/6", label: "Stage 6", desc: "Weight Fitting (Torch)" },
2626
{ path: "/uk/stage/7", label: "Stage 7", desc: "Local Area Calibration" },
2727
{ path: "/uk/stage/8", label: "Stage 8", desc: "Validation & Output" },
28+
{ path: "/uk/stage/9", label: "Stage 9", desc: "OA Calibration", underConstruction: true },
29+
{ path: "/uk/stage/10", label: "Stage 10", desc: "Weight Regularisation", underConstruction: true },
2830
];
2931

3032
const nodeLegend = [
@@ -37,6 +39,7 @@ const nodeLegend = [
3739
{ color: "bg-pink-200 border-pink-400", label: "US-Specific" },
3840
{ color: "bg-teal-200 border-teal-400", label: "UK-Specific" },
3941
{ color: "bg-gray-100 border-gray-300 border-dashed", label: "Absent" },
42+
{ color: "bg-amber-200 border-amber-400 border-dashed", label: "Under construction" },
4043
];
4144

4245
export default function Sidebar() {
@@ -74,15 +77,18 @@ export default function Sidebar() {
7477
? "bg-blue-100 text-blue-900 font-semibold"
7578
: s.absent
7679
? "text-gray-400 hover:bg-gray-100"
77-
: "text-gray-700 hover:bg-gray-100"
80+
: s.underConstruction
81+
? "text-amber-700 hover:bg-amber-50"
82+
: "text-gray-700 hover:bg-gray-100"
7883
}`
7984
}
8085
>
81-
<div className={`font-medium text-[13px] ${s.absent ? "text-gray-400" : ""}`}>
86+
<div className={`font-medium text-[13px] ${s.absent ? "text-gray-400" : ""} ${s.underConstruction ? "text-amber-700" : ""}`}>
8287
{s.label}
8388
{s.absent && " (absent)"}
89+
{s.underConstruction && " (under construction)"}
8490
</div>
85-
<div className={`text-[10px] mt-0.5 ${s.absent ? "text-gray-300" : "text-gray-500"}`}>
91+
<div className={`text-[10px] mt-0.5 ${s.absent ? "text-gray-300" : s.underConstruction ? "text-amber-500" : "text-gray-500"}`}>
8692
{s.desc}
8793
</div>
8894
</NavLink>

‎docs/pipeline-diagrams/src/components/nodes/CustomNodes.jsx‎

Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -169,6 +169,22 @@ export function AbsentNode({ data, selected }) {
169169
);
170170
}
171171

172+
export function UnderConstructionNode({ data, selected }) {
173+
return (
174+
<NodeShell bg="bg-amber-50" border="border-amber-400" dashed selected={selected}>
175+
<div className="font-bold text-amber-800 text-[11px] uppercase tracking-wide mb-0.5">
176+
Under Construction
177+
</div>
178+
<NodeBody data={data} />
179+
{data.pr && (
180+
<div className="mt-1 inline-block bg-amber-100 text-amber-700 text-[9px] font-semibold px-1.5 py-0.5 rounded">
181+
PR {data.pr}
182+
</div>
183+
)}
184+
</NodeShell>
185+
);
186+
}
187+
172188
export function DataTableNode({ data }) {
173189
return (
174190
<div className="bg-white rounded-lg shadow-md border-2 border-gray-300 p-3 min-w-[400px] max-w-[600px]">
@@ -218,5 +234,6 @@ export const nodeTypes = {
218234
us_specific: USSpecificNode,
219235
uk_specific: UKSpecificNode,
220236
absent: AbsentNode,
237+
under_construction: UnderConstructionNode,
221238
data_table: DataTableNode,
222239
};

‎docs/pipeline-diagrams/src/stages/Overview.jsx‎

Lines changed: 33 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -106,6 +106,23 @@ const nodes = [
106106
elkPartition: 1,
107107
},
108108
},
109+
110+
// Under construction
111+
{
112+
id: "orchestrator",
113+
type: "under_construction",
114+
data: {
115+
label: "Pipeline Orchestrator",
116+
subtitle: "5-step automated pipeline with resume support",
117+
details: [
118+
"Pre-baked Modal images",
119+
"Auto-trigger on merge to main",
120+
"Atomic staging \u2192 production promotion",
121+
],
122+
pr: "#611",
123+
elkPartition: 5,
124+
},
125+
},
109126
];
110127

111128
const edges = [
@@ -168,26 +185,35 @@ const edges = [
168185
label: "51 state + 435 district + 1 city H5 files",
169186
},
170187

171-
// Modal edges: modal (x=850) -> stages (x=400) = right to left -> sl/tr
188+
// Modal edges -> orchestrator
172189
{
173-
id: "e-modal1-s5",
190+
id: "e-modal1-orch",
174191
source: "modal1",
175-
target: "stage5",
192+
target: "orchestrator",
176193
...runsOn,
177194
},
178195
{
179-
id: "e-modal2-s6",
196+
id: "e-modal2-orch",
180197
source: "modal2",
181-
target: "stage6",
198+
target: "orchestrator",
182199
...runsOn,
183200
},
184201
{
185-
id: "e-modal3-s7",
202+
id: "e-modal3-orch",
186203
source: "modal3",
187-
target: "stage7",
204+
target: "orchestrator",
188205
...runsOn,
189206
},
190207

208+
// Orchestrator coordinates stages 5-8
209+
{
210+
id: "e-orch-s5",
211+
source: "orchestrator",
212+
target: "stage5",
213+
...informational,
214+
label: "coordinates",
215+
},
216+
191217
// HuggingFace edges
192218
// hf1 (x=-100, y=800) -> stage0 (x=400, y=0): route UP from hf1
193219
// then RIGHT to stage0, keeping the vertical segment on the far left

‎docs/pipeline-diagrams/src/stages/Stage2.jsx‎

Lines changed: 31 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -213,6 +213,23 @@ const nodes = [
213213
},
214214
},
215215

216+
// =====================
217+
// UNDER CONSTRUCTION
218+
// =====================
219+
{
220+
id: "rerandomize",
221+
type: "under_construction",
222+
data: {
223+
label: "rerandomize_category_takeup()",
224+
subtitle: "Category-dependent takeup re-randomization",
225+
details: [
226+
"EITC, WIC during calibration cloning",
227+
"Two-pass _simulate_clone() flow",
228+
],
229+
pr: "#540",
230+
},
231+
},
232+
216233
// =====================
217234
// MISSING
218235
// =====================
@@ -345,6 +362,20 @@ const edges = [
345362
target: "ss_reconcile",
346363
...usesUtility,
347364
},
365+
// Under construction connections
366+
{
367+
id: "e-record-rerand",
368+
source: "record_double",
369+
target: "rerandomize",
370+
...dataFlow,
371+
},
372+
{
373+
id: "e-rerand-qrf1",
374+
source: "rerandomize",
375+
target: "qrf_pass1",
376+
...dataFlow,
377+
},
378+
348379
// Single QRF utility fans out to 3 targets
349380
{
350381
id: "e-util-qrf-qrf1",

‎docs/pipeline-diagrams/src/stages/Stage4.jsx‎

Lines changed: 44 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -206,6 +206,36 @@ const nodes = [
206206
},
207207
},
208208

209+
// =============================================
210+
// UNDER CONSTRUCTION nodes
211+
// =============================================
212+
{
213+
id: "puf_impute_module",
214+
type: "under_construction",
215+
data: {
216+
label: "puf_impute.py",
217+
subtitle: "Extracted PUF imputation module",
218+
details: [
219+
"Replaces part of extended_cps.py",
220+
"Standalone PUF clone + QRF flow",
221+
],
222+
pr: "#540",
223+
},
224+
},
225+
{
226+
id: "source_impute_module",
227+
type: "under_construction",
228+
data: {
229+
label: "source_impute.py",
230+
subtitle: "Extracted source imputation module",
231+
details: [
232+
"ACS/SIPP/SCF orchestration",
233+
"Replaces part of extended_cps.py",
234+
],
235+
pr: "#540",
236+
},
237+
},
238+
209239
// =============================================
210240
// US-SPECIFIC nodes
211241
// =============================================
@@ -421,6 +451,20 @@ const edges = [
421451
target: "scf_qrf",
422452
...usesUtility,
423453
},
454+
455+
// Under construction connections
456+
{
457+
id: "e-puf-impute-geo",
458+
source: "puf_impute_module",
459+
target: "geo_assign",
460+
...dataFlow,
461+
},
462+
{
463+
id: "e-source-impute-acs",
464+
source: "source_impute_module",
465+
target: "acs_qrf",
466+
...dataFlow,
467+
},
424468
];
425469

426470
export default function Stage4() {

‎docs/pipeline-diagrams/src/stages/Stage5.jsx‎

Lines changed: 43 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -320,6 +320,35 @@ const nodes = [
320320
},
321321
},
322322

323+
// =============================================
324+
// UNDER CONSTRUCTION nodes
325+
// =============================================
326+
{
327+
id: "cms_data",
328+
type: "under_construction",
329+
data: {
330+
label: "CMS Marketplace Data",
331+
subtitle: "Rating area price menus + county plan behavior",
332+
details: [
333+
"aca_marketplace_rating_area_price_menu_2024.csv",
334+
"aca_marketplace_county_plan_behavior_2024.csv",
335+
],
336+
pr: "#618",
337+
},
338+
},
339+
{
340+
id: "marketplace_selection",
341+
type: "under_construction",
342+
data: {
343+
label: "marketplace_plan_selection.py",
344+
subtitle: "Bronze vs benchmark plan selection layer",
345+
details: [
346+
"Models households leaving part of ACA credit unused",
347+
],
348+
pr: "#618",
349+
},
350+
},
351+
323352
// =============================================
324353
// MISSING nodes (x=850, below utilities)
325354
// =============================================
@@ -560,6 +589,20 @@ const edges = [
560589
target: "sparse_build",
561590
...usesUtility,
562591
},
592+
593+
// Under construction connections (PR #618)
594+
{
595+
id: "e-cms-marketplace",
596+
source: "cms_data",
597+
target: "marketplace_selection",
598+
...dataFlow,
599+
},
600+
{
601+
id: "e-marketplace-sparse",
602+
source: "marketplace_selection",
603+
target: "sparse_build",
604+
...dataFlow,
605+
},
563606
];
564607

565608
export default function Stage5() {

‎docs/pipeline-diagrams/src/stages/uk/Overview.jsx‎

Lines changed: 47 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,7 @@ import {
33
dataFlow,
44
producesArtifact,
55
informational,
6+
runsOn,
67
} from "../../constants/edgeStyles";
78

89
const nodes = [
@@ -120,6 +121,46 @@ const nodes = [
120121
],
121122
},
122123
},
124+
// Under construction stages
125+
{
126+
id: "stage9",
127+
type: "under_construction",
128+
data: {
129+
label: "Stage 9: OA calibration",
130+
subtitle: "Output Area-level calibration (~235K OAs)",
131+
details: [
132+
"Clone-and-assign, L0 engine",
133+
"Sparse matrix, SQLite targets",
134+
"Per-area H5 publishing",
135+
],
136+
pr: "#291",
137+
},
138+
},
139+
{
140+
id: "stage10",
141+
type: "under_construction",
142+
data: {
143+
label: "Stage 10: Weight regularisation",
144+
subtitle: "Adversarial diagnostics pipeline",
145+
details: [
146+
"Influence detection, TVAE model",
147+
"Adversarial loop + recalibration",
148+
],
149+
pr: "#296",
150+
},
151+
},
152+
153+
// Under construction infrastructure
154+
{
155+
id: "modal_gpu",
156+
type: "under_construction",
157+
data: {
158+
label: "Modal (GPU)",
159+
subtitle: "T4/A10G containers",
160+
details: ["Offloads calibration to GPU"],
161+
pr: "#279",
162+
},
163+
},
123164
];
124165

125166
const edges = [
@@ -132,6 +173,12 @@ const edges = [
132173
{ id: "e-5-6", source: "stage5", target: "stage6", ...dataFlow, label: "target matrix" },
133174
{ id: "e-6-7", source: "stage6", target: "stage7", ...dataFlow, label: "constituency weights" },
134175
{ id: "e-7-8", source: "stage7", target: "stage8", ...producesArtifact, label: "calibrated dataset" },
176+
177+
// Under construction edges
178+
{ id: "e-7-9", source: "stage7", target: "stage9", ...dataFlow, label: "calibrated FRS" },
179+
{ id: "e-7-10", source: "stage7", target: "stage10", ...dataFlow, label: "calibrated weights" },
180+
{ id: "e-modal-6", source: "modal_gpu", target: "stage6", ...runsOn },
181+
{ id: "e-modal-7", source: "modal_gpu", target: "stage7", ...runsOn },
135182
];
136183

137184
export default function Overview() {

0 commit comments

Comments
 (0)