Skip to content

Commit 19a26a0

Browse files
anth-volkclaude
anth-volk
and
claude
committed
Revert to plain modal run for matrix jobs
Revert the modal deploy + Function.from_name() approach — it didn't stream container logs. Go back to modal run per matrix job, which streams logs natively. Accept the image rebuild overhead for now while we investigate the cache miss root cause. Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
1 parent 8543284 commit 19a26a0

2 files changed

Lines changed: 48 additions & 123 deletions

File tree

‎.github/workflows/push.yaml‎

Lines changed: 47 additions & 122 deletions
Original file line numberDiff line numberDiff line change
@@ -15,59 +15,31 @@ jobs:
1515
- name: Check formatting
1616
run: ruff format --check .
1717

18-
# ── Deploy Modal app (build image once) ─────────────────────
19-
deploy-modal:
18+
# ── Download prerequisites ──────────────────────────────────
19+
download-prerequisites:
2020
runs-on: ubuntu-latest
2121
needs: lint
2222
if: github.event.head_commit.message != 'Update package version'
23-
outputs:
24-
app-name: ${{ steps.deploy.outputs.app_name }}
2523
env:
2624
MODAL_TOKEN_ID: ${{ secrets.MODAL_TOKEN_ID }}
2725
MODAL_TOKEN_SECRET: ${{ secrets.MODAL_TOKEN_SECRET }}
26+
HUGGING_FACE_TOKEN: ${{ secrets.HUGGING_FACE_TOKEN }}
2827
steps:
2928
- uses: actions/checkout@v4
30-
- uses: actions/setup-python@v5
31-
with:
32-
python-version: "3.13"
33-
- name: Install Modal CLI
34-
run: pip install modal
35-
- name: Deploy Modal app
36-
id: deploy
37-
run: |
38-
APP_NAME="policyengine-us-data-ci-${{ github.run_id }}"
39-
echo "app_name=${APP_NAME}" >> "$GITHUB_OUTPUT"
40-
MODAL_APP_NAME="${APP_NAME}" modal deploy modal_app/data_build.py
41-
42-
# ── Download prerequisites ──────────────────────────────────
43-
download-prerequisites:
44-
runs-on: ubuntu-latest
45-
needs: deploy-modal
46-
env:
47-
MODAL_TOKEN_ID: ${{ secrets.MODAL_TOKEN_ID }}
48-
MODAL_TOKEN_SECRET: ${{ secrets.MODAL_TOKEN_SECRET }}
49-
steps:
5029
- uses: actions/setup-python@v5
5130
with:
5231
python-version: "3.13"
5332
- name: Install Modal CLI
5433
run: pip install modal
5534
- name: Download prerequisites on Modal
5635
run: |
57-
python -c "
58-
import modal
59-
with modal.enable_output():
60-
fn = modal.Function.from_name('${{ needs.deploy-modal.outputs.app-name }}', 'run_single_script')
61-
result = fn.remote(
62-
script_name='download_prerequisites',
63-
branch='${{ github.ref_name }}',
64-
)
65-
print(result)
66-
"
36+
modal run modal_app/data_build.py \
37+
--script download_prerequisites \
38+
--branch=${{ github.ref_name }}
6739
6840
# ── Phase 1: Independent datasets (parallel) ───────────────
6941
phase1:
70-
needs: [deploy-modal, download-prerequisites]
42+
needs: download-prerequisites
7143
runs-on: ubuntu-latest
7244
strategy:
7345
fail-fast: true
@@ -76,29 +48,24 @@ jobs:
7648
env:
7749
MODAL_TOKEN_ID: ${{ secrets.MODAL_TOKEN_ID }}
7850
MODAL_TOKEN_SECRET: ${{ secrets.MODAL_TOKEN_SECRET }}
51+
HUGGING_FACE_TOKEN: ${{ secrets.HUGGING_FACE_TOKEN }}
7952
steps:
53+
- uses: actions/checkout@v4
8054
- uses: actions/setup-python@v5
8155
with:
8256
python-version: "3.13"
8357
- name: Install Modal CLI
8458
run: pip install modal
8559
- name: "Build + test: ${{ matrix.dataset }}"
8660
run: |
87-
python -c "
88-
import modal
89-
with modal.enable_output():
90-
fn = modal.Function.from_name('${{ needs.deploy-modal.outputs.app-name }}', 'run_single_script')
91-
result = fn.remote(
92-
script_name='${{ matrix.dataset }}',
93-
branch='${{ github.ref_name }}',
94-
run_tests=True,
95-
)
96-
print(result)
97-
"
61+
modal run modal_app/data_build.py \
62+
--script ${{ matrix.dataset }} \
63+
--run-tests \
64+
--branch=${{ github.ref_name }}
9865
9966
# ── Phase 2: CPS + PUF (depend on Phase 1) ─────────────────
10067
phase2:
101-
needs: [deploy-modal, phase1]
68+
needs: phase1
10269
runs-on: ubuntu-latest
10370
strategy:
10471
fail-fast: true
@@ -107,56 +74,46 @@ jobs:
10774
env:
10875
MODAL_TOKEN_ID: ${{ secrets.MODAL_TOKEN_ID }}
10976
MODAL_TOKEN_SECRET: ${{ secrets.MODAL_TOKEN_SECRET }}
77+
HUGGING_FACE_TOKEN: ${{ secrets.HUGGING_FACE_TOKEN }}
11078
steps:
79+
- uses: actions/checkout@v4
11180
- uses: actions/setup-python@v5
11281
with:
11382
python-version: "3.13"
11483
- name: Install Modal CLI
11584
run: pip install modal
11685
- name: "Build + test: ${{ matrix.dataset }}"
11786
run: |
118-
python -c "
119-
import modal
120-
with modal.enable_output():
121-
fn = modal.Function.from_name('${{ needs.deploy-modal.outputs.app-name }}', 'run_single_script')
122-
result = fn.remote(
123-
script_name='${{ matrix.dataset }}',
124-
branch='${{ github.ref_name }}',
125-
run_tests=True,
126-
)
127-
print(result)
128-
"
87+
modal run modal_app/data_build.py \
88+
--script ${{ matrix.dataset }} \
89+
--run-tests \
90+
--branch=${{ github.ref_name }}
12991
13092
# ── Phase 3: Extended CPS (depends on Phase 2) ─────────────
13193
phase3:
132-
needs: [deploy-modal, phase2]
94+
needs: phase2
13395
runs-on: ubuntu-latest
13496
env:
13597
MODAL_TOKEN_ID: ${{ secrets.MODAL_TOKEN_ID }}
13698
MODAL_TOKEN_SECRET: ${{ secrets.MODAL_TOKEN_SECRET }}
99+
HUGGING_FACE_TOKEN: ${{ secrets.HUGGING_FACE_TOKEN }}
137100
steps:
101+
- uses: actions/checkout@v4
138102
- uses: actions/setup-python@v5
139103
with:
140104
python-version: "3.13"
141105
- name: Install Modal CLI
142106
run: pip install modal
143107
- name: "Build + test: extended_cps"
144108
run: |
145-
python -c "
146-
import modal
147-
with modal.enable_output():
148-
fn = modal.Function.from_name('${{ needs.deploy-modal.outputs.app-name }}', 'run_single_script')
149-
result = fn.remote(
150-
script_name='extended_cps',
151-
branch='${{ github.ref_name }}',
152-
run_tests=True,
153-
)
154-
print(result)
155-
"
109+
modal run modal_app/data_build.py \
110+
--script extended_cps \
111+
--run-tests \
112+
--branch=${{ github.ref_name }}
156113
157114
# ── Phase 4: Enhanced + Stratified CPS (depend on Phase 3) ─
158115
phase4:
159-
needs: [deploy-modal, phase3]
116+
needs: phase3
160117
runs-on: ubuntu-latest
161118
strategy:
162119
fail-fast: true
@@ -165,29 +122,24 @@ jobs:
165122
env:
166123
MODAL_TOKEN_ID: ${{ secrets.MODAL_TOKEN_ID }}
167124
MODAL_TOKEN_SECRET: ${{ secrets.MODAL_TOKEN_SECRET }}
125+
HUGGING_FACE_TOKEN: ${{ secrets.HUGGING_FACE_TOKEN }}
168126
steps:
127+
- uses: actions/checkout@v4
169128
- uses: actions/setup-python@v5
170129
with:
171130
python-version: "3.13"
172131
- name: Install Modal CLI
173132
run: pip install modal
174133
- name: "Build + test: ${{ matrix.dataset }}"
175134
run: |
176-
python -c "
177-
import modal
178-
with modal.enable_output():
179-
fn = modal.Function.from_name('${{ needs.deploy-modal.outputs.app-name }}', 'run_single_script')
180-
result = fn.remote(
181-
script_name='${{ matrix.dataset }}',
182-
branch='${{ github.ref_name }}',
183-
run_tests=True,
184-
)
185-
print(result)
186-
"
135+
modal run modal_app/data_build.py \
136+
--script ${{ matrix.dataset }} \
137+
--run-tests \
138+
--branch=${{ github.ref_name }}
187139
188140
# ── Phase 5: Source imputed + Small enhanced (depend on 4) ──
189141
phase5:
190-
needs: [deploy-modal, phase4]
142+
needs: phase4
191143
runs-on: ubuntu-latest
192144
strategy:
193145
fail-fast: true
@@ -196,29 +148,24 @@ jobs:
196148
env:
197149
MODAL_TOKEN_ID: ${{ secrets.MODAL_TOKEN_ID }}
198150
MODAL_TOKEN_SECRET: ${{ secrets.MODAL_TOKEN_SECRET }}
151+
HUGGING_FACE_TOKEN: ${{ secrets.HUGGING_FACE_TOKEN }}
199152
steps:
153+
- uses: actions/checkout@v4
200154
- uses: actions/setup-python@v5
201155
with:
202156
python-version: "3.13"
203157
- name: Install Modal CLI
204158
run: pip install modal
205159
- name: "Build + test: ${{ matrix.dataset }}"
206160
run: |
207-
python -c "
208-
import modal
209-
with modal.enable_output():
210-
fn = modal.Function.from_name('${{ needs.deploy-modal.outputs.app-name }}', 'run_single_script')
211-
result = fn.remote(
212-
script_name='${{ matrix.dataset }}',
213-
branch='${{ github.ref_name }}',
214-
run_tests=True,
215-
)
216-
print(result)
217-
"
161+
modal run modal_app/data_build.py \
162+
--script ${{ matrix.dataset }} \
163+
--run-tests \
164+
--branch=${{ github.ref_name }}
218165
219166
# ── Remaining integration tests (depend on Phase 4) ─────────
220167
remaining-tests:
221-
needs: [deploy-modal, phase4]
168+
needs: phase4
222169
runs-on: ubuntu-latest
223170
strategy:
224171
fail-fast: false
@@ -229,41 +176,19 @@ jobs:
229176
env:
230177
MODAL_TOKEN_ID: ${{ secrets.MODAL_TOKEN_ID }}
231178
MODAL_TOKEN_SECRET: ${{ secrets.MODAL_TOKEN_SECRET }}
179+
HUGGING_FACE_TOKEN: ${{ secrets.HUGGING_FACE_TOKEN }}
232180
steps:
181+
- uses: actions/checkout@v4
233182
- uses: actions/setup-python@v5
234183
with:
235184
python-version: "3.13"
236185
- name: Install Modal CLI
237186
run: pip install modal
238187
- name: "Test: ${{ matrix.test }}"
239188
run: |
240-
python -c "
241-
import modal
242-
with modal.enable_output():
243-
fn = modal.Function.from_name('${{ needs.deploy-modal.outputs.app-name }}', 'run_integration_test')
244-
result = fn.remote(
245-
test_path='${{ matrix.test }}',
246-
branch='${{ github.ref_name }}',
247-
)
248-
print(result)
249-
"
250-
251-
# ── Cleanup Modal deployment ────────────────────────────────
252-
cleanup-modal:
253-
needs: [deploy-modal, phase5, remaining-tests]
254-
runs-on: ubuntu-latest
255-
if: always()
256-
env:
257-
MODAL_TOKEN_ID: ${{ secrets.MODAL_TOKEN_ID }}
258-
MODAL_TOKEN_SECRET: ${{ secrets.MODAL_TOKEN_SECRET }}
259-
steps:
260-
- uses: actions/setup-python@v5
261-
with:
262-
python-version: "3.13"
263-
- name: Install Modal CLI
264-
run: pip install modal
265-
- name: Stop deployed app
266-
run: modal app stop ${{ needs.deploy-modal.outputs.app-name }} 2>/dev/null || true
189+
modal run modal_app/data_build.py \
190+
--test ${{ matrix.test }} \
191+
--branch=${{ github.ref_name }}
267192
268193
# ── Manual approval gate ────────────────────────────────────
269194
approval-gate:

‎modal_app/data_build.py‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -19,7 +19,7 @@
1919

2020
from modal_app.images import cpu_image as image
2121

22-
app = modal.App(os.environ.get("MODAL_APP_NAME", "policyengine-us-data"))
22+
app = modal.App("policyengine-us-data")
2323

2424
hf_secret = modal.Secret.from_name("huggingface-token")
2525
gcp_secret = modal.Secret.from_name("gcp-credentials")

0 commit comments

Comments
 (0)