Skip to content

Commit aebec27

Browse files
charlesviencharlesvien
authored
chore(ci): Warm shared caches and cut per-job install time (#3676)
1 parent 9a06399 commit aebec27

7 files changed

Lines changed: 190 additions & 88 deletions

File tree

‎.github/actions/restore-turbo-cache/action.yml‎

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,13 @@
1+
name: Restore turbo cache
2+
description: Restore-only turbo cache. warm-caches.yml seeds it on main pushes; PR workflows never save, so per-PR entries don't pollute the cache store.
3+
4+
runs:
5+
using: composite
6+
steps:
7+
- name: Restore turbo cache
8+
uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0
9+
with:
10+
path: .turbo/cache
11+
key: turbo-${{ github.sha }}
12+
restore-keys: |
13+
turbo-

‎.github/workflows/build.yml‎

Lines changed: 8 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -59,29 +59,17 @@ jobs:
5959
node-version: 22
6060
cache: "pnpm"
6161

62+
- name: Restore turbo cache
63+
uses: ./.github/actions/restore-turbo-cache
64+
6265
- name: Install dependencies
6366
run: pnpm install --frozen-lockfile
67+
env:
68+
# The packaged app is never launched, so the Electron-ABI rebuild is unused.
69+
SKIP_ELECTRON_REBUILD: "1"
6470

65-
- name: Build electron-trpc
66-
run: pnpm --filter @posthog/electron-trpc build
67-
68-
- name: Build platform
69-
run: pnpm --filter @posthog/platform build
70-
71-
- name: Build shared
72-
run: pnpm --filter @posthog/shared build
73-
74-
- name: Build git
75-
run: pnpm --filter @posthog/git build
76-
77-
- name: Build enricher
78-
run: pnpm --filter @posthog/enricher build
79-
80-
- name: Build harness
81-
run: pnpm --filter @posthog/harness build
82-
83-
- name: Build agent
84-
run: pnpm --filter agent build
71+
- name: Build packages
72+
run: pnpm run build:deps
8573

8674
- name: Build code
8775
run: pnpm --filter code build

‎.github/workflows/code-storybook.yml‎

Lines changed: 6 additions & 32 deletions
Original file line numberDiff line numberDiff line change
@@ -115,8 +115,13 @@ jobs:
115115
restore-keys: |
116116
playwright-chromium-${{ runner.os }}-
117117
118+
- name: Restore turbo cache
119+
uses: ./.github/actions/restore-turbo-cache
120+
118121
- name: Install dependencies
119122
run: pnpm install --frozen-lockfile
123+
env:
124+
SKIP_ELECTRON_REBUILD: "1"
120125

121126
- name: Install Playwright Chromium
122127
if: steps.playwright-cache.outputs.cache-hit != 'true'
@@ -130,38 +135,7 @@ jobs:
130135
run: pnpm exec playwright install-deps chromium
131136

132137
- name: Build workspace packages
133-
# Bare `wait` returns the exit status of only the last job passed to
134-
# it, silently swallowing failures from the others. `wait_all` waits
135-
# on each PID individually and checks its own status so any failed
136-
# build fails the step.
137-
run: |
138-
wait_all() {
139-
status=0
140-
for pid in "$@"; do
141-
wait "$pid" || status=$?
142-
done
143-
[ "$status" -eq 0 ]
144-
}
145-
146-
pnpm --filter @posthog/electron-trpc build &
147-
pid1=$!
148-
(pnpm --filter @posthog/shared build && pnpm --filter @posthog/platform build) &
149-
pid2=$!
150-
wait_all "$pid1" "$pid2"
151-
152-
# @posthog/agent imports the dist of @posthog/git and
153-
# @posthog/harness, so both must finish before the last group starts.
154-
pnpm --filter @posthog/git build &
155-
pid3=$!
156-
pnpm --filter @posthog/harness build &
157-
pid4=$!
158-
wait_all "$pid3" "$pid4"
159-
160-
pnpm --filter @posthog/enricher build &
161-
pid5=$!
162-
pnpm --filter @posthog/agent build &
163-
pid6=$!
164-
wait_all "$pid5" "$pid6"
138+
run: pnpm run build:deps
165139

166140
- name: Build Storybook
167141
working-directory: apps/code

‎.github/workflows/test.yml‎

Lines changed: 21 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -62,15 +62,19 @@ jobs:
6262
node-version: 22
6363
cache: "pnpm"
6464

65+
# Only dependency builds are cached, tests themselves always run.
66+
- name: Restore turbo cache
67+
uses: ./.github/actions/restore-turbo-cache
68+
6569
- name: Install dependencies
6670
run: pnpm install --frozen-lockfile
71+
env:
72+
# The Electron-ABI rebuild would be undone by the Node rebuild below.
73+
SKIP_ELECTRON_REBUILD: "1"
6774

6875
- name: Rebuild native modules for Node
6976
run: node scripts/rebuild-better-sqlite3-node.mjs
7077

71-
- name: Build dependencies
72-
run: pnpm --filter @posthog/electron-trpc build
73-
7478
- name: Run tests
7579
run: pnpm test
7680

@@ -127,22 +131,16 @@ jobs:
127131
restore-keys: |
128132
electron-${{ runner.os }}-
129133
134+
# Seeded on Linux; turbo artifacts are plain JS, so cross-OS restore is fine.
135+
- name: Restore turbo cache
136+
uses: ./.github/actions/restore-turbo-cache
137+
138+
# apps/code's postinstall rebuilds better-sqlite3 for Electron here.
130139
- name: Install dependencies
131140
run: pnpm install --frozen-lockfile
132141

133-
- name: Rebuild better-sqlite3 for Electron
134-
run: node scripts/rebuild-better-sqlite3-electron.mjs
135-
136142
- name: Build packages
137-
run: |
138-
pnpm --filter @posthog/electron-trpc build &
139-
pnpm --filter @posthog/platform build &
140-
pnpm --filter @posthog/shared build
141-
pnpm --filter @posthog/git build
142-
pnpm --filter @posthog/enricher build
143-
wait
144-
pnpm --filter @posthog/harness build
145-
pnpm --filter agent build
143+
run: pnpm run build:deps
146144

147145
- name: Package Electron app
148146
run: pnpm --filter code run package
@@ -230,22 +228,24 @@ jobs:
230228
persist-credentials: false
231229

232230
- name: Setup pnpm
233-
uses: pnpm/action-setup@b906affcce14559ad1aafd4ab0e942779e9f58b1 # v4.3.0
231+
uses: pnpm/action-setup@0ebf47130e4866e96fce0953f49152a61190b271 # v6.0.9
234232

235233
- name: Setup Node.js
236-
uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4.4.0
234+
uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.4.0
237235
with:
238236
node-version: 22
239237
cache: "pnpm"
240238

239+
- name: Restore turbo cache
240+
uses: ./.github/actions/restore-turbo-cache
241+
241242
- name: Install dependencies
242243
run: pnpm install --frozen-lockfile
244+
env:
245+
SKIP_ELECTRON_REBUILD: "1"
243246

244247
- name: Build agent dependencies
245-
run: |
246-
pnpm --filter @posthog/shared run build
247-
pnpm --filter @posthog/git run build
248-
pnpm --filter @posthog/enricher run build
248+
run: pnpm exec turbo build --filter='@posthog/agent^...'
249249

250250
- name: Download native codex binary
251251
# Non-fatal at the STEP so a failure surfaces as the fail-loud binary guard

‎.github/workflows/typecheck.yml‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -59,8 +59,13 @@ jobs:
5959
node-version: 22
6060
cache: "pnpm"
6161

62+
- name: Restore turbo cache
63+
uses: ./.github/actions/restore-turbo-cache
64+
6265
- name: Install dependencies
6366
run: pnpm install --frozen-lockfile
67+
env:
68+
SKIP_ELECTRON_REBUILD: "1"
6469

6570
- name: Run type check
6671
run: pnpm run typecheck

‎.github/workflows/warm-caches.yml‎

Lines changed: 117 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,117 @@
1+
name: Warm caches
2+
3+
# GitHub only shares caches saved on main with other branches, and the PR
4+
# workflows never run on main. This job seeds the caches they restore.
5+
6+
on:
7+
push:
8+
branches: [main]
9+
workflow_dispatch:
10+
11+
concurrency:
12+
group: warm-caches-${{ github.ref }}
13+
cancel-in-progress: true
14+
15+
jobs:
16+
linux:
17+
runs-on: ubuntu-latest
18+
timeout-minutes: 15
19+
permissions:
20+
contents: read
21+
steps:
22+
- name: Checkout
23+
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
24+
with:
25+
persist-credentials: false
26+
27+
- name: Setup pnpm
28+
uses: pnpm/action-setup@0ebf47130e4866e96fce0953f49152a61190b271 # v6.0.9
29+
30+
- name: Setup Node.js
31+
uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.4.0
32+
with:
33+
node-version: 22
34+
cache: "pnpm"
35+
36+
# No restore-keys: a fresh save per commit keeps the cache from growing
37+
# forever, at the cost of a cold build on every main push.
38+
- name: Cache turbo artifacts
39+
uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0
40+
with:
41+
path: .turbo/cache
42+
key: turbo-${{ github.sha }}
43+
44+
# Same key as code-storybook.yml's storybook job.
45+
- name: Cache Playwright browsers
46+
id: playwright-cache
47+
uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0
48+
with:
49+
path: ~/.cache/ms-playwright
50+
key: playwright-chromium-${{ runner.os }}-${{ hashFiles('pnpm-lock.yaml') }}
51+
52+
- name: Install dependencies
53+
run: pnpm install --frozen-lockfile
54+
env:
55+
SKIP_ELECTRON_REBUILD: "1"
56+
57+
- name: Seed turbo cache (build + typecheck)
58+
run: |
59+
pnpm run build:deps
60+
pnpm run typecheck
61+
62+
- name: Install Playwright Chromium
63+
if: steps.playwright-cache.outputs.cache-hit != 'true'
64+
working-directory: apps/code
65+
run: pnpm exec playwright install chromium
66+
67+
macos:
68+
runs-on: macos-latest
69+
timeout-minutes: 30
70+
permissions:
71+
contents: read
72+
steps:
73+
- name: Checkout
74+
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
75+
with:
76+
persist-credentials: false
77+
78+
- name: Setup pnpm
79+
uses: pnpm/action-setup@0ebf47130e4866e96fce0953f49152a61190b271 # v6.0.9
80+
81+
- name: Setup Node.js
82+
id: node
83+
uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.4.0
84+
with:
85+
node-version: 22
86+
cache: "pnpm"
87+
88+
# Same keys as test.yml's integration-test job.
89+
- name: Cache Electron binary
90+
id: electron-cache
91+
uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0
92+
with:
93+
path: ~/Library/Caches/electron
94+
key: electron-${{ runner.os }}-${{ hashFiles('pnpm-lock.yaml') }}
95+
96+
- name: Cache Playwright browsers
97+
id: playwright-cache
98+
uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0
99+
with:
100+
path: ~/Library/Caches/ms-playwright
101+
key: playwright-${{ runner.os }}-${{ hashFiles('pnpm-lock.yaml') }}
102+
103+
# macOS minutes are 10x; do nothing when every cache is already warm.
104+
- name: Install dependencies
105+
if: >-
106+
steps.node.outputs.cache-hit != 'true' ||
107+
steps.electron-cache.outputs.cache-hit != 'true' ||
108+
steps.playwright-cache.outputs.cache-hit != 'true'
109+
# Electron's own postinstall fills ~/Library/Caches/electron; the sqlite
110+
# rebuild only touches node_modules, which is not cached, so skip it.
111+
run: pnpm install --frozen-lockfile
112+
env:
113+
SKIP_ELECTRON_REBUILD: "1"
114+
115+
- name: Install Playwright Chromium
116+
if: steps.playwright-cache.outputs.cache-hit != 'true'
117+
run: pnpm --filter code exec playwright install chromium

‎apps/code/scripts/postinstall.sh‎

Lines changed: 20 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -8,23 +8,28 @@ set -e
88
REPO_ROOT="$(cd ../.. && pwd)"
99
SCRIPTS_DIR="$(cd "$(dirname "$0")" && pwd)"
1010

11-
# Self-heal missing Electron binary.
12-
# pnpm skips package-level postinstall scripts when the lockfile is already
13-
# satisfied, so if node_modules/electron/dist gets wiped (interrupted download,
14-
# cache eviction, arch change, manual cleanup), `pnpm install` won't notice —
15-
# and `electron-vite dev` then fails with "Electron failed to install
16-
# correctly, please delete node_modules/electron and try installing again".
17-
# Detect the missing binary and invoke Electron's own install script to fetch it.
18-
ELECTRON_DIST="$REPO_ROOT/node_modules/electron/dist"
19-
if [ ! -d "$ELECTRON_DIST" ] || [ -z "$(ls -A "$ELECTRON_DIST" 2>/dev/null)" ]; then
20-
echo "Electron binary missing at $ELECTRON_DIST — downloading..."
21-
node "$REPO_ROOT/node_modules/electron/install.js"
22-
fi
11+
# CI jobs that never run Electron set this to skip ~60s of node-gyp per install.
12+
if [ "$SKIP_ELECTRON_REBUILD" = "1" ] || [ "$SKIP_ELECTRON_REBUILD" = "true" ]; then
13+
echo "SKIP_ELECTRON_REBUILD=$SKIP_ELECTRON_REBUILD: skipping Electron binary check and native rebuild."
14+
else
15+
# Self-heal missing Electron binary.
16+
# pnpm skips package-level postinstall scripts when the lockfile is already
17+
# satisfied, so if node_modules/electron/dist gets wiped (interrupted download,
18+
# cache eviction, arch change, manual cleanup), `pnpm install` won't notice —
19+
# and `electron-vite dev` then fails with "Electron failed to install
20+
# correctly, please delete node_modules/electron and try installing again".
21+
# Detect the missing binary and invoke Electron's own install script to fetch it.
22+
ELECTRON_DIST="$REPO_ROOT/node_modules/electron/dist"
23+
if [ ! -d "$ELECTRON_DIST" ] || [ -z "$(ls -A "$ELECTRON_DIST" 2>/dev/null)" ]; then
24+
echo "Electron binary missing at $ELECTRON_DIST — downloading..."
25+
node "$REPO_ROOT/node_modules/electron/install.js"
26+
fi
2327

24-
echo "Rebuilding native modules for Electron..."
28+
echo "Rebuilding native modules for Electron..."
2529

26-
cd "$REPO_ROOT"
27-
node scripts/rebuild-better-sqlite3-electron.mjs
30+
cd "$REPO_ROOT"
31+
node scripts/rebuild-better-sqlite3-electron.mjs
32+
fi
2833

2934
# Restore the execute bit on node-pty's spawn-helper. pnpm extracts node-pty's
3035
# prebuilt binaries without preserving the executable mode, so the helper lands

0 commit comments

Comments
 (0)