@@ -19,18 +19,38 @@ name: Lake cache shadow (master)
1919# # host than the S3 API endpoint; e.g. an R2 public r2.dev/custom domain).
2020# LAKE_CACHE_ARTIFACT_ENDPOINT_PUBLIC — e.g. https://pub-<hash>.r2.dev/<prefix>/artifacts
2121# LAKE_CACHE_REVISION_ENDPOINT_PUBLIC — e.g. https://pub-<hash>.r2.dev/<prefix>/revisions
22+ # # Optional. Default toolchain override for every run, including
23+ # # scheduled ones; the dispatch input takes precedence. Unset to run
24+ # # on the repo pin.
25+ # LAKE_SHADOW_TOOLCHAIN_OVERRIDE — e.g. leanprover/lean4-pr-releases:pr-release-14301
2226#
2327# Jobs:
2428# build_and_stage Build mathlib + deps with Lake's artifact cache
2529# enabled, then stage the resulting .ltar files.
2630# upload Push staged artifacts to the cache bucket via
2731# `lake cache put-staged`, then record a per-run manifest
28- # under cache/analysis/ and report carryover vs the prior run.
32+ # under cache/analysis/<toolchain-slug>/ and report
33+ # carryover vs the prior run on the same toolchain.
2934# consume Fresh checkout, fetch from the cache bucket via
3035# `lake cache get`, run `lake build` against the
3136# rehydrated cache, then verify with `--rehash`.
3237# report Post a per-run summary to the `CI admins` Zulip
3338# stream (topic `Lake cache shadow`).
39+ #
40+ # Toolchain override:
41+ # The `toolchain_override` input, or else the LAKE_SHADOW_TOOLCHAIN_OVERRIDE
42+ # variable, swaps the toolchain the whole pipeline runs on: build_and_stage
43+ # resolves it into its `toolchain` output, and the downstream jobs stamp
44+ # that output into their checkouts, so every job runs the same lake. Its
45+ # lean must be behaviorally compatible with the repo pin, e.g. a Lake change
46+ # cherry-picked onto the pinned release's lineage as a lean4 pr-release.
47+ # Input hashes incorporate the toolchain, so all runs safely share one
48+ # artifact scope. The analysis chain (warm-start pointer + carryover
49+ # manifests) is keyed per toolchain under analysis/<slug>/, so pinned and
50+ # override runs each warm-start from and diff against their own lineage.
51+ # A toolchain's first run — or a pr-release tag republished under the same
52+ # name — misses the legacy cache and all prior artifacts, costing one
53+ # full-turnover source build.
3454
3555on :
3656 schedule :
5474 # Mathlib.Topology.Basic Mathlib.Combinatorics.SimpleGraph.Basic Mathlib.RingTheory.Ideal.Basic
5575 # or add `Archive Counterexamples` to also shadow those libraries.
5676 default : Mathlib
77+ toolchain_override :
78+ description : >-
79+ optional Lean toolchain to run the pipeline on instead of the repo
80+ pin; see the "Toolchain override" note at the top of this file.
81+ Empty falls back to the LAKE_SHADOW_TOOLCHAIN_OVERRIDE repository
82+ variable; the pipeline runs on the repo pin when both are empty.
83+ required : false
84+ type : string
85+ default : ' '
5786
5887permissions :
5988 contents : read
@@ -67,6 +96,9 @@ defaults:
6796 shell : bash -euo pipefail {0}
6897
6998env :
99+ # Effective toolchain override: dispatch input, else repository variable,
100+ # else the repo pin.
101+ TOOLCHAIN_OVERRIDE : ${{ inputs.toolchain_override || vars.LAKE_SHADOW_TOOLCHAIN_OVERRIDE || '' }}
70102 # Scope prefix for all puts and gets in this workflow. Namespaces the
71103 # workflow's artifacts within the cache bucket.
72104 SHADOW_SCOPE : mathlib4-master-shadow
78110 name : Build + stage
79111 if : ${{ github.repository == 'leanprover-community/mathlib4' }}
80112 runs-on : pr
81- timeout-minutes : 90
113+ # 240 covers the full source build of a toolchain generation's first run.
114+ timeout-minutes : ${{ (inputs.toolchain_override || vars.LAKE_SHADOW_TOOLCHAIN_OVERRIDE) && 240 || 90 }}
82115 outputs :
83116 sha : ${{ steps.resolve.outputs.sha }}
84117 toolchain : ${{ steps.resolve.outputs.toolchain }}
@@ -110,6 +143,15 @@ jobs:
110143 fetch-depth : 2
111144 path : pr-branch
112145
146+ # Before the resolve step, so the run's recorded toolchain is the one
147+ # actually used.
148+ - name : Apply toolchain override
149+ if : ${{ env.TOOLCHAIN_OVERRIDE != '' }}
150+ shell : bash -euo pipefail {0}
151+ run : |
152+ printf '%s\n' "$TOOLCHAIN_OVERRIDE" > pr-branch/lean-toolchain
153+ echo "::notice::toolchain override: $TOOLCHAIN_OVERRIDE"
154+
113155 - name : Resolve sha & toolchain
114156 id : resolve
115157 shell : bash -euo pipefail {0}
@@ -124,7 +166,10 @@ jobs:
124166 shell : bash -euo pipefail {0}
125167 run : |
126168 mkdir -p pr-branch/.lake/
127- mkdir -p .cache/mathlib/
169+ # The landrun ruleset mounts $HOME/.cache/mathlib read-only and
170+ # errors out if the path is missing. Runs that skip legacy hydration
171+ # reach the first landrun step with it absent.
172+ mkdir -p "$HOME/.cache/mathlib/"
128173 mkdir -p _work
129174
130175 - name : install elan
@@ -135,6 +180,17 @@ jobs:
135180 ./elan-init.sh -y --default-toolchain none
136181 echo "$HOME/.elan/bin" >> "${GITHUB_PATH}"
137182
183+ # `elan which` below does not auto-install non-pinned toolchains. The
184+ # uninstall forces a fresh download: pr-release tags are republished in
185+ # place, and elan skips toolchains it already has, so a persistent
186+ # runner would otherwise keep running a stale binary under that name.
187+ - name : Install override toolchain
188+ if : ${{ env.TOOLCHAIN_OVERRIDE != '' }}
189+ shell : bash -euo pipefail {0}
190+ run : |
191+ elan toolchain uninstall "$TOOLCHAIN_OVERRIDE" || true
192+ elan toolchain install "$TOOLCHAIN_OVERRIDE"
193+
138194 - name : set toolchain directory
139195 shell : bash -euo pipefail {0}
140196 run : |
@@ -168,7 +224,10 @@ jobs:
168224 cd pr-branch
169225 lake env
170226
227+ # The legacy cache is keyed to the repo's pinned toolchain — cold for an
228+ # override toolchain, where the warm-start step below takes its place.
171229 - name : Hydrate .lake/build via legacy cache
230+ if : ${{ env.TOOLCHAIN_OVERRIDE == '' }}
172231 shell : bash -euo pipefail {0}
173232 run : |
174233 cd pr-branch
@@ -199,6 +258,28 @@ jobs:
199258 awk '/^package mathlib where/,/^$/' lakefile.lean
200259 echo "::endgroup::"
201260
261+ # Seed Lake's artifact cache from the previous run on the same
262+ # toolchain, whose rev comes from that toolchain's analysis chain
263+ # (`analysis/<slug>/_latest.txt`). Only content the cache can't serve
264+ # is packed or built; a toolchain's first run has no chain yet and
265+ # proceeds from source.
266+ - name : Warm start from shadow scope
267+ shell : bash -euo pipefail {0}
268+ env :
269+ LAKE_CACHE_ARTIFACT_ENDPOINT : ${{ vars.LAKE_CACHE_ARTIFACT_ENDPOINT_PUBLIC }}
270+ LAKE_CACHE_REVISION_ENDPOINT : ${{ vars.LAKE_CACHE_REVISION_ENDPOINT_PUBLIC }}
271+ run : |
272+ # Must mirror the slug in the upload job's carryover step.
273+ slug="$(printf %s "$(cat pr-branch/lean-toolchain)" | tr -c 'A-Za-z0-9._-' '-')"
274+ prev="$(curl -fsS "${LAKE_CACHE_ARTIFACT_ENDPOINT%/artifacts}/analysis/$slug/_latest.txt" 2>/dev/null || true)"
275+ if [ -z "$prev" ]; then
276+ echo "::notice::no prior run recorded for chain $slug; proceeding without a warm start"
277+ exit 0
278+ fi
279+ cd pr-branch
280+ lake cache get --scope="$SHADOW_SCOPE" --rev="$prev" \
281+ || echo "::warning::warm start from rev ${prev:0:12} failed; proceeding without it"
282+
202283 # Incremental build: with the legacy cache having hydrated .lake/build/
203284 # and the lakefile patched, Lake's pipeline runs to pack/cache any
204285 # modules whose .ltar+mapping aren't yet in Lake's cache. Module
@@ -277,13 +358,21 @@ jobs:
277358 LAKE_CACHE_ARTIFACT_ENDPOINT : ${{ vars.LAKE_CACHE_ARTIFACT_ENDPOINT }}
278359 LAKE_CACHE_REVISION_ENDPOINT : ${{ vars.LAKE_CACHE_REVISION_ENDPOINT }}
279360 steps :
280- - name : Checkout mathlib (for lean-toolchain pin )
361+ - name : Checkout mathlib (workspace for put-staged )
281362 uses : actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
282363 with :
283364 ref : ${{ needs.build_and_stage.outputs.sha }}
284365 path : pr-branch
285366 fetch-depth : 1
286367
368+ # Stamp the effective toolchain from build_and_stage's resolve step, so
369+ # put-staged runs the same lake that produced the staging; the elan
370+ # proxy auto-installs it on first use.
371+ - name : Pin effective toolchain
372+ env :
373+ EFFECTIVE_TOOLCHAIN : ${{ needs.build_and_stage.outputs.toolchain }}
374+ run : printf '%s\n' "$EFFECTIVE_TOOLCHAIN" > pr-branch/lean-toolchain
375+
287376 - name : install elan + matching lake
288377 run : |
289378 curl -o elan-init.sh -sSfL https://elan.lean-lang.org/elan-init.sh
@@ -318,7 +407,8 @@ jobs:
318407 --toolchain="${{ needs.build_and_stage.outputs.toolchain }}" 2>&1 | tee /tmp/put.log
319408
320409 # Cache carryover analysis: diff this run's uploaded artifact set against
321- # the previous run's, using a tiny per-run manifest kept in the bucket.
410+ # the previous run's on the same toolchain, via tiny per-run manifests
411+ # kept in the bucket under the toolchain's analysis/<slug>/ prefix.
322412 # Here:
323413 # carryover = artifacts also present last run;
324414 # new = this run's churn (≈ how much Mathlib changed since the last build).
@@ -331,8 +421,11 @@ jobs:
331421 AUTH : ${{ vars.LAKE_CACHE_ARTIFACT_ENDPOINT }}
332422 GH_TOKEN : ${{ secrets.GITHUB_TOKEN }}
333423 REPO : ${{ github.repository }}
424+ EFFECTIVE_TOOLCHAIN : ${{ needs.build_and_stage.outputs.toolchain }}
334425 run : |
335- base="${AUTH%/artifacts}/analysis"
426+ # Must mirror the slug in build_and_stage's warm-start step.
427+ slug="$(printf %s "$EFFECTIVE_TOOLCHAIN" | tr -c 'A-Za-z0-9._-' '-')"
428+ base="${AUTH%/artifacts}/analysis/$slug"
336429 sha="${{ needs.build_and_stage.outputs.sha }}"
337430 sig=(--aws-sigv4 aws:amz:auto:s3 --user "$LAKE_CACHE_KEY")
338431 # grep exits 1 on no match (handled by the total==0 branch); tolerate it under pipefail.
@@ -379,7 +472,7 @@ jobs:
379472 new_content="$(numfmt --to=iec-i --suffix=B "$new_bytes") across ${new} new artifact(s) vs rev ${prev:0:12}${distnote}"
380473 pct=$(awk "BEGIN{printf \"%.1f\", ($total>0)?100*$carry/$total:0}")
381474 note=""
382- [ "$total" -gt 0 ] && [ "$new" -eq "$total" ] && note=" — full turnover, likely a toolchain/generation change , not source churn"
475+ [ "$total" -gt 0 ] && [ "$new" -eq "$total" ] && note=" — full turnover, likely a republished toolchain binary , not source churn"
383476 summary="${carry}/${total} (${pct}%) carried over, ${new} new vs rev ${prev:0:12}${note}"
384477 fi
385478 echo "::notice::cache carryover: ${summary}"
@@ -396,7 +489,7 @@ jobs:
396489 needs : [build_and_stage, upload]
397490 if : ${{ github.repository == 'leanprover-community/mathlib4' }}
398491 runs-on : ubuntu-latest
399- timeout-minutes : 60
492+ timeout-minutes : 90
400493 outputs :
401494 cache_health : ${{ steps.cachehealth.outputs.health }}
402495 env :
@@ -416,6 +509,14 @@ jobs:
416509 path : pr-branch
417510 fetch-depth : 1
418511
512+ # Stamp the effective toolchain from build_and_stage's resolve step, so
513+ # the replay runs the same lake that built the artifacts; the elan
514+ # proxy auto-installs it on first use.
515+ - name : Pin effective toolchain
516+ env :
517+ EFFECTIVE_TOOLCHAIN : ${{ needs.build_and_stage.outputs.toolchain }}
518+ run : printf '%s\n' "$EFFECTIVE_TOOLCHAIN" > pr-branch/lean-toolchain
519+
419520 - name : install elan
420521 run : |
421522 curl -o elan-init.sh -sSfL https://elan.lean-lang.org/elan-init.sh
@@ -533,6 +634,9 @@ jobs:
533634 {
534635 echo "msg<<MSG_EOF"
535636 echo "**Lake cache shadow** ([run](${RUN_URL}), rev \`${SHA:0:12}\`)"
637+ if [ -n "${TOOLCHAIN_OVERRIDE:-}" ]; then
638+ echo "🧪 toolchain override: \`${TOOLCHAIN_OVERRIDE}\`"
639+ fi
536640 echo ""
537641 echo "- $(emoji "$BUILD") build_and_stage: \`$BUILD\`"
538642 echo "- $(emoji "$UPLOAD") upload: \`$UPLOAD\`"
0 commit comments