Skip to content

Commit 4c1c682

Browse files
namancsenamancse
authored
Merge pull request #115 from microsoft/user/namjain/reprobuild-pipeline-golden-working
Add reproducible builds support in OHCL-Linux-Kernel
2 parents 46ee03c + 842705e commit 4c1c682

9 files changed

Lines changed: 1752 additions & 59 deletions

File tree

‎.github/workflows/ci.yaml‎

Lines changed: 196 additions & 53 deletions
Original file line numberDiff line numberDiff line change
@@ -12,77 +12,220 @@ on:
1212

1313
jobs:
1414
build_linux:
15-
name: Build Linux kernel
15+
name: Build Linux kernel (reproducible, run ${{ matrix.repro_run }})
1616
runs-on: ubuntu-latest
1717
strategy:
18+
fail-fast: false
1819
matrix:
1920
arch: ['x64', 'cvm-x64', 'arm64']
21+
repro_run: [1, 2]
22+
env:
23+
# Stable build identity used to derive SOURCE_DATE_EPOCH and metadata.
24+
KBUILD_BUILD_USER: builder
25+
KBUILD_BUILD_HOST: nixos
2026
steps:
2127
- name: Checkout source
2228
uses: actions/checkout@v4
29+
with:
30+
# Need the HEAD commit's timestamp for SOURCE_DATE_EPOCH; depth=1 is sufficient.
31+
fetch-depth: 1
2332

24-
- name: Install dependencies
25-
run: |
26-
sudo apt-get update
27-
sudo apt-get install -y build-essential libncurses-dev bison flex libssl-dev libelf-dev
28-
[ "${{ matrix.arch }}" == "arm64" ] && sudo apt-get install -y gcc-aarch64-linux-gnu || true
33+
- name: Install Nix
34+
uses: cachix/install-nix-action@v27
35+
with:
36+
extra_nix_config: |
37+
experimental-features = nix-command flakes
38+
accept-flake-config = true
2939
30-
- name: Build kernel
40+
- name: Map ci arch to build args
41+
id: args
3142
run: |
32-
if [ "${{ matrix.arch }}" == "x64" ] || [ "${{ matrix.arch }}" == "cvm-x64" ]; then
33-
export makeargs="ARCH=x86_64 KCONFIG_CONFIG=Microsoft/hcl-x64.config INSTALL_MOD_PATH=$PWD/out/modules"
34-
export targets="olddefconfig vmlinux modules modules_install"
35-
elif [ "${{ matrix.arch }}" == "arm64" ]; then
36-
export makeargs="ARCH=arm64 CROSS_COMPILE=aarch64-linux-gnu- KCONFIG_CONFIG=Microsoft/hcl-arm64.config INSTALL_MOD_PATH=$PWD/out/modules"
37-
export targets="olddefconfig vmlinux Image modules modules_install"
38-
fi
43+
case "${{ matrix.arch }}" in
44+
x64)
45+
echo "build_arch=amd64" >> "$GITHUB_OUTPUT"
46+
echo "config=Microsoft/hcl-x64.config" >> "$GITHUB_OUTPUT"
47+
echo "is_cvm=0" >> "$GITHUB_OUTPUT"
48+
;;
49+
cvm-x64)
50+
echo "build_arch=amd64" >> "$GITHUB_OUTPUT"
51+
echo "config=Microsoft/hcl-x64.config" >> "$GITHUB_OUTPUT"
52+
echo "is_cvm=1" >> "$GITHUB_OUTPUT"
53+
;;
54+
arm64)
55+
echo "build_arch=arm64" >> "$GITHUB_OUTPUT"
56+
echo "config=Microsoft/hcl-arm64.config" >> "$GITHUB_OUTPUT"
57+
echo "is_cvm=0" >> "$GITHUB_OUTPUT"
58+
;;
59+
esac
3960
40-
[ "${{ matrix.arch }}" == "cvm-x64" ] && ./Microsoft/merge-cvm-config.sh
41-
make $makeargs -j$(nproc) $targets
61+
- name: Merge CVM config fragment
62+
if: steps.args.outputs.is_cvm == '1'
63+
run: |
64+
# Run the merge inside the Nix dev shell so flex/bison/gcc are present
65+
# and so that scripts/kconfig builds against the same toolchain that
66+
# will be used for the actual kernel compile.
67+
nix develop -i \
68+
-k HOME -k USER -k TERM \
69+
--command bash -ec './Microsoft/merge-cvm-config.sh'
4270
43-
- name: Prepare for packaging
71+
- name: Build kernel reproducibly
72+
env:
73+
# SOURCE_DATE_EPOCH is derived from the HEAD commit by the pipeline
74+
# script when not pre-set; exporting it here makes it visible in logs.
75+
GIT_SHA: ${{ github.sha }}
4476
run: |
45-
export artifacts_dir="release_artifacts/${{ matrix.arch }}"
46-
if [ "${{ matrix.arch }}" == "arm64" ]; then
47-
export objcopy="aarch64-linux-gnu-objcopy"
48-
else
49-
export objcopy="objcopy"
50-
fi
77+
# The pipeline script copies the source into /tmp/ohcl-kernel-build/src
78+
# to normalise embedded paths. Place the build dir OUTSIDE the source
79+
# so it survives that rsync (which excludes ./build, ./out, ./compare).
80+
BUILD_DIR="${RUNNER_TEMP:-/tmp}/build-output-${{ matrix.arch }}-run${{ matrix.repro_run }}"
81+
rm -rf "$BUILD_DIR"
82+
mkdir -p "$BUILD_DIR"
83+
echo "BUILD_DIR=$BUILD_DIR" >> "$GITHUB_ENV"
5184
52-
mkdir -p $artifacts_dir
53-
cp vmlinux $artifacts_dir
54-
$objcopy --only-keep-debug --compress-debug-sections $artifacts_dir/vmlinux $artifacts_dir/vmlinux.dbg
55-
$objcopy --strip-all --add-gnu-debuglink=$artifacts_dir/vmlinux.dbg $artifacts_dir/vmlinux
56-
[ "${{ matrix.arch }}" == "arm64" ] && cp arch/arm64/boot/Image $artifacts_dir/Image || true
85+
export SOURCE_DATE_EPOCH="$(git log -1 --pretty=%ct)"
86+
echo "SOURCE_DATE_EPOCH=$SOURCE_DATE_EPOCH"
87+
echo "SOURCE_DATE_EPOCH=$SOURCE_DATE_EPOCH" >> "$GITHUB_ENV"
5788
58-
if [ "${{ matrix.arch }}" == "cvm-x64" ]; then
59-
cp Microsoft/hcl-x64.config $artifacts_dir/kernel_config
60-
else
61-
cp Microsoft/hcl-${{ matrix.arch }}.config $artifacts_dir/kernel_config
62-
fi
89+
./Microsoft/build-hcl-kernel-pipeline.sh \
90+
--source-dir "$PWD" \
91+
--build-dir "$BUILD_DIR" \
92+
--config "${{ steps.args.outputs.config }}" \
93+
--arch "${{ steps.args.outputs.build_arch }}" \
94+
--reproducible
6395
64-
echo "{" > $artifacts_dir/kernel_build_metadata.json
65-
echo " \"git_branch\": \"${{ github.ref_name }}\"," >> $artifacts_dir/kernel_build_metadata.json
66-
echo " \"git_revision\": \"${{ github.sha }}\"," >> $artifacts_dir/kernel_build_metadata.json
67-
echo " \"build_id\": \"${{ github.run_id }}\"," >> $artifacts_dir/kernel_build_metadata.json
68-
echo " \"build_name\": \"$(date +"%Y%m%d")\"" >> $artifacts_dir/kernel_build_metadata.json
69-
echo "}" >> $artifacts_dir/kernel_build_metadata.json
70-
71-
mv out/modules/lib/modules/*/ $artifacts_dir/modules
72-
rm $artifacts_dir/modules/build
73-
mkdir $artifacts_dir/modules/debug-files/
74-
find $artifacts_dir/modules -name '*.ko' | while read -r mod; do
75-
export outmod="$artifacts_dir/modules/debug-files/$(basename $mod)"
76-
$objcopy --only-keep-debug --compress-debug-sections "$mod" "$outmod.dbg"
77-
$objcopy --strip-unneeded --add-gnu-debuglink "$outmod.dbg" "$mod"
78-
done
96+
- name: Package artifacts
97+
run: |
98+
# Packaging runs inside the Nix shell so the cross objcopy is on PATH
99+
# for arm64 module stripping.
100+
nix develop -i \
101+
-k HOME -k USER -k TERM \
102+
-k BUILD_DIR \
103+
--command bash -ec '
104+
bash ./Microsoft/package-ci-artifacts.sh \
105+
"$BUILD_DIR" \
106+
"${{ matrix.arch }}" \
107+
"$PWD" \
108+
"$PWD/release_artifacts" \
109+
"${{ github.ref_name }}" \
110+
"${{ github.sha }}" \
111+
"${{ github.run_id }}"
112+
'
113+
114+
- name: Print reproducibility hashes
115+
run: |
116+
ART="release_artifacts/${{ matrix.arch }}"
117+
echo "## Reproducibility hashes (${{ matrix.arch }} / run ${{ matrix.repro_run }})" >> "$GITHUB_STEP_SUMMARY"
118+
echo '```' >> "$GITHUB_STEP_SUMMARY"
119+
( cd "$ART" && sha256sum vmlinux vmlinux.dbg kernel_config \
120+
$( [ -f Image ] && echo Image ) ) | tee -a "$GITHUB_STEP_SUMMARY"
121+
echo '```' >> "$GITHUB_STEP_SUMMARY"
79122
80123
- name: Upload build artifacts
81124
uses: actions/upload-artifact@v4
82125
with:
83-
name: ${{ matrix.arch }}-build
126+
name: ${{ matrix.arch }}-build-run${{ matrix.repro_run }}
84127
path: release_artifacts/${{ matrix.arch }}/
85128

129+
compare_repro:
130+
name: Compare reproducibility hashes
131+
needs: build_linux
132+
runs-on: ubuntu-latest
133+
steps:
134+
- name: Download x64 run 1 artifacts
135+
uses: actions/download-artifact@v4
136+
with:
137+
name: x64-build-run1
138+
path: compare/x64/run1/
139+
140+
- name: Download x64 run 2 artifacts
141+
uses: actions/download-artifact@v4
142+
with:
143+
name: x64-build-run2
144+
path: compare/x64/run2/
145+
146+
- name: Download cvm-x64 run 1 artifacts
147+
uses: actions/download-artifact@v4
148+
with:
149+
name: cvm-x64-build-run1
150+
path: compare/cvm-x64/run1/
151+
152+
- name: Download cvm-x64 run 2 artifacts
153+
uses: actions/download-artifact@v4
154+
with:
155+
name: cvm-x64-build-run2
156+
path: compare/cvm-x64/run2/
157+
158+
- name: Download arm64 run 1 artifacts
159+
uses: actions/download-artifact@v4
160+
with:
161+
name: arm64-build-run1
162+
path: compare/arm64/run1/
163+
164+
- name: Download arm64 run 2 artifacts
165+
uses: actions/download-artifact@v4
166+
with:
167+
name: arm64-build-run2
168+
path: compare/arm64/run2/
169+
170+
- name: Compare hashes
171+
run: |
172+
set -euo pipefail
173+
status=0
174+
175+
echo "## Reproducibility comparison" | tee -a "$GITHUB_STEP_SUMMARY"
176+
echo "" | tee -a "$GITHUB_STEP_SUMMARY"
177+
echo "| Arch | File | Run1 SHA256 | Run2 SHA256 | Result |" | tee -a "$GITHUB_STEP_SUMMARY"
178+
echo "| --- | --- | --- | --- | --- |" | tee -a "$GITHUB_STEP_SUMMARY"
179+
180+
for arch in x64 cvm-x64 arm64; do
181+
run1="compare/$arch/run1"
182+
run2="compare/$arch/run2"
183+
sha1="/tmp/${arch}.run1.tsv"
184+
sha2="/tmp/${arch}.run2.tsv"
185+
merged="/tmp/${arch}.merged.tsv"
186+
187+
( cd "$run1" && find . -type f -print0 | LC_ALL=C sort -z | xargs -0 sha256sum ) \
188+
| awk '{print $2 "\t" $1}' > "$sha1"
189+
( cd "$run2" && find . -type f -print0 | LC_ALL=C sort -z | xargs -0 sha256sum ) \
190+
| awk '{print $2 "\t" $1}' > "$sha2"
191+
192+
awk -F '\t' '
193+
NR == FNR { run1[$1] = $2; next }
194+
{ run2[$1] = $2 }
195+
END {
196+
for (f in run1) seen[f] = 1
197+
for (f in run2) seen[f] = 1
198+
for (f in seen) {
199+
h1 = (f in run1) ? run1[f] : "MISSING"
200+
h2 = (f in run2) ? run2[f] : "MISSING"
201+
r = (h1 == h2) ? "match" : "mismatch"
202+
printf "%s\t%s\t%s\t%s\n", f, h1, h2, r
203+
}
204+
}
205+
' "$sha1" "$sha2" | sort > "$merged"
206+
207+
while IFS=$'\t' read -r file h1 h2 result; do
208+
echo "| $arch | $file | $h1 | $h2 | $result |" | tee -a "$GITHUB_STEP_SUMMARY"
209+
if [[ "$result" == "mismatch" ]]; then
210+
status=1
211+
fi
212+
done < "$merged"
213+
214+
if [[ "$status" -ne 0 ]]; then
215+
echo "::error::Reproducibility mismatch detected for $arch"
216+
echo "--- $arch mismatches ---"
217+
awk -F '\t' '$4 == "mismatch" {print $1 " run1=" $2 " run2=" $3}' "$merged" | sed -n '1,200p'
218+
fi
219+
done
220+
221+
echo "" | tee -a "$GITHUB_STEP_SUMMARY"
222+
if [[ "$status" -eq 0 ]]; then
223+
echo "All reproducibility hashes matched across both runs." | tee -a "$GITHUB_STEP_SUMMARY"
224+
else
225+
echo "One or more reproducibility comparisons mismatched." | tee -a "$GITHUB_STEP_SUMMARY"
226+
exit 1
227+
fi
228+
86229
build_perf:
87230
name: Build Perf
88231
runs-on: ubuntu-22.04
@@ -122,7 +265,7 @@ jobs:
122265

123266
create_release:
124267
name: Create GitHub Release
125-
needs: [build_linux, build_perf]
268+
needs: [compare_repro, build_perf]
126269
if: startsWith(github.ref, 'refs/tags/')
127270
runs-on: ubuntu-latest
128271
steps:
@@ -131,17 +274,17 @@ jobs:
131274

132275
- uses: actions/download-artifact@v4
133276
with:
134-
name: x64-build
277+
name: x64-build-run1
135278
path: release_artifacts/x64/
136279

137280
- uses: actions/download-artifact@v4
138281
with:
139-
name: cvm-x64-build
282+
name: cvm-x64-build-run1
140283
path: release_artifacts/cvm-x64/
141284

142285
- uses: actions/download-artifact@v4
143286
with:
144-
name: arm64-build
287+
name: arm64-build-run1
145288
path: release_artifacts/arm64/
146289

147290
- uses: actions/download-artifact@v4

0 commit comments

Comments
 (0)