Skip to content

Commit 1bd546f

Browse files
Nvidia zone kernels (#97)
* checkpoint * hacks * fixup * fixup * Fixup * Fixup this too * Centralize firmware stuff * Add This * Fix * Add firmware bits * Fixups * fuckit * Extend CI to just build latest of all flavors * fixup * ?? * mktempify * Hmm * A small tweak to build drm_kms_helper.ko required by Nvidia GPU driver Signed-off-by: Surbhi Kakarya <surbhi@edera.dev> * test this * Revert "test this" This reverts commit 2969573. * Make this a module --------- Signed-off-by: Surbhi Kakarya <surbhi@edera.dev> Co-authored-by: Surbhi Kakarya <surbhi@edera.dev>
1 parent ebbc145 commit 1bd546f

8 files changed

Lines changed: 184 additions & 45 deletions

File tree

.github/workflows/test.yml

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -8,7 +8,7 @@ permissions:
88
contents: read
99
packages: read
1010
env:
11-
TEST_MATRIX_SPEC: "only-latest:flavor=host,zone,zone-amdgpu"
11+
TEST_MATRIX_SPEC: "only-latest:flavor=zone-nvidiagpu"
1212
jobs:
1313
test:
1414
name: test

config.yaml

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,9 @@ flavors:
1010
- name: zone-amdgpu
1111
constraints:
1212
lower: '6.1'
13+
- name: zone-nvidiagpu
14+
constraints:
15+
lower: '6.1'
1316
- name: zone-openpax
1417
constraints:
1518
series:
Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
# nvidiagpu on aarch64 isn't supported yet, this is a stubfile
Lines changed: 32 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,32 @@
1+
#
2+
# Edera kernel config snippet
3+
# - Generated from delta config: configs/x86_64/zone-amdgpu.fragment.config
4+
# - Against base config: /tmp/tmp.pTwNs4lJKY-zone-5.4.293-amd64/kernel/config
5+
#
6+
CONFIG_PCI_DEBUG=y
7+
CONFIG_DRM=y
8+
CONFIG_DRM_CLIENT=y
9+
CONFIG_DRM_CLIENT_LIB=m
10+
CONFIG_DRM_CLIENT_SELECTION=m
11+
CONFIG_DRM_CLIENT_SETUP=y
12+
13+
14+
CONFIG_DRM_DISPLAY_HELPER=m
15+
CONFIG_DRM_DISPLAY_DP_HELPER=y
16+
CONFIG_DRM_DISPLAY_DSC_HELPER=y
17+
CONFIG_DRM_DISPLAY_HDCP_HELPER=y
18+
CONFIG_DRM_DISPLAY_HDMI_HELPER=y
19+
CONFIG_DRM_KMS_HELPER=y
20+
CONFIG_DRM_TTM=m
21+
CONFIG_DRM_EXEC=m
22+
CONFIG_DRM_BUDDY=m
23+
CONFIG_DRM_TTM_HELPER=m
24+
CONFIG_DRM_SUBALLOC_HELPER=m
25+
CONFIG_DRM_SCHED=m
26+
CONFIG_DRM_AMDGPU=m
27+
CONFIG_DRM_I915=n
28+
CONFIG_DRM_NOUVEAU=n
29+
30+
CONFIG_ACPI_WMI=y
31+
CONFIG_WMI_BMOF=y
32+
CONFIG_MXM_WMI=y

hack/build/build.sh

Lines changed: 15 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -20,37 +20,33 @@ KERNEL_MODULES_VER="$(ls "${MODULES_INSTALL_PATH}/lib/modules")"
2020

2121
mkdir -p "${ADDONS_OUTPUT_PATH}"
2222

23-
# Firmware handling (will go in ${ADDONS_PATH}/firmware, siblings with `${ADDONS_PATH}/modules`)
24-
# Note that this assumes the archive is a .tar file, and has already been validated elsewhere.
25-
if [ -n "${FIRMWARE_URL}" ]; then
26-
mkdir -p "${FIRMWARE_OUTPUT_PATH}"
27-
echo "untarring firmware at $FIRMWARE_URL"
28-
tar -xf "${FIRMWARE_URL}" -C "${FIRMWARE_OUTPUT_PATH}" --strip-components=1
29-
# For amdgpu zone kernel, we only want the amdgpu firmwares, so remove the rest to keep the addons small
30-
if [ "${KERNEL_FLAVOR}" = "zone-amdgpu" ]; then
31-
OLDDIR=$PWD
32-
cd "${FIRMWARE_OUTPUT_PATH}"
33-
find . -maxdepth 1 ! -name 'amdgpu' ! -name "." -exec rm -rf {} +
34-
# Compress firmwares on-disk
35-
# As of 6.x kernels the kconfig explicitly says you must use crc32 or none, not the default crc64.
36-
xz -C crc32 amdgpu/*
37-
cd "${OLDDIR}"
38-
fi
39-
fi
23+
# shellcheck source-path=SCRIPTDIR source=nvidiagpu-common.sh
24+
. "${KERNEL_DIR}/hack/build/nvidiagpu-common.sh"
25+
26+
# shellcheck source-path=SCRIPTDIR source=firmware.sh
27+
. "${KERNEL_DIR}/hack/build/firmware.sh"
4028

4129
mv "${MODULES_INSTALL_PATH}/lib/modules/${KERNEL_MODULES_VER}" "${MODULES_OUTPUT_PATH}"
4230
rm -rf "${MODULES_INSTALL_PATH}"
4331
[ -L "${MODULES_OUTPUT_PATH}/build" ] && unlink "${MODULES_OUTPUT_PATH}/build"
4432

4533
mksquashfs "${ADDONS_OUTPUT_PATH}" "${ADDONS_SQUASHFS_PATH}" -all-root
4634

35+
SQUASH_SIZE=$(stat -c %s "${ADDONS_SQUASHFS_PATH}")
36+
37+
# Generally we want to keep zone kernels small, because large kernels -> longer pull times -> increased zone cold boot times.
38+
# We don't really care how big the host kernel is.
39+
# Nvidia kernels have chonker firmwares, even compressed (like 200MB total size), so not much we can really do there.
4740
case "$KERNEL_FLAVOR" in
4841
zone-debug)
4942
# Skip the check for zone-debug, it is allowed to be big
5043
;;
44+
zone-nvidiagpu)
45+
# Firmware means this is unavoidably quite large
46+
;;
5147
zone*)
52-
if [ "$(stat -c %s "${ADDONS_SQUASHFS_PATH}")" -gt 52428800 ]; then
53-
echo "ERROR: squashfs is >50MB in size which is undesirable for the 'zone' kernel, validate kconfig options!" >&2
48+
if [ "${SQUASH_SIZE}" -gt 52428800 ]; then
49+
echo "ERROR: squashfs is >50MB in size (${SQUASH_SIZE} bytes) which is undesirable for the 'zone' kernel, validate kconfig options!" >&2
5450
exit 1
5551
fi
5652
;;

hack/build/common.sh

Lines changed: 0 additions & 25 deletions
Original file line numberDiff line numberDiff line change
@@ -36,29 +36,6 @@ if [ -z "${KERNEL_FLAVOR}" ]; then
3636
KERNEL_FLAVOR="zone"
3737
fi
3838

39-
# Getting the pubkey from the signature is slightly stupid, ideally we should maintain
40-
# a list of valid keys out-of-band, this is best-effort.
41-
if [ -n "${FIRMWARE_SIG_URL}" ]; then
42-
echo "Found firmware signature $FIRMWARE_SIG_URL, attempting validation"
43-
44-
# Check if FIRMWARE_URL exists and is a regular file
45-
if [ ! -f "$FIRMWARE_URL" ]; then
46-
echo "ERROR: $FIRMWARE_URL does not exist or is not a regular file"
47-
echo "ERROR: If this was intentional, consider unsetting '$FIRMWARE_SIG_URL'"
48-
exit 1
49-
fi
50-
51-
KEY_INFO=$(gpg --verify "$FIRMWARE_SIG_URL" "$FIRMWARE_URL" 2>&1) || true
52-
KEY_ID=$(echo "$KEY_INFO" | grep -E "using .* key|key ID" | grep -oE "[A-F0-9]{40}|[A-F0-9]{16,}")
53-
gpg --recv-key "$KEY_ID"
54-
unxz "$FIRMWARE_URL"
55-
# We've uncompressed it, update the env var so later stuff points at the right file
56-
FIRMWARE_URL="${FIRMWARE_URL%.xz}"
57-
gpg --verify "$FIRMWARE_SIG_URL" || { echo "ERROR: signature ${FIRMWARE_SIG_URL} cannot validate ${FIRMWARE_URL}"; exit 1; }
58-
else
59-
echo "No firmware signature defined, no validation will be performed"
60-
fi
61-
6239
KERNEL_SRC="${KERNEL_DIR}/src/linux-${KERNEL_VERSION}-${TARGET_ARCH_STANDARD}"
6340
KERNEL_OBJ="${KERNEL_DIR}/obj/linux-${KERNEL_VERSION}-${TARGET_ARCH_STANDARD}"
6441

@@ -202,8 +179,6 @@ ADDONS_OUTPUT_PATH="${OUTPUT_DIR}/addons"
202179
# shellcheck disable=SC2034
203180
MODULES_OUTPUT_PATH="${ADDONS_OUTPUT_PATH}/modules"
204181
# shellcheck disable=SC2034
205-
FIRMWARE_OUTPUT_PATH="${ADDONS_OUTPUT_PATH}/firmware"
206-
# shellcheck disable=SC2034
207182
ADDONS_SQUASHFS_PATH="${OUTPUT_DIR}/addons.squashfs"
208183
# shellcheck disable=SC2034
209184
METADATA_PATH="${OUTPUT_DIR}/metadata"

hack/build/firmware.sh

Lines changed: 86 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,86 @@
1+
#!/bin/sh
2+
set -e
3+
4+
# shellcheck disable=SC2034
5+
FIRMWARE_OUTPUT_PATH="${ADDONS_OUTPUT_PATH}/firmware"
6+
7+
# Any firmwares (if any needed) that should ultimately end up in the squashfs
8+
# should be sitting at this location when this script finishes.
9+
# (will go in ${ADDONS_PATH}/firmware, siblings with `${ADDONS_PATH}/modules`)
10+
mkdir -p "${FIRMWARE_OUTPUT_PATH}"
11+
12+
if [ -n "${FIRMWARE_SIG_URL}" ]; then
13+
# Getting the pubkey from the signature is slightly stupid, ideally we should maintain
14+
# a list of valid keys out-of-band, this is best-effort.
15+
echo "Found firmware signature $FIRMWARE_SIG_URL, attempting validation"
16+
17+
# Check if FIRMWARE_URL exists and is a regular file
18+
if [ ! -f "$FIRMWARE_URL" ]; then
19+
echo "ERROR: $FIRMWARE_URL does not exist or is not a regular file"
20+
echo "ERROR: If this was intentional, consider unsetting '$FIRMWARE_SIG_URL'"
21+
exit 1
22+
fi
23+
24+
KEY_INFO=$(gpg --verify "$FIRMWARE_SIG_URL" "$FIRMWARE_URL" 2>&1) || true
25+
KEY_ID=$(echo "$KEY_INFO" | grep -E "using .* key|key ID" | grep -oE "[A-F0-9]{40}|[A-F0-9]{16,}")
26+
gpg --recv-key "$KEY_ID"
27+
unxz "$FIRMWARE_URL"
28+
# We've uncompressed it, update the env var so later stuff points at the right file
29+
FIRMWARE_URL="${FIRMWARE_URL%.xz}"
30+
gpg --verify "$FIRMWARE_SIG_URL" || { echo "ERROR: signature ${FIRMWARE_SIG_URL} cannot validate ${FIRMWARE_URL}"; exit 1; }
31+
else
32+
echo "No firmware signature defined, no validation will be performed"
33+
fi
34+
35+
36+
# Note that this assumes the archive is a .tar file, and has already been validated elsewhere.
37+
if [ -n "${FIRMWARE_URL}" ]; then
38+
echo "untarring firmware at $FIRMWARE_URL"
39+
tar -xf "${FIRMWARE_URL}" -C "${FIRMWARE_OUTPUT_PATH}" --strip-components=1
40+
# For amdgpu zone kernel, we only want the amdgpu firmwares, so remove the rest to keep the addons small
41+
if [ "${KERNEL_FLAVOR}" = "zone-amdgpu" ]; then
42+
OLDDIR=$PWD
43+
cd "${FIRMWARE_OUTPUT_PATH}"
44+
find . -maxdepth 1 ! -name 'amdgpu' ! -name "." -exec rm -rf {} +
45+
# Compress firmwares on-disk
46+
# As of 6.x kernels the kconfig explicitly says you must use crc32 or none, not the default crc64.
47+
xz -C crc32 amdgpu/*
48+
cd "${OLDDIR}"
49+
fi
50+
fi
51+
52+
# For nvidia kernel, firmware is distributed via their out-of-tree .run userspace
53+
# package, which we need to fetch and extract.
54+
if [ "${KERNEL_FLAVOR}" = "zone-nvidiagpu" ]; then
55+
# Check that we already set NV_VERSION when building the out-of-tree kmod,
56+
# naturally, we must fetch the matching runtime package.
57+
if [ -z "${NV_VERSION}" ]; then
58+
echo "ERROR: flavor is zone-nvidiagpu but no NV_VERSION is set"
59+
exit 1
60+
fi
61+
62+
OLDDIR=$PWD
63+
64+
# TBH it's probably the same firmware regardless
65+
if [ "${TARGET_ARCH_STANDARD}" = "aarch64" ]; then
66+
NV_RUN_FILE="NVIDIA-Linux-aarch64-${NV_VERSION}.run"
67+
NV_RUN_URL="https://us.download.nvidia.com/XFree86/aarch64/${NV_VERSION}/${NV_RUN_FILE}"
68+
elif [ "${TARGET_ARCH_STANDARD}" = "x86_64" ]; then
69+
NV_RUN_FILE="NVIDIA-Linux-x86_64-${NV_VERSION}.run"
70+
NV_RUN_URL="https://us.download.nvidia.com/XFree86/Linux-x86_64/${NV_VERSION}/${NV_RUN_FILE}"
71+
fi
72+
73+
NV_EXTRACT_PATH="$(mktemp -d)/extracted-${NV_VERSION}"
74+
mkdir -p "$NV_EXTRACT_PATH"
75+
76+
echo "Downloading NVIDIA runtime package for driver ${NV_VERSION} from: $NV_RUN_URL"
77+
curl -L -o "$NV_EXTRACT_PATH/$NV_RUN_FILE" "$NV_RUN_URL"
78+
chmod +x "$NV_EXTRACT_PATH/$NV_RUN_FILE"
79+
"$NV_EXTRACT_PATH/$NV_RUN_FILE" -x --target "$NV_EXTRACT_PATH/out"
80+
# Compress firmwares on-disk
81+
# As of 6.x kernels the kconfig explicitly says you must use crc32 or none, not the default crc64.
82+
xz -C crc32 "$NV_EXTRACT_PATH"/out/firmware/*
83+
ls -lah "$NV_EXTRACT_PATH/out/firmware/"
84+
cp "$NV_EXTRACT_PATH"/out/firmware/*.xz "${FIRMWARE_OUTPUT_PATH}"
85+
cd "${OLDDIR}"
86+
fi

hack/build/nvidiagpu-common.sh

Lines changed: 46 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,46 @@
1+
#!/bin/sh
2+
set -e
3+
4+
if [ "${KERNEL_FLAVOR}" != "zone-nvidiagpu" ]; then
5+
return
6+
fi
7+
8+
NV_KMOD_REPO_OWNER=NVIDIA
9+
NV_KMOD_REPO_NAME=open-gpu-kernel-modules
10+
11+
rm -rf "$NV_EXTRACT_PATH"
12+
13+
echo "Fetching latest nvidia module release"
14+
15+
RELEASE_JSON=$(curl -s "https://api.github.com/repos/${NV_KMOD_REPO_OWNER}/${NV_KMOD_REPO_NAME}/releases/latest")
16+
NV_VERSION=$(echo "$RELEASE_JSON" | grep -o '"tag_name": *"[^"]*"' | sed 's/"tag_name": *"\(.*\)"/\1/')
17+
LATEST_RELEASE_URL=$(echo "$RELEASE_JSON" | grep -o '"tarball_url": *"[^"]*"' | sed 's/"tarball_url": *"\(.*\)"/\1/')
18+
19+
if [ -z "$LATEST_RELEASE_URL" ] || [ -z "$NV_VERSION" ]; then
20+
echo "Failed to fetch latest release information"
21+
exit 1
22+
fi
23+
24+
echo "Building NVIDIA driver version: $NV_VERSION"
25+
26+
NV_WORKDIR="$(mktemp -d)/nvidia-modules/${NV_VERSION}"
27+
ARCHIVE="$NV_WORKDIR/driver-src.tar.gz"
28+
29+
mkdir -p "$NV_WORKDIR"
30+
31+
curl -L -o "$ARCHIVE" "$LATEST_RELEASE_URL"
32+
tar -xzf "$ARCHIVE" -C "$NV_WORKDIR"
33+
34+
OLDPWD=$(pwd)
35+
cd "$NV_WORKDIR"/"$NV_KMOD_REPO_OWNER"-*
36+
37+
make -C . SYSSRC="${KERNEL_SRC}" SYSOUT="${KERNEL_OBJ}" TARGET_ARCH="${TARGET_ARCH_KERNEL}" -j"${KERNEL_BUILD_JOBS}" "${CROSS_COMPILE_MAKE}" modules
38+
39+
echo "Nvidia $NV_VERSION build done"
40+
41+
make -C . SYSSRC="${KERNEL_SRC}" SYSOUT="${KERNEL_OBJ}" TARGET_ARCH="${TARGET_ARCH_KERNEL}" -j"${KERNEL_BUILD_JOBS}" "${CROSS_COMPILE_MAKE}" INSTALL_MOD_PATH="${MODULES_INSTALL_PATH}" modules_install
42+
43+
echo "Nvidia $NV_VERSION install done"
44+
45+
cd "$OLDPWD"
46+

0 commit comments

Comments
 (0)