Skip to content

Commit 437d37a

Browse files
mpenny-githubmpenny-github
authored
Add support for ubuntu noble (#188)
Update the ci and test suites to run against noble in parallel. Compatability fixes: - Ported xdp-root-shim to libbpf 1.x that ships with noble, while preserving libbpf 0.5 that's included on focal. - Changed the xdp test to only compiles the CLI tool (which has no DPDK deps) instead of also pulling in glb-config-check / glb-director-pcap / glb-director-stub-server (which need the DPDK headers, which are absent on noble). - Added -I .../arch/x86/include/generated flag to support noble's newer 6.8 kernel headers. They require auto-generated asm/* headers that live only in that directory. Without it, the BPF clang compile fails with a missing-header error. - Added $(EXTRA_BPF_INCLUDES) flag. The is an empty-by-default extension hook that lets the Dockerfile/CI inject extra include paths without editing the Makefile. - Converted all 7 BPF maps from the legacy struct bpf_map_def SEC("maps") syntax (rejected by libbpf 1.0+) to the modern BTF-style anonymous-struct SEC(".maps") syntax - Added -g to clang in the BPF Makefile so the resulting ELF actually contains the .BTF section that BTF-style maps require. - Convert the various tentative definitions of debug (from log.h) to proper declarations to avoid linking issues.
1 parent e886f20 commit 437d37a

18 files changed

Lines changed: 308 additions & 57 deletions

File tree

‎.github/workflows/ci.yml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -11,7 +11,7 @@ jobs:
1111
strategy:
1212
fail-fast: false
1313
matrix:
14-
distro: [focal]
14+
distro: [focal, noble]
1515
steps:
1616
- uses: actions/checkout@v6
1717
- name: Run package build ${{ matrix.distro }}

‎.github/workflows/test.yml‎

Lines changed: 9 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -20,7 +20,7 @@ jobs:
2020
strategy:
2121
fail-fast: false
2222
matrix:
23-
distro: [focal]
23+
distro: [focal, noble]
2424
steps:
2525
- name: Checkout code
2626
uses: actions/checkout@v6
@@ -49,7 +49,14 @@ jobs:
4949
fail-fast: false
5050
matrix:
5151
test-suite: [director, director-xdp, healthcheck, redirect]
52-
distro: [focal]
52+
distro: [focal, noble]
53+
exclude:
54+
# The DPDK-based glb-director only builds on focal (DPDK 17 / KNI).
55+
# On other distros the supported forwarder is glb-director-xdp, so
56+
# skip the director suite there. This mirrors the same exclusion in
57+
# script/test-local.
58+
- distro: noble
59+
test-suite: director
5360
steps:
5461
- name: Checkout code
5562
uses: actions/checkout@v6

‎Makefile‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,11 +1,23 @@
1+
# Set GLB_SKIP_DPDK_DIRECTOR=1 to skip building the DPDK glb-director package.
2+
# In that mode we still build glb-director-cli (a runtime dep of glb-director-xdp)
3+
# via GLB_CLI_ONLY=1. This is used on distros where DPDK 17 / KNI is unavailable
4+
# (e.g. Ubuntu noble).
5+
GLB_SKIP_DPDK_DIRECTOR ?=
6+
17
mkdeb:
28
make -C src/glb-redirect mkdeb
39
make -C src/glb-healthcheck mkdeb
410
cd src/glb-director-xdp && script/create-packages
11+
ifeq ($(GLB_SKIP_DPDK_DIRECTOR),1)
12+
cd src/glb-director && GLB_CLI_ONLY=1 script/create-packages
13+
else
514
cd src/glb-director && script/create-packages
15+
endif
616

717
clean:
818
make -C src/glb-redirect clean
919
make -C src/glb-healthcheck clean
20+
ifneq ($(GLB_SKIP_DPDK_DIRECTOR),1)
1021
make -C src/glb-director clean
22+
endif
1123
make -C src/glb-director/cli clean

‎script/Dockerfile.noble‎

Lines changed: 107 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,107 @@
1+
FROM --platform=linux/amd64 ubuntu:noble@sha256:c4a8d5503dfb2a3eb8ab5f807da5bc69a85730fb49b5cfca2330194ebcc41c7b
2+
3+
ARG DEBIAN_FRONTEND=noninteractive
4+
5+
RUN apt-get update && apt-get -y install curl git
6+
7+
# Build deps for glb-director-cli and glb-director-xdp.
8+
# Note: we do NOT install DPDK on noble. The DPDK-based glb-director is only
9+
# built on focal; on noble the supported forwarder is glb-director-xdp.
10+
RUN apt-get update && apt-get install --assume-yes \
11+
build-essential \
12+
wget \
13+
pkg-config \
14+
libjansson-dev \
15+
libsystemd-dev
16+
17+
# iptables / DKMS
18+
RUN apt-get update
19+
RUN apt-get install --assume-yes --fix-broken \
20+
wget \
21+
pkg-config \
22+
libsystemd-dev \
23+
dkms \
24+
dh-dkms \
25+
dpkg-dev \
26+
fakeroot \
27+
debhelper \
28+
libxtables-dev
29+
30+
# noble's dkms 3.x dropped `dkms mkdeb`, which the glb-redirect Makefile needs
31+
# (template-dkms-mkdeb + `dkms mkdeb --source-only`). Restore it by extracting
32+
# focal's dkms 2.x alongside 3.x and shimming `dkms mkdeb` to the legacy binary.
33+
#
34+
# We `apt-get download` the focal dkms from a GPG-verified focal source (full
35+
# apt chain of trust, no hardcoded/unverified URL). The focal pockets are
36+
# pinned below noble's so they're only used for this download, never installs.
37+
RUN set -eux; \
38+
keyring=/usr/share/keyrings/ubuntu-archive-keyring.gpg; \
39+
printf 'deb [signed-by=%s] http://archive.ubuntu.com/ubuntu focal main\ndeb [signed-by=%s] http://archive.ubuntu.com/ubuntu focal-updates main\n' "$keyring" "$keyring" > /etc/apt/sources.list.d/focal-dkms.list; \
40+
printf 'Package: *\nPin: release n=focal\nPin-Priority: 100\n\nPackage: *\nPin: release n=focal-updates\nPin-Priority: 100\n' > /etc/apt/preferences.d/focal-dkms.pref; \
41+
apt-get update; \
42+
# Sanity check: the focal pin must not change the dkms candidate (stays 3.x).
43+
case "$(apt-cache policy dkms | awk '/Candidate:/{print $2}')" in 3.*) : ;; *) echo "ERROR: dkms candidate changed by focal pin" >&2; exit 1 ;; esac; \
44+
cd /tmp; \
45+
apt-get download dkms/focal-updates; \
46+
deb="$(ls dkms_*_all.deb)"; \
47+
dpkg-deb -x "$deb" /tmp/dkms-focal; \
48+
cp -r /tmp/dkms-focal/etc/dkms/template-dkms-mkdeb /etc/dkms/template-dkms-mkdeb; \
49+
install -m 0755 /tmp/dkms-focal/usr/sbin/dkms /usr/sbin/dkms-legacy; \
50+
rm -rf /tmp/dkms-focal "/tmp/$deb"; \
51+
# Remove the temporary focal source + pin so they don't linger in the image.
52+
rm -f /etc/apt/sources.list.d/focal-dkms.list /etc/apt/preferences.d/focal-dkms.pref; \
53+
apt-get update; \
54+
printf '#!/bin/sh\nif [ "$1" = "mkdeb" ]; then exec /usr/sbin/dkms-legacy "$@"; fi\nexec /usr/sbin/dkms.real "$@"\n' > /usr/local/bin/dkms; \
55+
chmod 0755 /usr/local/bin/dkms; \
56+
mv /usr/sbin/dkms /usr/sbin/dkms.real
57+
58+
# golang
59+
RUN ARCH=$(dpkg --print-architecture) && wget --quiet https://golang.org/dl/go1.24.5.linux-${ARCH}.tar.gz -O- | tar -C /usr/local -zxvf -
60+
ENV GOROOT /usr/local/go
61+
ENV GOPATH /go
62+
ENV GOFLAGS=-buildvcs=false
63+
ENV PATH="${GOPATH}/bin:${GOROOT}/bin:${PATH}"
64+
65+
66+
# fpm for packaging
67+
RUN apt-get update && apt-get install -y ruby ruby-dev rubygems build-essential
68+
69+
# See fpm dependency breakage issue: https://github.com/jordansissel/fpm/issues/1918
70+
RUN gem install --version 2.7.6 dotenv
71+
RUN gem install ffi -f
72+
RUN gem install rake fpm
73+
74+
# XDP
75+
# linux-libc-dev must be upgraded to get a bpf.h that matches what we use. the rest match what we do in Vagrant for testing.
76+
RUN apt-get update && apt install -y apt-transport-https curl software-properties-common
77+
RUN apt-get update && apt install -y iproute2 libbpf-dev linux-libc-dev clang-20 clang-tools-20
78+
79+
# The xdp bpf/Makefile defaults CLANG/LLC to clang-10/llc-10 (focal). On noble
80+
# we ship clang-20 instead, so steer the BPF build at it.
81+
ENV CLANG=clang-20
82+
ENV LLC=llc-20
83+
84+
85+
# Hack because the kernel headers are not installed in the right place (linuxkit vs generic)
86+
RUN ln -s /usr/src/$(ls /usr/src/ | grep generic) /usr/src/linux-headers-$(uname -r)
87+
88+
# Hack for C99 math
89+
RUN sed -i '1s/^/#define __USE_C99_MATH\n/' /usr/src/$(ls /usr/src/ | grep generic)/include/linux/kasan-checks.h
90+
RUN sed -i '2s/^/#include <stdbool.h>\n/' /usr/src/$(ls /usr/src/ | grep generic)/include/linux/kasan-checks.h
91+
92+
# Newer kernel headers (6.8 in noble) added linux/kcsan-checks.h which uses
93+
# `size_t` without pulling stddef.h, breaking the bpf clang build. Inject a
94+
# stddef.h up front, mirroring the kasan-checks.h hack above.
95+
RUN sed -i '1s|^|#include <stddef.h>\n|' /usr/src/$(ls /usr/src/ | grep generic)/include/linux/kcsan-checks.h
96+
97+
98+
# Python test dependencies (scapy/pytest etc.) used by the test suites.
99+
RUN apt-get update && apt-get install -y python3 python3-pip python3-dev
100+
COPY requirements.txt /tmp/requirements.txt
101+
RUN pip3 install --no-cache-dir --break-system-packages -r /tmp/requirements.txt
102+
103+
# valgrind is required by the glb-director test suite
104+
RUN apt-get update && apt-get install -y valgrind
105+
106+
# netcat and jq are required by the glb-healthcheck test suite
107+
RUN apt-get update && apt-get install -y netcat-openbsd jq tcpdump

‎script/cibuild-create-packages‎

Lines changed: 9 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -30,10 +30,18 @@ begin_fold "Building packages"
3030
rm -rf tmp/build/
3131
mkdir -p tmp/build/
3232

33+
# The DPDK glb-director sub-build only works on focal (DPDK 17 / KNI).
34+
# On other distros we skip it but still produce glb-director-cli (a runtime
35+
# dep of glb-director-xdp).
36+
EXTRA_MAKE_ENV=""
37+
if [ "$DISTRO" != "focal" ]; then
38+
EXTRA_MAKE_ENV="GLB_SKIP_DPDK_DIRECTOR=1"
39+
fi
40+
3341
docker run --rm \
3442
--volume "$HOSTPATH":/glb-director \
3543
"glb-director-build-$DISTRO" \
3644
bash -c "cd /glb-director &&
37-
make BUILDDIR=/glb-director/tmp/build clean mkdeb"
45+
make BUILDDIR=/glb-director/tmp/build $EXTRA_MAKE_ENV clean mkdeb"
3846
)
3947
end_fold

‎script/test-local‎

Lines changed: 7 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -25,7 +25,13 @@ cd "$HOSTPATH"
2525
echo "==> Building Docker image for ${DISTRO}..."
2626
docker build --platform linux/amd64 --file "${DOCKERFILE}" --tag "${IMAGE}" .
2727

28-
TEST_SUITES=(director director-xdp healthcheck redirect)
28+
# The DPDK-based glb-director only builds on focal (DPDK 17 / KNI).
29+
# On other distros, skip that suite; XDP is the supported forwarder.
30+
if [[ "$DISTRO" == "focal" ]]; then
31+
TEST_SUITES=(director director-xdp healthcheck redirect)
32+
else
33+
TEST_SUITES=(director-xdp healthcheck redirect)
34+
fi
2935

3036
for suite in "${TEST_SUITES[@]}"; do
3137
echo ""

‎src/glb-director-xdp/bpf/Makefile‎

Lines changed: 5 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,14 +1,14 @@
11
all: glb_encap.o glb_encap_trace.o passer.o tailcall.o
22

3-
CLANG=clang-10
4-
LLC=llc-10
3+
CLANG?=clang-10
4+
LLC?=llc-10
55

66
ifeq ($(KVER),)
77
KVER=$(shell uname -r)
88
endif
99

1010
%.o: %.c
11-
$(CLANG) -c -O2 -emit-llvm -o - -D__KERNEL__ \
11+
$(CLANG) -c -O2 -emit-llvm -o - -D__KERNEL__ -g \
1212
-Wall \
1313
-Wno-gnu-variable-sized-type-not-at-end \
1414
-Wno-address-of-packed-member \
@@ -23,6 +23,8 @@ endif
2323
-I /usr/src/linux-headers-$(KVER:-amd64=-common)/include/uapi \
2424
-I /usr/src/linux-headers-$(KVER:-amd64=-common)/include \
2525
-I /usr/src/linux-headers-$(KVER)/include \
26+
-I /usr/src/linux-headers-$(KVER)/arch/x86/include/generated \
27+
$(EXTRA_BPF_INCLUDES) \
2628
-I include/ \
2729
-I ../.. \
2830
$< > .tmp.ll

‎src/glb-director-xdp/bpf/glb_encap.c‎

Lines changed: 39 additions & 34 deletions
Original file line numberDiff line numberDiff line change
@@ -52,7 +52,12 @@ typedef struct {
5252

5353
/* xdpcap integration */
5454
#include "xdpcap_hook.h"
55-
struct bpf_map_def SEC("maps") xdpcap_hook = XDPCAP_HOOK();
55+
struct {
56+
__uint(type, BPF_MAP_TYPE_PROG_ARRAY);
57+
__uint(key_size, sizeof(int));
58+
__uint(value_size, sizeof(int));
59+
__uint(max_entries, 5);
60+
} xdpcap_hook SEC(".maps");
5661

5762
typedef struct glb_bind {
5863
uint32_t ipv4;
@@ -61,11 +66,11 @@ typedef struct glb_bind {
6166
uint16_t port;
6267
} __attribute__((__packed__)) glb_bind_t;
6368

64-
struct bpf_map_def SEC("maps") config_bits = {
65-
.type = BPF_MAP_TYPE_ARRAY,
66-
.key_size = sizeof(uint32_t),
67-
.value_size = 6, // maximum size stored
68-
.max_entries = 5,
69+
struct {
70+
__uint(type, BPF_MAP_TYPE_ARRAY);
71+
__uint(key_size, sizeof(uint32_t));
72+
__uint(value_size, 6); // maximum size stored
73+
__uint(max_entries, 5);
6974

7075
/*
7176
0: 6 byes of gateway dst MAC
@@ -74,36 +79,36 @@ struct bpf_map_def SEC("maps") config_bits = {
7479
3: 4 bytes of glb_director_hash_fields
7580
4: 4 bytes of glb_director_hash_fields (alt)
7681
*/
77-
};
78-
79-
struct bpf_map_def SEC("maps") glb_binds = {
80-
.type = BPF_MAP_TYPE_HASH,
81-
.key_size = sizeof(struct glb_bind),
82-
.value_size = sizeof(uint32_t),
83-
.max_entries = BPF_MAX_BINDS,
84-
};
85-
86-
struct bpf_map_def SEC("maps") glb_tables = {
87-
.type = BPF_MAP_TYPE_ARRAY_OF_MAPS,
88-
.key_size = sizeof(uint32_t),
89-
.max_entries = 4096,
90-
};
91-
92-
struct bpf_map_def SEC("maps") glb_table_secrets = {
93-
.type = BPF_MAP_TYPE_ARRAY,
94-
.key_size = sizeof(uint32_t),
82+
} config_bits SEC(".maps");
83+
84+
struct {
85+
__uint(type, BPF_MAP_TYPE_HASH);
86+
__uint(key_size, sizeof(struct glb_bind));
87+
__uint(value_size, sizeof(uint32_t));
88+
__uint(max_entries, BPF_MAX_BINDS);
89+
} glb_binds SEC(".maps");
90+
91+
struct {
92+
__uint(type, BPF_MAP_TYPE_ARRAY_OF_MAPS);
93+
__uint(key_size, sizeof(uint32_t));
94+
__uint(max_entries, 4096);
95+
} glb_tables SEC(".maps");
96+
97+
struct {
98+
__uint(type, BPF_MAP_TYPE_ARRAY);
99+
__uint(key_size, sizeof(uint32_t));
95100
#define GLB_FMT_SECURE_KEY_BYTES 16
96-
.value_size = GLB_FMT_SECURE_KEY_BYTES,
97-
.max_entries = 4096,
98-
};
99-
100-
struct bpf_map_def SEC("maps") glb_global_packet_counters = {
101-
.type = BPF_MAP_TYPE_PERCPU_ARRAY,
102-
.key_size = sizeof(uint32_t),
103-
.value_size = sizeof(struct glb_global_stats),
101+
__uint(value_size, GLB_FMT_SECURE_KEY_BYTES);
102+
__uint(max_entries, 4096);
103+
} glb_table_secrets SEC(".maps");
104+
105+
struct {
106+
__uint(type, BPF_MAP_TYPE_PERCPU_ARRAY);
107+
__uint(key_size, sizeof(uint32_t));
108+
__uint(value_size, sizeof(struct glb_global_stats));
104109
/* we don't actually need an array, but PERCPU_* only has multi-element types */
105-
.max_entries = 1,
106-
};
110+
__uint(max_entries, 1);
111+
} glb_global_packet_counters SEC(".maps");
107112

108113
static __always_inline uint16_t compute_ipv4_checksum(void *iph) {
109114
uint16_t *iph16 = (uint16_t *)iph;

‎src/glb-director-xdp/bpf/tailcall.c‎

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -19,12 +19,12 @@
1919

2020
#define ROOT_ARRAY_SIZE 3
2121

22-
struct bpf_map_def SEC("maps") root_array = {
23-
.type = BPF_MAP_TYPE_PROG_ARRAY,
24-
.key_size = sizeof(__u32),
25-
.value_size = sizeof(__u32),
26-
.max_entries = ROOT_ARRAY_SIZE,
27-
};
22+
struct {
23+
__uint(type, BPF_MAP_TYPE_PROG_ARRAY);
24+
__uint(key_size, sizeof(__u32));
25+
__uint(value_size, sizeof(__u32));
26+
__uint(max_entries, ROOT_ARRAY_SIZE);
27+
} root_array SEC(".maps");
2828

2929
SEC("xdp-root")
3030
int xdp_root(struct xdp_md *ctx) {

‎src/glb-director-xdp/bpf/xdpcap_hook.h‎

Lines changed: 10 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -38,7 +38,16 @@
3838
* Create a bpf map suitable for use as an xdpcap hook point.
3939
*
4040
* For example:
41-
* struct bpf_map_def xdpcap_hook = XDPCAP_HOOK();
41+
* struct {
42+
* __uint(type, BPF_MAP_TYPE_PROG_ARRAY);
43+
* __uint(key_size, sizeof(int));
44+
* __uint(value_size, sizeof(int));
45+
* __uint(max_entries, 5);
46+
* } xdpcap_hook SEC(".maps");
47+
*
48+
* The legacy XDPCAP_HOOK() initializer macro is kept for source compat with
49+
* older callers but new code should declare the map directly using the BTF
50+
* style above (libbpf 1.0+ rejects the legacy "maps" section).
4251
*/
4352
#define XDPCAP_HOOK() { \
4453
.type = BPF_MAP_TYPE_PROG_ARRAY, \

0 commit comments

Comments
 (0)