Skip to content

Commit 24f1102

Browse files
committed
mirror: bpftool: Move LLVM functionality into plugin (fixes #262)
Signed-off-by: Korenberg Mark <socketpair@gmail.com>
1 parent 48f7cc7 commit 24f1102

6 files changed

Lines changed: 269 additions & 61 deletions

File tree

README.md

Lines changed: 20 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -132,8 +132,22 @@ $ cd src
132132
$ EXTRA_LDFLAGS=-static make
133133
```
134134

135-
Note that to use the LLVM disassembler with static builds, we need a static
136-
version of the LLVM library installed on the system:
135+
The LLVM disassembler used for `bpftool prog dump jited` is built as a separate
136+
plugin, at `$(libdir)/bpftool/bpftool-llvm.so`. This keeps the base bpftool free
137+
of the (large) libLLVM dependency.
138+
139+
The plugin path is baked into `bpftool` at build time, so pass the same `libdir`
140+
to both `make` and `make install`. Otherwise `bpftool` looks for the plugin at
141+
`/usr/local/lib/bpftool/bpftool-llvm.so`.
142+
143+
There are two independent static-linking knobs:
144+
145+
- `EXTRA_LDFLAGS=-static` makes the `bpftool` binary itself static. The plugin
146+
is unaffected and stays a shared object.
147+
- `LLVM_LINK_STATIC=1` links libLLVM into the plugin statically. This is also
148+
done automatically when the LLVM installation ships only static libraries.
149+
150+
To embed a static libLLVM into the plugin, build LLVM statically first:
137151

138152
1. Download a precompiled LLVM release or build it locally.
139153

@@ -156,12 +170,13 @@ version of the LLVM library installed on the system:
156170
$ make -j -C llvm_build llvm-config llvm-libraries
157171
```
158172

159-
2. Build bpftool with `EXTRA_LDFLAGS` set to `-static`, and by passing the
160-
path to the relevant `llvm-config`.
173+
2. Build bpftool with `LLVM_LINK_STATIC=1`, passing the path to the relevant
174+
`llvm-config` and the `libdir` the plugin will be installed under (add
175+
`EXTRA_LDFLAGS=-static` too if you also want a static `bpftool` binary).
161176

162177
```console
163178
$ cd bpftool
164-
$ LLVM_CONFIG=../../llvm_build/bin/llvm-config EXTRA_LDFLAGS=-static make -j -C src
179+
$ LLVM_CONFIG=../../llvm_build/bin/llvm-config LLVM_LINK_STATIC=1 libdir=/usr/lib make -j -C src
165180
```
166181

167182
### Build bpftool's man pages

src/.gitignore

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,7 @@
33
*.d
44
/bootstrap/
55
/bpftool
6+
/bpftool-llvm.so
67
FEATURE-DUMP.bpftool
78
feature
89
libbpf

src/Makefile

Lines changed: 56 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -60,6 +60,7 @@ $(LIBBPF_BOOTSTRAP)-clean: FORCE | $(LIBBPF_BOOTSTRAP_OUTPUT)
6060
$(Q)$(MAKE) -C $(BPF_DIR) OUTPUT=$(LIBBPF_BOOTSTRAP_OUTPUT) clean >/dev/null
6161

6262
prefix ?= /usr/local
63+
libdir ?= $(prefix)/lib
6364
bash_compdir ?= /usr/share/bash-completion/completions
6465

6566
CFLAGS += -O2
@@ -151,6 +152,8 @@ include $(wildcard $(OUTPUT)*.d)
151152
all: $(OUTPUT)bpftool
152153

153154
SRCS := $(wildcard *.c)
155+
# llvm_disasm.c is compiled separately into the bpftool-llvm.so plugin.
156+
SRCS := $(filter-out llvm_disasm.c,$(SRCS))
154157

155158
ifeq ($(feature-llvm),1)
156159
ifneq ($(SKIP_LLVM),1)
@@ -159,19 +162,38 @@ endif
159162
endif
160163

161164
ifeq ($(HAS_LLVM),1)
165+
# The libLLVM-based JIT disassembler is built as a separate plugin,
166+
# bpftool-llvm.so, which is the only object that links against libLLVM.
167+
# bpftool loads it lazily with dlopen() (see jit_disasm.c), so the bpftool
168+
# binary itself keeps no dependency on the large libLLVM shared object.
162169
CFLAGS += -DHAVE_LLVM_SUPPORT
170+
CFLAGS += -DLLVM_PLUGIN_DIR='"$(libdir)/bpftool"'
171+
# dlopen() lives in libc on modern glibc, but keep -ldl for portability.
172+
LIBS += -ldl
173+
174+
# Flags used to build the plugin itself (the only part that needs libLLVM).
163175
LLVM_CONFIG_LIB_COMPONENTS := mcdisassembler all-targets
164-
# llvm-config always adds -D_GNU_SOURCE, however, it may already be in CFLAGS
165-
# (e.g. when bpftool build is called from selftests build as selftests
166-
# Makefile includes lib.mk which sets -D_GNU_SOURCE) which would cause
167-
# compilation error due to redefinition. Let's filter it out here.
168-
CFLAGS += $(filter-out -D_GNU_SOURCE,$(shell $(LLVM_CONFIG) --cflags))
169-
LIBS += $(shell $(LLVM_CONFIG) --libs $(LLVM_CONFIG_LIB_COMPONENTS))
170-
ifeq ($(shell $(LLVM_CONFIG) --shared-mode),static)
171-
LIBS += $(shell $(LLVM_CONFIG) --system-libs $(LLVM_CONFIG_LIB_COMPONENTS))
172-
LIBS += -lstdc++
176+
# llvm-config always adds -D_GNU_SOURCE, which llvm_disasm.c already defines;
177+
# filter it out to avoid a redefinition warning.
178+
LLVM_PLUGIN_CFLAGS := $(filter-out -D_GNU_SOURCE,$(shell $(LLVM_CONFIG) --cflags))
179+
180+
# Embed libLLVM into the plugin statically when requested with
181+
# LLVM_LINK_STATIC=1, or when this LLVM install only ships static libraries
182+
# ("llvm-config --shared-mode" reports "static"). Otherwise link the shared
183+
# libLLVM, which is the only runtime dependency of the plugin.
184+
ifeq ($(LLVM_LINK_STATIC),1)
185+
LLVM_STATIC := 1
186+
else ifeq ($(shell $(LLVM_CONFIG) --shared-mode),static)
187+
LLVM_STATIC := 1
188+
endif
189+
ifeq ($(LLVM_STATIC),1)
190+
LLVM_PLUGIN_LIBS := $(shell $(LLVM_CONFIG) --link-static --libs $(LLVM_CONFIG_LIB_COMPONENTS))
191+
LLVM_PLUGIN_LIBS += $(shell $(LLVM_CONFIG) --link-static --system-libs $(LLVM_CONFIG_LIB_COMPONENTS))
192+
LLVM_PLUGIN_LIBS += -lstdc++
193+
else
194+
LLVM_PLUGIN_LIBS := $(shell $(LLVM_CONFIG) --libs $(LLVM_CONFIG_LIB_COMPONENTS))
173195
endif
174-
LDFLAGS += $(shell $(LLVM_CONFIG) --ldflags)
196+
LLVM_PLUGIN_LDFLAGS := $(shell $(LLVM_CONFIG) --ldflags)
175197
else
176198
ifneq ($(SKIP_LIBBFD),1)
177199
# Fall back on libbfd
@@ -270,6 +292,20 @@ $(BPFTOOL_BOOTSTRAP): $(BOOTSTRAP_OBJS) $(LIBBPF_BOOTSTRAP)
270292
$(OUTPUT)bpftool: $(OBJS) $(LIBBPF)
271293
$(QUIET_LINK)$(CC) $(CFLAGS) $(LDFLAGS) $(OBJS) $(LIBS) -o $@
272294

295+
ifeq ($(HAS_LLVM),1)
296+
all: $(OUTPUT)bpftool-llvm.so
297+
298+
$(OUTPUT)llvm_disasm.o: llvm_disasm.c
299+
$(QUIET_CC)$(CC) $(CFLAGS) $(LLVM_PLUGIN_CFLAGS) -fPIC -c -MMD $< -o $@
300+
301+
# The plugin is a shared object by definition, so drop a global -static (e.g.
302+
# from EXTRA_LDFLAGS for a static bpftool) which would conflict with -shared.
303+
# Embedding libLLVM statically is controlled separately (see LLVM_LINK_STATIC).
304+
$(OUTPUT)bpftool-llvm.so: $(OUTPUT)llvm_disasm.o
305+
$(QUIET_LINK)$(CC) $(CFLAGS) $(filter-out -static,$(LDFLAGS)) \
306+
$(LLVM_PLUGIN_LDFLAGS) -shared -o $@ $< $(LLVM_PLUGIN_LIBS)
307+
endif
308+
273309
$(BOOTSTRAP_OUTPUT)%.o: %.c $(LIBBPF_BOOTSTRAP_INTERNAL_HDRS) | $(BOOTSTRAP_OUTPUT)
274310
$(QUIET_CC)$(HOSTCC) $(HOST_CFLAGS) -c -MMD $< -o $@
275311

@@ -282,17 +318,25 @@ feature-detect-clean:
282318

283319
clean: $(LIBBPF)-clean $(LIBBPF_BOOTSTRAP)-clean
284320
$(call QUIET_CLEAN, bpftool)
285-
$(Q)$(RM) -- $(OUTPUT)bpftool $(OUTPUT)*.o $(OUTPUT)*.d
321+
$(Q)$(RM) -- $(OUTPUT)bpftool $(OUTPUT)bpftool-llvm.so $(OUTPUT)*.o $(OUTPUT)*.d
286322
$(Q)$(RM) -- $(OUTPUT)*.skel.h $(OUTPUT)vmlinux.h
287323
$(Q)$(RM) -r -- $(LIBBPF_OUTPUT) $(BOOTSTRAP_OUTPUT)
288324
$(call QUIET_CLEAN, core-gen)
289325
$(Q)$(RM) -- $(OUTPUT)FEATURE-DUMP.bpftool
290326
$(Q)$(RM) -r -- $(OUTPUT)feature/
291327

328+
ifeq ($(HAS_LLVM),1)
329+
install-bin: $(OUTPUT)bpftool-llvm.so
330+
endif
292331
install-bin: $(OUTPUT)bpftool
293332
$(call QUIET_INSTALL, bpftool)
294333
$(Q)$(INSTALL) -m 0755 -d $(DESTDIR)$(prefix)/sbin
295334
$(Q)$(INSTALL) $(OUTPUT)bpftool $(DESTDIR)$(prefix)/sbin/bpftool
335+
ifeq ($(HAS_LLVM),1)
336+
$(call QUIET_INSTALL, bpftool-llvm.so)
337+
$(Q)$(INSTALL) -m 0755 -d $(DESTDIR)$(libdir)/bpftool
338+
$(Q)$(INSTALL) -m 0755 $(OUTPUT)bpftool-llvm.so $(DESTDIR)$(libdir)/bpftool/bpftool-llvm.so
339+
endif
296340

297341
install: install-bin
298342
$(Q)$(INSTALL) -m 0755 -d $(DESTDIR)$(bash_compdir)
@@ -301,6 +345,7 @@ install: install-bin
301345
uninstall:
302346
$(call QUIET_UNINST, bpftool)
303347
$(Q)$(RM) -- $(DESTDIR)$(prefix)/sbin/bpftool
348+
$(Q)$(RM) -- $(DESTDIR)$(libdir)/bpftool/bpftool-llvm.so
304349
$(Q)$(RM) -- $(DESTDIR)$(bash_compdir)/bpftool
305350

306351
doc:

src/jit_disasm.c

Lines changed: 69 additions & 45 deletions
Original file line numberDiff line numberDiff line change
@@ -25,10 +25,9 @@
2525
#include <bpf/libbpf.h>
2626

2727
#ifdef HAVE_LLVM_SUPPORT
28-
#include <llvm-c/Core.h>
29-
#include <llvm-c/Disassembler.h>
30-
#include <llvm-c/Target.h>
31-
#include <llvm-c/TargetMachine.h>
28+
#include <dlfcn.h>
29+
30+
#include "llvm_disasm.h"
3231
#endif
3332

3433
#ifdef HAVE_LIBBFD_SUPPORT
@@ -45,7 +44,32 @@ static int oper_count;
4544
#ifdef HAVE_LLVM_SUPPORT
4645
#define DISASM_SPACER
4746

48-
typedef LLVMDisasmContextRef disasm_ctx_t;
47+
/*
48+
* The libLLVM-based disassembler used for "bpftool prog dump jited" lives in a
49+
* separate plugin, bpftool-llvm.so, which is the only object linked against
50+
* libLLVM. This keeps the bpftool binary itself free of a hard dependency on
51+
* the (large) libLLVM shared object: the plugin is loaded lazily with dlopen()
52+
* the first time a JITed image actually needs to be disassembled, with its
53+
* entry points resolved by dlsym(). See llvm_disasm.c for the plugin.
54+
*
55+
* LLVM_PLUGIN_DIR is the install directory baked in at build time
56+
* ($(libdir)/bpftool). When set, the plugin is loaded from that absolute
57+
* location; otherwise only the bare file name is used, i.e. the plugin is
58+
* looked up via the dynamic linker search path (or the current directory).
59+
*/
60+
#ifdef LLVM_PLUGIN_DIR
61+
#define LLVM_PLUGIN_PATH LLVM_PLUGIN_DIR "/bpftool-llvm.so"
62+
#else
63+
#define LLVM_PLUGIN_PATH "bpftool-llvm.so"
64+
#endif
65+
66+
typedef void *disasm_ctx_t;
67+
68+
static void *llvm_plugin_handle;
69+
static __typeof__(&bpftool_llvm_init) p_bpftool_llvm_init;
70+
static __typeof__(&bpftool_llvm_create_context) p_bpftool_llvm_create_context;
71+
static __typeof__(&bpftool_llvm_destroy_context) p_bpftool_llvm_destroy_context;
72+
static __typeof__(&bpftool_llvm_disassemble) p_bpftool_llvm_disassemble;
4973

5074
static int printf_json(char *s)
5175
{
@@ -63,18 +87,39 @@ static int printf_json(char *s)
6387
return 0;
6488
}
6589

66-
/* This callback to set the ref_type is necessary to have the LLVM disassembler
67-
* print PC-relative addresses instead of byte offsets for branch instruction
68-
* targets.
69-
*/
70-
static const char *
71-
symbol_lookup_callback(__maybe_unused void *disasm_info,
72-
__maybe_unused uint64_t ref_value,
73-
uint64_t *ref_type, __maybe_unused uint64_t ref_PC,
74-
__maybe_unused const char **ref_name)
90+
static int load_llvm_plugin(void)
7591
{
76-
*ref_type = LLVMDisassembler_ReferenceType_InOut_None;
77-
return NULL;
92+
if (llvm_plugin_handle)
93+
return 0;
94+
95+
/* Load the plugin by its absolute install path. */
96+
llvm_plugin_handle = dlopen(LLVM_PLUGIN_PATH, RTLD_NOW | RTLD_LOCAL);
97+
if (!llvm_plugin_handle) {
98+
p_err("failed to load %s, install it to disassemble JITed programs: %s",
99+
LLVM_PLUGIN_PATH, dlerror());
100+
return -1;
101+
}
102+
103+
#define RESOLVE(name) \
104+
do { \
105+
p_##name = (__typeof__(p_##name))dlsym(llvm_plugin_handle, \
106+
#name); \
107+
if (!p_##name) { \
108+
p_err("%s is missing symbol %s: %s", \
109+
LLVM_PLUGIN_PATH, #name, dlerror()); \
110+
dlclose(llvm_plugin_handle); \
111+
llvm_plugin_handle = NULL; \
112+
return -1; \
113+
} \
114+
} while (0)
115+
116+
RESOLVE(bpftool_llvm_init);
117+
RESOLVE(bpftool_llvm_create_context);
118+
RESOLVE(bpftool_llvm_destroy_context);
119+
RESOLVE(bpftool_llvm_disassemble);
120+
#undef RESOLVE
121+
122+
return 0;
78123
}
79124

80125
static int
@@ -83,28 +128,7 @@ init_context(disasm_ctx_t *ctx, const char *arch,
83128
__maybe_unused unsigned char *image, __maybe_unused ssize_t len,
84129
__maybe_unused __u64 func_ksym)
85130
{
86-
char *triple;
87-
88-
if (arch)
89-
triple = LLVMNormalizeTargetTriple(arch);
90-
else
91-
triple = LLVMGetDefaultTargetTriple();
92-
if (!triple) {
93-
p_err("Failed to retrieve triple");
94-
return -1;
95-
}
96-
97-
/*
98-
* Enable all aarch64 ISA extensions so the disassembler can handle any
99-
* instruction the kernel JIT might emit (e.g. ARM64 LSE atomics).
100-
*/
101-
if (!strncmp(triple, "aarch64", 7))
102-
*ctx = LLVMCreateDisasmCPUFeatures(triple, "", "+all", NULL, 0, NULL,
103-
symbol_lookup_callback);
104-
else
105-
*ctx = LLVMCreateDisasm(triple, NULL, 0, NULL, symbol_lookup_callback);
106-
LLVMDisposeMessage(triple);
107-
131+
*ctx = p_bpftool_llvm_create_context(arch);
108132
if (!*ctx) {
109133
p_err("Failed to create disassembler");
110134
return -1;
@@ -115,7 +139,7 @@ init_context(disasm_ctx_t *ctx, const char *arch,
115139

116140
static void destroy_context(disasm_ctx_t *ctx)
117141
{
118-
LLVMDisposeMessage(*ctx);
142+
p_bpftool_llvm_destroy_context(*ctx);
119143
}
120144

121145
static int
@@ -125,8 +149,8 @@ disassemble_insn(disasm_ctx_t *ctx, unsigned char *image, ssize_t len, int pc,
125149
char buf[256];
126150
int count;
127151

128-
count = LLVMDisasmInstruction(*ctx, image + pc, len - pc, func_ksym + pc,
129-
buf, sizeof(buf));
152+
count = p_bpftool_llvm_disassemble(*ctx, image, len, pc, func_ksym,
153+
buf, sizeof(buf));
130154
if (json_output)
131155
printf_json(buf);
132156
else
@@ -137,10 +161,10 @@ disassemble_insn(disasm_ctx_t *ctx, unsigned char *image, ssize_t len, int pc,
137161

138162
int disasm_init(void)
139163
{
140-
LLVMInitializeAllTargetInfos();
141-
LLVMInitializeAllTargetMCs();
142-
LLVMInitializeAllDisassemblers();
143-
return 0;
164+
if (load_llvm_plugin())
165+
return -1;
166+
167+
return p_bpftool_llvm_init();
144168
}
145169
#endif /* HAVE_LLVM_SUPPORT */
146170

0 commit comments

Comments
 (0)