kernel/bpf-developer-tutorial
1
0
Fork 0
mirror of https://github.com/eunomia-bpf/bpf-developer-tutorial.git synced 2026-05-08 06:42:16 +08:00
Code Issues Packages Projects Releases Wiki Activity

Rust and BTF CO-RE for userspace (#103)

Browse Source 
* add rust examples

* update btf

* update btf

* update docs
This commit is contained in:
云微
2024-01-26 03:33:17 +00:00
committed by GitHub
parent 778e36ba90
commit 4671af2739
31 changed files with 1307 additions and 2 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
src/38-btf-uprobe/.gitignore vendored Normal file
View File

@@ -0,0 +1,4 @@
.output
uprobe
merge-btf
*.btf

142
src/38-btf-uprobe/Makefile Normal file
View File

@@ -0,0 +1,142 @@
# SPDX-License-Identifier: (LGPL-2.1 OR BSD-2-Clause)
OUTPUT := .output
CLANG ?= clang
LIBBPF_SRC := $(abspath ../third_party/libbpf/src)
BPFTOOL_SRC := $(abspath ../third_party/bpftool/src)
LIBBPF_OBJ := $(abspath $(OUTPUT)/libbpf.a)
BPFTOOL_OUTPUT ?= $(abspath $(OUTPUT)/bpftool)
BPFTOOL ?= $(BPFTOOL_OUTPUT)/bootstrap/bpftool
ARCH ?= $(shell uname -m | sed 's/x86_64/x86/' \
| sed 's/arm.*/arm/' \
| sed 's/aarch64/arm64/' \
| sed 's/ppc64le/powerpc/' \
| sed 's/mips.*/mips/' \
| sed 's/riscv64/riscv/' \
| sed 's/loongarch64/loongarch/')
VMLINUX := ../third_party/vmlinux/$(ARCH)/vmlinux.h
# Use our own libbpf API headers and Linux UAPI headers distributed with
# libbpf to avoid dependency on system-wide headers, which could be missing or
# outdated
INCLUDES := -I$(OUTPUT) -I../third_party/libbpf/include/ -I../third_party/libbpf/include/uapi -I$(dir $(VMLINUX))
CFLAGS := -g -Wall
ALL_LDFLAGS := $(LDFLAGS) $(EXTRA_LDFLAGS)
APPS = uprobe # minimal minimal_legacy kprobe fentry usdt sockfilter tc ksyscall
CARGO ?= $(shell which cargo)
ifeq ($(strip $(CARGO)),)
BZS_APPS :=
else
BZS_APPS := # profile
APPS += $(BZS_APPS)
# Required by libblazesym
ALL_LDFLAGS += -lrt -ldl -lpthread -lm
endif
# Get Clang's default includes on this system. We'll explicitly add these dirs
# to the includes list when compiling with `-target bpf` because otherwise some
# architecture-specific dirs will be "missing" on some architectures/distros -
# headers such as asm/types.h, asm/byteorder.h, asm/socket.h, asm/sockios.h,
# sys/cdefs.h etc. might be missing.
#
# Use '-idirafter': Don't interfere with include mechanics except where the
# build would have failed anyways.
CLANG_BPF_SYS_INCLUDES ?= $(shell $(CLANG) -v -E - </dev/null 2>&1 \
| sed -n '/<...> search starts here:/,/End of search list./{ s| \(/.*\)|-idirafter \1|p }')
ifeq ($(V),1)
Q =
msg =
else
Q = @
msg = @printf ' %-8s %s%s\n' \
"$(1)" \
"$(patsubst $(abspath $(OUTPUT))/%,%,$(2))" \
"$(if $(3), $(3))";
MAKEFLAGS += --no-print-directory
endif
define allow-override
$(if $(or $(findstring environment,$(origin $(1))),\
$(findstring command line,$(origin $(1)))),,\
$(eval $(1) = $(2)))
endef
$(call allow-override,CC,$(CROSS_COMPILE)cc)
$(call allow-override,LD,$(CROSS_COMPILE)ld)
.PHONY: all
all: $(APPS) merge-btf
.PHONY: clean
clean:
$(call msg,CLEAN)
$(Q)rm -rf $(OUTPUT) $(APPS)
$(OUTPUT) $(OUTPUT)/libbpf $(BPFTOOL_OUTPUT):
$(call msg,MKDIR,$@)
$(Q)mkdir -p $@
# Build libbpf
$(LIBBPF_OBJ): $(wildcard $(LIBBPF_SRC)/*.[ch] $(LIBBPF_SRC)/Makefile) | $(OUTPUT)/libbpf
$(call msg,LIB,$@)
$(Q)$(MAKE) -C $(LIBBPF_SRC) BUILD_STATIC_ONLY=1 \
OBJDIR=$(dir $@)/libbpf DESTDIR=$(dir $@) \
INCLUDEDIR= LIBDIR= UAPIDIR= \
install
# Build bpftool
$(BPFTOOL): | $(BPFTOOL_OUTPUT)
$(call msg,BPFTOOL,$@)
$(Q)$(MAKE) ARCH= CROSS_COMPILE= OUTPUT=$(BPFTOOL_OUTPUT)/ -C $(BPFTOOL_SRC) bootstrap
$(LIBBLAZESYM_SRC)/target/release/libblazesym.a::
$(Q)cd $(LIBBLAZESYM_SRC) && $(CARGO) build --features=cheader,dont-generate-test-files --release
$(LIBBLAZESYM_OBJ): $(LIBBLAZESYM_SRC)/target/release/libblazesym.a | $(OUTPUT)
$(call msg,LIB, $@)
$(Q)cp $(LIBBLAZESYM_SRC)/target/release/libblazesym.a $@
$(LIBBLAZESYM_HEADER): $(LIBBLAZESYM_SRC)/target/release/libblazesym.a | $(OUTPUT)
$(call msg,LIB,$@)
$(Q)cp $(LIBBLAZESYM_SRC)/target/release/blazesym.h $@
# Build BPF code
$(OUTPUT)/%.bpf.o: %.bpf.c $(LIBBPF_OBJ) $(wildcard %.h) $(VMLINUX) | $(OUTPUT) $(BPFTOOL)
$(call msg,BPF,$@)
$(Q)$(CLANG) -g -O2 -target bpf -D__TARGET_ARCH_$(ARCH) \
$(INCLUDES) $(CLANG_BPF_SYS_INCLUDES) \
-c $(filter %.c,$^) -o $(patsubst %.bpf.o,%.tmp.bpf.o,$@)
$(Q)$(BPFTOOL) gen object $@ $(patsubst %.bpf.o,%.tmp.bpf.o,$@)
# Generate BPF skeletons
$(OUTPUT)/%.skel.h: $(OUTPUT)/%.bpf.o | $(OUTPUT) $(BPFTOOL)
$(call msg,GEN-SKEL,$@)
$(Q)$(BPFTOOL) gen skeleton $< > $@
# Build user-space code
$(patsubst %,$(OUTPUT)/%.o,$(APPS)): %.o: %.skel.h
$(OUTPUT)/%.o: %.c $(wildcard %.h) | $(OUTPUT)
$(call msg,CC,$@)
$(Q)$(CC) $(CFLAGS) $(INCLUDES) -c $(filter %.c,$^) -o $@
$(patsubst %,$(OUTPUT)/%.o,$(BZS_APPS)): $(LIBBLAZESYM_HEADER)
$(BZS_APPS): $(LIBBLAZESYM_OBJ)
# Build application binary
$(APPS): %: $(OUTPUT)/%.o $(LIBBPF_OBJ) | $(OUTPUT)
$(call msg,BINARY,$@)
$(Q)$(CC) $(CFLAGS) $^ $(ALL_LDFLAGS) -lelf -lz -o $@
merge-btf: merge-btf.c
$(call msg,CC,$@)
$(Q)$(CC) $(CFLAGS) merge-btf.c $(LIBBPF_OBJ) $(INCLUDES) $(ALL_LDFLAGS) -lelf -lz -o $@
# delete failed targets
.DELETE_ON_ERROR:
# keep intermediate (.skel.h, .bpf.o, etc) targets
.SECONDARY:

266
src/38-btf-uprobe/README.md Normal file
View File

@@ -0,0 +1,266 @@
# 借助 eBPF 和 BTF，让用户态也能一次编译、到处运行
在现代 Linux 系统中，eBPF（扩展的 Berkeley Packet Filter）是一项强大而灵活的技术。它允许在内核中运行沙盒化程序，类似于虚拟机环境，为扩展内核功能提供了一种既安全又不会导致系统崩溃或安全风险的方法。
eBPF 中的 “co-re” 代表“一次编译、到处运行”。这是其关键特征之一，用于解决 eBPF 程序在不同内核版本间兼容性的主要挑战。eBPF 的 CO-RE 功能可以实现在不同的内核版本上运行同一 eBPF 程序，而无需重新编译。
利用 eBPF 的 Uprobe 功能，可以追踪用户空间应用程序并访问其内部数据结构。然而，用户空间应用程序的 CO-RE 实践目前尚不完善。本文将介绍一种新方法，利用 CO-RE 为用户空间应用程序确保 eBPF 程序在不同应用版本间的兼容性，从而避免了多次编译的需求。例如，在从加密流量中捕获 SSL/TLS 明文数据时，你或许不需要为每个版本的 OpenSSL 维护一个单独的 eBPF 程序。
为了在用户空间应用程序中实现eBPF的“一次编译、到处运行”(Co-RE)特性，我们需要利用BPF类型格式(BTF)来克服传统eBPF程序的一些限制。这种方法的关键在于为用户空间程序提供与内核类似的类型信息和兼容性支持，从而使得eBPF程序能够更灵活地应对不同版本的用户空间应用和库。
本文是eBPF开发者教程的一部分，详细内容可访问[这里](https://eunomia.dev/tutorials/)。源代码在[GitHub库](https://github.com/eunomia-bpf/bpf-developer-tutorial)中可用。
## 为什么我们需要CO-RE？
- **内核依赖性**：传统的eBPF程序和它们被编译的特定Linux内核版本紧密耦合。这是因为它们依赖于内核的特定内部数据结构和API，这些可能在内核版本间变化。
- **可移植性问题**：如果你想在带有不同内核版本的不同Linux系统上运行一个eBPF程序，你通常需要为每个内核版本重新编译eBPF程序，这是一个麻烦而低效的过程。
### Co-RE的解决方案
- **抽象内核依赖性**：Co-RE使eBPF程序更具可移植性，通过使用BPF类型格式(BTF)和重定位来抽象特定的内核依赖。
- **BPF类型格式（BTF）**：BTF提供了关于内核中数据结构和函数的丰富类型信息。这些元数据允许eBPF程序在运行时理解内核结构的布局。
- **重定位**：编译支持Co-RE的eBPF程序包含在加载时解析的重定位。这些重定位根据运行内核的实际布局和地址调整程序对内核数据结构和函数的引用。
### Co-RE的优点
1. **编写一次，任何地方运行**：编译有Co-RE的eBPF程序可以在不同的内核版本上运行，无需重新编译。这大大简化了在多样环境中部署和维护eBPF程序。
2. **安全和稳定**：Co-RE保持了eBPF的安全性，确保程序不会导致内核崩溃，遵守安全约束。
3. **简单的开发**：开发者不需要关注每个内核版本的具体情况，这简化了eBPF程序的开发。
## 用户空间应用程序CO-RE的问题
eBPF也支持追踪用户空间应用程序。Uprobe是一个用户空间探针，允许对用户空间程序进行动态仪表装置。探针位置包括函数入口、特定偏移和函数返回。
BTF是为内核设计的，生成自vmlinux，它可以帮助eBPF程序方便地兼容不同的内核版本。
但是，用户空间应用程序也需要CO-RE。例如，SSL/TLS uprobe被广泛用于从加密流量中捕获明文数据。它是用用户空间库实现的，如OpenSSL、GnuTLS、NSS等。用户空间应用程序和库也有各种版本，如果我们需要为每个版本编译和维护eBPF程序，那就会很复杂。
下面是一些新的工具和方法来帮助我们为用户空间应用程序启用CO-RE。
## 用户空间程序的BTF
这是一个简单的uprobe例子，它可以捕获用户空间程序的`add_test`函数的调用和参数。你可以在`uprobe.bpf.c`中添加`#define BPF_NO_PRESERVE_ACCESS_INDEX`来确保eBPF程序可以在没有`struct data`的BTF的情况下编译。
```c
#define BPF_NO_GLOBAL_DATA
#define BPF_NO_PRESERVE_ACCESS_INDEX
#include <vmlinux.h>
#include <bpf/bpf_helpers.h>
#include <bpf/bpf_tracing.h>
struct data {
int a;
int c;
int d;
};
SEC("uprobe/examples/btf-base:add_test")
int BPF_UPROBE(add_test, struct data *d)
{
int a = 0, c = 0;
bpf_probe_read_user(&a, sizeof(a), &d->a);
bpf_probe_read_user(&c, sizeof(c), &d->c);
bpf_printk("add_test(&d) %d + %d = %d\n", a, c, a + c);
return a + c;
}
char LICENSE[] SEC("license") = "Dual BSD/GPL";
```
然后，我们有两个不同版本的用户空间程序，`examples/btf-base`和`examples/btf-base-new`。两个版本中的struct `data`是不同的。
`examples/btf-base`：
```c
// use a different struct
struct data {
int a;
int c;
int d;
};
int add_test(struct data *d) {
return d->a + d->c;
}
int main(int argc, char **argv) {
struct data d = {1, 3, 4};
printf("add_test(&d) = %d\n", add_test(&d));
return 0;
}
```
`examples/btf-base-new`：
```c
struct data {
int a;
int b;
int c;
int d;
};
int add_test(struct data *d) {
return d->a + d->c;
}
int main(int argc, char **argv) {
struct data d = {1, 2, 3, 4};
printf("add_test(&d) = %d\n", add_test(&d));
return 0;
}
```
我们可以使用pahole和clang来生成每个版本的btf。制作示例并生成btf:
```sh
make -C example # it's like: pahole --btf_encode_detached base.btf btf-base.o
```
然后我们执行eBPF程序和用户空间程序。 对于 `btf-base`：
```sh
sudo ./uprobe examples/btf-base
```
也是用户空间程序：
```console
$ examples/btf-base
add_test(&d) = 4
```
我们将看到：
```console
$ sudo cat /sys/kernel/debug/tracing/trace_pipe\
<...>-25458 [000] ...11 27694.081465: bpf_trace_printk: add_test(&d) 1 + 3 = 4
```
对于 `btf-base-new`：
```sh
sudo ./uprobe examples/btf-base-new
```
同时也是用户空间程序：
```console
$ examples/btf-base-new
add_test(&d) = 4
```
但我们可以看到：
```console
$ sudo cat /sys/kernel/debug/tracing/trace_pipe\
<...>-25809 [001] ...11 27828.314224: bpf_trace_printk: add_test(&d) 1 + 2 = 3
```
结果是不同的，因为两个版本中的struct `data`是不同的。eBPF程序无法与不同版本的用户空间程序兼容。
## 使用用户空间程序的BTF
在`uprobe.bpf.c`中注释掉`#define BPF_NO_PRESERVE_ACCESS_INDEX` ，以确保eBPF程序可以以`struct data`的BTF编译。
```c
#define BPF_NO_GLOBAL_DATA
// #define BPF_NO_PRESERVE_ACCESS_INDEX
#include <vmlinux.h>
#include <bpf/bpf_helpers.h>
#include <bpf/bpf_tracing.h>
#ifndef BPF_NO_PRESERVE_ACCESS_INDEX
#pragma clang attribute push (__attribute__((preserve_access_index)), apply_to = record)
#endif
struct data {
int a;
int c;
int d;
};
#ifndef BPF_NO_PRESERVE_ACCESS_INDEX
#pragma clang attribute pop
#endif
SEC("uprobe/examples/btf-base:add_test")
int BPF_UPROBE(add_test, struct data *d)
{
int a = 0, c = 0;
bpf_probe_read_user(&a, sizeof(a), &d->a);
bpf_probe_read_user(&c, sizeof(c), &d->c);
bpf_printk("add_test(&d) %d + %d = %d\n", a, c, a + c);
return a + c;
}
char LICENSE[] SEC("license") = "Dual BSD/GPL";
```
`struct data`的记录在eBPF程序中被保留下来。然后，我们可以使用 `btf-base.btf`来编译eBPF程序。
将用户btf与内核btf合并，这样我们就有了一个完整的内核和用户空间的btf:
```sh
./merge-btf /sys/kernel/btf/vmlinux examples/base.btf target-base.btf
```
然后我们使用用户空间程序执行eBPF程序。 对于 `btf-base`：
```console
$ sudo ./uprobe examples/btf-base target-base.btf
...
libbpf: prog 'add_test': relo #1: patched insn #4 (ALU/ALU64) imm 0 -> 0
libbpf: prog 'add_test': relo #2: <byte_off> [7] struct data.c (0:1 @ offset 4)
libbpf: prog 'add_test': relo #2: matching candidate #0 <byte_off> [133110] struct data.c (0:1 @ offset 4)
libbpf: prog 'add_test': relo #2: patched insn #11 (ALU/ALU64) imm 4 -> 4
...
```
执行用户空间程序并获取结果：
```console
$ sudo cat /sys/kernel/debug/tracing/trace_pipe
[sudo] password for yunwei37:
<...>-26740 [001] ...11 28180.156220: bpf_trace_printk: add_test(&d) 1 + 3 = 4
```
还可以对另一个版本的用户空间程序`btf-base-new`做同样的操作:
```console
$ ./merge-btf /sys/kernel/btf/vmlinux examples/base-new.btf target-base-new.btf
$ sudo ./uprobe examples/btf-base-new target-base-new.btf
....
libbpf: sec 'uprobe/examples/btf-base:add_test': found 3 CO-RE relocations
libbpf: CO-RE relocating [2] struct pt_regs: found target candidate [357] struct pt_regs in [vmlinux]
libbpf: prog 'add_test': relo #0: <byte_off> [2] struct pt_regs.di (0:14 @ offset 112)
libbpf: prog 'add_test': relo #0: matching candidate #0 <byte_off> [357] struct pt_regs.di (0:14 @ offset 112)
libbpf: prog 'add_test': relo #0: patched insn #0 (LDX/ST/STX) off 112 -> 112
libbpf: CO-RE relocating [7] struct data: found target candidate [133110] struct data in [vmlinux]
libbpf: prog 'add_test': relo #1: <byte_off> [7] struct data.a (0:0 @ offset 0)
libbpf: prog 'add_test': relo #1: matching candidate #0 <byte_off> [133110] struct data.a (0:0 @ offset 0)
libbpf: prog 'add_test': relo #1: patched insn #4 (ALU/ALU64) imm 0 -> 0
libbpf: prog 'add_test': relo #2: <byte_off> [7] struct data.c (0:1 @ offset 4)
libbpf: prog 'add_test': relo #2: matching candidate #0 <byte_off> [133110] struct data.c (0:2 @ offset 8)
libbpf: prog 'add_test': relo #2: patched insn #11 (ALU/ALU64) imm 4 -> 8
libbpf: elf: symbol address match for 'add_test' in 'examples/btf-base-new': 0x1140
Successfully started! Press Ctrl+C to stop.
```
结果是正确的：
```console
$ sudo cat /sys/kernel/debug/tracing/trace_pipe
[sudo] password for yunwei37:
<...>-26740 [001] ...11 28180.156220: bpf_trace_printk: add_test(&d) 1 + 3 = 4
```
## 结论
- **灵活性和兼容性**：在用户空间eBPF程序中使用BTF大大增强了它们在不同版本的用户空间应用程序和库之间的灵活性和兼容性。
- **简化了复杂性**：这种方法显著减少了维护不同版本的用户空间应用程序的eBPF程序的复杂性，因为它消除了需要多个程序版本的需要。
- **更广泛的应用**：虽然你的例子关注于SSL/TLS监控，但是这种方法在性能监控、安全和用户空间应用程序的调试等方面有更广泛的应用。
这个示例展示了eBPF在实践中的重要进步，将其强大的功能扩展到更动态地处理用户空间应用程序在Linux环境中。对于处理现代Linux系统复杂性的软件工程师和系统管理员来说，这是一个引人注目的解决方案。
如果你想了解更多关于eBPF知识和实践，你可以访问我们的教程代码库<https://github.com/eunomia-bpf/bpf-developer-tutorial>或者网站<https://eunomia.dev/tutorials/>获得更多示例和完整教程。

267
src/38-btf-uprobe/README_en.md Normal file
View File

@@ -0,0 +1,267 @@
# Compile Once, Run Everywhere for userspace trace with eBPF and BTF
eBPF, short for extended Berkeley Packet Filter, is a powerful and versatile technology used in modern Linux systems. It allows for the running of sandboxed programs in a virtual machine-like environment within the kernel, providing a safe way to extend the capabilities of the kernel without the risk of crashing the system or compromising security.
The term "co-re" in the context of eBPF stands for "Compile Once, Run Everywhere". This is a key feature of eBPF that addresses a major challenge: the compatibility of eBPF programs across different kernel versions. The CO-RE of eBPF program can help us to run the eBPF program on different kernel versions without the need for recompilation.
With eBPF Uprobe, you can also trace userspace applications and access their internal data structures. However, there is no CO-RE practice for userspace. This blog introduces a new approach to leverage CO-RE for user-space applications, ensuring eBPF programs remain compatible across different application versions without the need for multiple compilations. For example, you may not need to maintain a separate eBPF program for each version of OpenSSL when capturing SSL/TLS plaintext data from encrypted traffic.
This article is part of the eBPF Developer Tutorial, and for more detailed content, you can visit [here](https://eunomia.dev/tutorials/). The source code is available on the [GitHub repository](https://github.com/eunomia-bpf/bpf-developer-tutorial).
## Why we need CO-RE?
- **Kernel Dependencies**: Traditional eBPF programs are tightly coupled with the specific Linux kernel version they are compiled for. This is because they rely on specific internal data structures and kernel APIs which can change between kernel versions.
- **Portability Issues**: If you wanted to run an eBPF program on different Linux systems with different kernel versions, you'd traditionally have to recompile the eBPF program for each kernel version, which is a cumbersome and inefficient process.
### The Co-RE Solution
- **Abstracting Kernel Dependencies**: Co-RE enables eBPF programs to be more portable by abstracting away specific kernel dependencies. This is achieved through the use of BPF Type Format (BTF) and relocations.
- **BPF Type Format (BTF)**: BTF provides rich type information about data structures and functions in the kernel. This metadata allows eBPF programs to understand the layout of kernel structures at runtime.
- **Relocations**: eBPF programs compiled with Co-RE support contain relocations that are resolved at load time. These relocations adjust the program's references to kernel data structures and functions according to the actual layout and addresses in the running kernel.
### Advantages of Co-RE
1. **Write Once, Run Anywhere**: eBPF programs compiled with Co-RE can run on different kernel versions without the need for recompilation. This greatly simplifies the deployment and maintenance of eBPF programs in diverse environments.
2. **Safety and Stability**: Co-RE maintains the safety guarantees of eBPF, ensuring that programs do not crash the kernel and adhere to security constraints.
3. **Ease of Development**: Developers don't need to worry about the specifics of each kernel version, which simplifies the development of eBPF programs.
## Problem: userspace application CO-RE
The eBPF also supports tracing userspace applications. Uprobe is a user-space probe that allows dynamic instrumentation in user-space programs. The probe locations include function entry, specific offsets, and function returns.
The BTF is designed for the kernel and generated from vmlinux, it can help the eBPF program to be easily compatible with different kernel versions.
The userspace application, however, also need CO-RE. For example, the SSL/TLS uprobe is widely used to capture the plaintext data from the encrypted traffic. It is implemented with the userspace library, such as OpenSSL, GnuTLS, NSS, etc. The userspace application and libraries also has different versions, it would be complex if we need to compile and maintain the eBPF program for each version.
Here is some new tools and methods to help us enable CO-RE for userspace application.
## No BTF for userspace program
This is a simple uprobe example, it can capture the function call and arguments of the `add_test` function in the userspace program. You can add `#define BPF_NO_PRESERVE_ACCESS_INDEX` in the `uprobe.bpf.c` to make sure the eBPF program can be compiled without BTF for `struct data`.
```c
#define BPF_NO_GLOBAL_DATA
#define BPF_NO_PRESERVE_ACCESS_INDEX
#include <vmlinux.h>
#include <bpf/bpf_helpers.h>
#include <bpf/bpf_tracing.h>
struct data {
int a;
int c;
int d;
};
SEC("uprobe/examples/btf-base:add_test")
int BPF_UPROBE(add_test, struct data *d)
{
int a = 0, c = 0;
bpf_probe_read_user(&a, sizeof(a), &d->a);
bpf_probe_read_user(&c, sizeof(c), &d->c);
bpf_printk("add_test(&d) %d + %d = %d\n", a, c, a + c);
return a + c;
}
char LICENSE[] SEC("license") = "Dual BSD/GPL";
```
Then, we have two different versions of the userspace program, `examples/btf-base` and `examples/btf-base-new`. The struct `data` is different in the two versions.
`examples/btf-base`:
```c
// use a different struct
struct data {
int a;
int c;
int d;
};
int add_test(struct data *d) {
return d->a + d->c;
}
int main(int argc, char **argv) {
struct data d = {1, 3, 4};
printf("add_test(&d) = %d\n", add_test(&d));
return 0;
}
```
`examples/btf-base-new`:
```c
struct data {
int a;
int b;
int c;
int d;
};
int add_test(struct data *d) {
return d->a + d->c;
}
int main(int argc, char **argv) {
struct data d = {1, 2, 3, 4};
printf("add_test(&d) = %d\n", add_test(&d));
return 0;
}
```
We can use pahole and clang to generate the btf for each version. make example and generate btf:
```sh
make -C example # it's like: pahole --btf_encode_detached base.btf btf-base.o
```
The we execute the eBPF program with the userspace program. for `btf-base`:
```sh
sudo ./uprobe examples/btf-base
```
And also the userspace program:
```console
$ examples/btf-base
add_test(&d) = 4
```
We will see:
```console
$ sudo cat /sys/kernel/debug/tracing/trace_pipe\
<...>-25458 [000] ...11 27694.081465: bpf_trace_printk: add_test(&d) 1 + 3 = 4
```
For `btf-base-new`:
```sh
sudo ./uprobe examples/btf-base-new
```
And also the userspace program:
```console
$ examples/btf-base-new
add_test(&d) = 4
```
But we will see:
```console
$ sudo cat /sys/kernel/debug/tracing/trace_pipe\
<...>-25809 [001] ...11 27828.314224: bpf_trace_printk: add_test(&d) 1 + 2 = 3
```
The result is different, because the struct `data` is different in the two versions. The eBPF program can't be compatible with different versions of the userspace program.
## Use BTF for userspace program
Comment the `#define BPF_NO_PRESERVE_ACCESS_INDEX` in the `uprobe.bpf.c` to make sure the eBPF program can be compiled with BTF for `struct data`.
```c
#define BPF_NO_GLOBAL_DATA
// #define BPF_NO_PRESERVE_ACCESS_INDEX
#include <vmlinux.h>
#include <bpf/bpf_helpers.h>
#include <bpf/bpf_tracing.h>
#ifndef BPF_NO_PRESERVE_ACCESS_INDEX
#pragma clang attribute push (__attribute__((preserve_access_index)), apply_to = record)
#endif
struct data {
int a;
int c;
int d;
};
#ifndef BPF_NO_PRESERVE_ACCESS_INDEX
#pragma clang attribute pop
#endif
SEC("uprobe/examples/btf-base:add_test")
int BPF_UPROBE(add_test, struct data *d)
{
int a = 0, c = 0;
bpf_probe_read_user(&a, sizeof(a), &d->a);
bpf_probe_read_user(&c, sizeof(c), &d->c);
bpf_printk("add_test(&d) %d + %d = %d\n", a, c, a + c);
return a + c;
}
char LICENSE[] SEC("license") = "Dual BSD/GPL";
```
The record of `struct data` is preserved in the eBPF program. Then, we can use the `btf-base.btf` to compile the eBPF program.
Merge user btf with kernel btf, so we have a complete btf for the kernel and userspace:
```sh
./merge-btf /sys/kernel/btf/vmlinux examples/base.btf target-base.btf
```
Then we execute the eBPF program with the userspace program. for `btf-base`:
```console
$ sudo ./uprobe examples/btf-base target-base.btf
...
libbpf: prog 'add_test': relo #1: patched insn #4 (ALU/ALU64) imm 0 -> 0
libbpf: prog 'add_test': relo #2: <byte_off> [7] struct data.c (0:1 @ offset 4)
libbpf: prog 'add_test': relo #2: matching candidate #0 <byte_off> [133110] struct data.c (0:1 @ offset 4)
libbpf: prog 'add_test': relo #2: patched insn #11 (ALU/ALU64) imm 4 -> 4
...
```
Execute the userspace program and get result:
```console
$ sudo cat /sys/kernel/debug/tracing/trace_pipe
[sudo] password for yunwei37:
<...>-26740 [001] ...11 28180.156220: bpf_trace_printk: add_test(&d) 1 + 3 = 4
```
Also, we do the same for another version of the userspace program `btf-base-new`:
```console
$ ./merge-btf /sys/kernel/btf/vmlinux examples/base-new.btf target-base-new.btf
$ sudo ./uprobe examples/btf-base-new target-base-new.btf
....
libbpf: sec 'uprobe/examples/btf-base:add_test': found 3 CO-RE relocations
libbpf: CO-RE relocating [2] struct pt_regs: found target candidate [357] struct pt_regs in [vmlinux]
libbpf: prog 'add_test': relo #0: <byte_off> [2] struct pt_regs.di (0:14 @ offset 112)
libbpf: prog 'add_test': relo #0: matching candidate #0 <byte_off> [357] struct pt_regs.di (0:14 @ offset 112)
libbpf: prog 'add_test': relo #0: patched insn #0 (LDX/ST/STX) off 112 -> 112
libbpf: CO-RE relocating [7] struct data: found target candidate [133110] struct data in [vmlinux]
libbpf: prog 'add_test': relo #1: <byte_off> [7] struct data.a (0:0 @ offset 0)
libbpf: prog 'add_test': relo #1: matching candidate #0 <byte_off> [133110] struct data.a (0:0 @ offset 0)
libbpf: prog 'add_test': relo #1: patched insn #4 (ALU/ALU64) imm 0 -> 0
libbpf: prog 'add_test': relo #2: <byte_off> [7] struct data.c (0:1 @ offset 4)
libbpf: prog 'add_test': relo #2: matching candidate #0 <byte_off> [133110] struct data.c (0:2 @ offset 8)
libbpf: prog 'add_test': relo #2: patched insn #11 (ALU/ALU64) imm 4 -> 8
libbpf: elf: symbol address match for 'add_test' in 'examples/btf-base-new': 0x1140
Successfully started! Press Ctrl+C to stop.
```
The result is correct:
```console
$ sudo cat /sys/kernel/debug/tracing/trace_pipe
[sudo] password for yunwei37:
<...>-26740 [001] ...11 28180.156220: bpf_trace_printk: add_test(&d) 1 + 3 = 4
```
## Conclusion
- **Flexibility and Compatibility**: The use of BTF in user-space eBPF programs greatly enhances their flexibility and compatibility across different versions of user-space applications and libraries.
- **Reduced Complexity**: This approach significantly reduces the complexity involved in maintaining eBPF programs for different versions of user-space applications, as it eliminates the need for multiple program versions.
- **Potential for Broader Application**: While your example focused on SSL/TLS monitoring, this methodology has broader applications in performance monitoring, security, and debugging of user-space applications.
This example showcases a significant advancement in the practical application of eBPF, extending its powerful features to more dynamically handle user-space applications in a Linux environment. It's a compelling solution for software engineers and system administrators dealing with the complexities of modern Linux systems.
If you want to learn more about eBPF knowledge and practices, you can visit our tutorial code repository <https://github.com/eunomia-bpf/bpf-developer-tutorial> or website <https://eunomia.dev/tutorials/> to get more examples and complete tutorials.

4
src/38-btf-uprobe/examples/.gitignore vendored Normal file
View File

@@ -0,0 +1,4 @@
*.o
*.btf
btf-base
btf-base-new

35
src/38-btf-uprobe/examples/Makefile Normal file
View File

@@ -0,0 +1,35 @@
# BPF compiler
BPF_CC = clang
# BPF C flags
BPF_CFLAGS = -O2 -target bpf -c -g
# BPF source files
BPF_SRCS = $(wildcard *.bpf.c)
# BPF object files
BPF_OBJS = $(BPF_SRCS:.c=.o)
all: $(BPF_OBJS) base.btf btf-base btf-base-new base-new.btf
%.bpf.o: %.bpf.c
$(BPF_CC) $(BPF_CFLAGS) $< -o $@
btf-base.o: btf-base.c
clang -g -c btf-base.c -o btf-base.o
btf-base-new.o: btf-base-new.c
clang -g -c btf-base-new.c -o btf-base-new.o
base.btf: btf-base.o
pahole --btf_encode_detached base.btf btf-base.o
base-new.btf: btf-base-new.o
pahole --btf_encode_detached base-new.btf btf-base-new.o
btf-base: btf-base.o
clang -g btf-base.o -o btf-base
btf-base-new: btf-base-new.o
clang -g btf-base-new.o -o btf-base-new
clean:
rm -f *.o *.btf btf-base btf-base-new

19
src/38-btf-uprobe/examples/btf-base-new.c Normal file
View File

@@ -0,0 +1,19 @@
#include <stdio.h>
struct data {
int a;
int b;
int c;
int d;
};
int add_test(struct data *d) {
return d->a + d->c;
}
int main(int argc, char **argv) {
struct data d = {1, 2, 3, 4};
printf("add_test(&d) = %d\n", add_test(&d));
return 0;
}

19
src/38-btf-uprobe/examples/btf-base.c Normal file
View File

@@ -0,0 +1,19 @@
#include <stdio.h>
// use a different struct
struct data {
int a;
int c;
int d;
};
int add_test(struct data *d) {
return d->a + d->c;
}
int main(int argc, char **argv) {
struct data d = {1, 3, 4};
printf("add_test(&d) = %d\n", add_test(&d));
return 0;
}

19
src/38-btf-uprobe/examples/btf-relo.bpf.c Normal file
View File

@@ -0,0 +1,19 @@
#ifndef BPF_NO_PRESERVE_ACCESS_INDEX
#pragma clang attribute push (__attribute__((preserve_access_index)), apply_to = record)
#endif
struct data {
int a;
int c;
int d;
};
#ifndef BPF_NO_PRESERVE_ACCESS_INDEX
#pragma clang attribute pop
#endif
int add_test(struct data *d) {
return d->a + d->c;
}

63
src/38-btf-uprobe/merge-btf.c Normal file
View File

@@ -0,0 +1,63 @@
#include <bpf/libbpf.h>
#include <bpf/bpf.h>
#include <bpf/btf.h>
#include <errno.h>
#include <stdio.h>
int main(int argc, char **argv)
{
char *btf_base_path = argv[1];
char *btf_src_path = argv[2];
char *btf_dst_path = argv[3];
struct btf *btf_src, *btf_base;
int err;
unsigned int size;
const void* btf_data;
FILE *fp;
if (argc != 4)
{
fprintf(stderr, "Usage: %s <btf_base> <btf_src> <btf_dst>\n", argv[0]);
fprintf(stderr, "Used for merge btf info");
return 1;
}
btf_base = btf__parse(btf_base_path, NULL);
if (!btf_base)
{
fprintf(stderr, "Failed to parse BTF object '%s': %s\n", btf_base_path, strerror(errno));
return 1;
}
btf_src = btf__parse(btf_src_path, NULL);
if (!btf_src)
{
fprintf(stderr, "Failed to parse BTF object '%s': %s\n", btf_src_path, strerror(errno));
return 1;
}
err = btf__add_btf(btf_base, btf_src);
if (err < 0)
{
fprintf(stderr, "Failed to add BTF object '%s': %s\n", btf_src_path, strerror(errno));
return 1;
}
btf_data = btf__raw_data(btf_base, &size);
if (!btf_data)
{
fprintf(stderr, "Failed to get raw data of BTF object '%s': %s\n", btf_base_path, strerror(errno));
return 1;
}
fp = fopen(btf_dst_path, "w");
if (!fp)
{
fprintf(stderr, "Failed to open BTF object '%s': %s\n", btf_dst_path, strerror(errno));
return 1;
}
fwrite(btf_data, size, 1, fp);
fclose(fp);
return 0;
}

32
src/38-btf-uprobe/uprobe.bpf.c Normal file
View File

@@ -0,0 +1,32 @@
#define BPF_NO_GLOBAL_DATA
// #define BPF_NO_PRESERVE_ACCESS_INDEX
#include <vmlinux.h>
#include <bpf/bpf_helpers.h>
#include <bpf/bpf_tracing.h>
#ifndef BPF_NO_PRESERVE_ACCESS_INDEX
#pragma clang attribute push (__attribute__((preserve_access_index)), apply_to = record)
#endif
struct data {
int a;
int c;
int d;
};
#ifndef BPF_NO_PRESERVE_ACCESS_INDEX
#pragma clang attribute pop
#endif
SEC("uprobe/examples/btf-base:add_test")
int BPF_UPROBE(add_test, struct data *d)
{
int a = 0, c = 0;
bpf_probe_read_user(&a, sizeof(a), &d->a);
bpf_probe_read_user(&c, sizeof(c), &d->c);
bpf_printk("add_test(&d) %d + %d = %d\n", a, c, a + c);
return a + c;
}
char LICENSE[] SEC("license") = "Dual BSD/GPL";

83
src/38-btf-uprobe/uprobe.c Normal file
View File

@@ -0,0 +1,83 @@
// SPDX-License-Identifier: (LGPL-2.1 OR BSD-2-Clause)
/* Copyright (c) 2020 Facebook */
#include <signal.h>
#include <stdio.h>
#include <time.h>
#include <stdint.h>
#include <sys/resource.h>
#include <bpf/libbpf.h>
#include <bpf/bpf.h>
#include <unistd.h>
#include <stdlib.h>
#include "uprobe.skel.h"
#define warn(...) fprintf(stderr, __VA_ARGS__)
static int libbpf_print_fn(enum libbpf_print_level level, const char *format,
va_list args)
{
return vfprintf(stderr, format, args);
}
static volatile bool exiting = false;
static void sig_handler(int sig)
{
exiting = true;
}
int main(int argc, char **argv)
{
struct uprobe_bpf *skel;
int err;
LIBBPF_OPTS(bpf_object_open_opts , opts,
);
LIBBPF_OPTS(bpf_uprobe_opts, uprobe_opts);
if (argc != 3 && argc != 2) {
fprintf(stderr, "Usage: %s <example-name> [<external-btf>]\n", argv[0]);
return 1;
}
if (argc == 3)
opts.btf_custom_path = argv[2];
/* Set up libbpf errors and debug info callback */
libbpf_set_print(libbpf_print_fn);
/* Cleaner handling of Ctrl-C */
signal(SIGINT, sig_handler);
signal(SIGTERM, sig_handler);
/* Load and verify BPF application */
skel = uprobe_bpf__open_opts(&opts);
if (!skel) {
fprintf(stderr, "Failed to open and load BPF skeleton\n");
return 1;
}
/* Load & verify BPF programs */
err = uprobe_bpf__load(skel);
if (err) {
fprintf(stderr, "Failed to load and verify BPF skeleton\n");
goto cleanup;
}
uprobe_opts.func_name = "add_test";
skel->links.add_test = bpf_program__attach_uprobe_opts(
skel->progs.add_test, -1 /* self pid */, argv[1] /* binary path */,
0 /* offset for function */, &uprobe_opts /* opts */);
if (!skel->links.add_test) {
err = -errno;
fprintf(stderr, "Failed to attach uprobe: %d\n", err);
goto cleanup;
}
printf("Successfully started! Press Ctrl+C to stop.\n");
fflush(stdout);
while (!exiting) {
sleep(1);
}
cleanup:
/* Clean up */
uprobe_bpf__destroy(skel);
return err < 0 ? -err : 0;
}