mirror of
https://github.com/eunomia-bpf/bpf-developer-tutorial.git
synced 2026-02-02 17:59:47 +08:00
add new topics (#132)
* add goroutine text * update * fix funclatency * update doc * update about nginx * update nginx * u[date co * fix compile
This commit is contained in:
云微
4
.github/workflows/test-eunomia.yaml
vendored
4
.github/workflows/test-eunomia.yaml
vendored
@@ -79,3 +79,7 @@ jobs:
|
||||
run: |
|
||||
./ecc src/25-signal/signal.bpf.c src/25-signal/signal.h
|
||||
sudo timeout -s 2 3 ./ecli run src/25-signal/package.json || if [ $? = 124 ]; then exit 0; else exit $?; fi
|
||||
- name: test 31 goroutine
|
||||
run: |
|
||||
./ecc src/31-goroutine/goroutine.bpf.c src/31-goroutine/goroutine.h
|
||||
# todo
|
||||
|
||||
4
.github/workflows/test-libbpf.yml
vendored
4
.github/workflows/test-libbpf.yml
vendored
@@ -61,6 +61,10 @@ jobs:
|
||||
run: |
|
||||
make -C src/30-sslsniff
|
||||
sudo timeout -s 2 3 src/30-sslsniff/sslsniff || if [ $? = 124 ]; then exit 0; else exit $?; fi
|
||||
|
||||
- name: test 33 funclatency
|
||||
run: |
|
||||
make -C src/33-funclatency
|
||||
- name: test 35-user-ringbuf
|
||||
run: |
|
||||
make -C src/35-user-ringbuf
|
||||
|
||||
@@ -1,33 +1,112 @@
|
||||
# goroutine trace
|
||||
# eBPF 实践教程：使用 eBPF 跟踪 Go 协程状态
|
||||
|
||||
**UNFINISHED YET**: The offset of goid field is hardcoded. It was only tested on the bundled `go-server-http`. It MAY NOT WORK on other go programs.
|
||||
Go 是 Google 创建的一种广受欢迎的编程语言，以其强大的并发模型而著称。Go 语言的一个重要特点是协程（goroutine）的使用——这些协程是轻量级、由 Go 运行时管理的线程，使得编写并发程序变得非常简单。然而，在实时环境中理解和跟踪这些协程的执行状态，尤其是在调试复杂系统时，可能会面临很大的挑战。
|
||||
|
||||
The bundled of program was compiled using go 1.17.0. The executable and source could be found at folder `go-server-http`.
|
||||
这时我们可以利用 eBPF（扩展伯克利包过滤器）技术。eBPF 最初设计用于网络数据包过滤，但随着时间的推移，eBPF 已经发展成为一个强大的工具，用于跟踪和监控系统行为。通过使用 eBPF，我们可以深入到内核，收集有关 Go 程序运行时行为的数据，包括协程的状态。本文将探讨如何使用 eBPF 跟踪 Go 程序中的协程状态转换。
|
||||
|
||||
This example traces the state switch of goroutines, and prints the corresponding state, goid, pid and tgid.
|
||||
## 背景：协程与 eBPF
|
||||
|
||||
```console
|
||||
root@mnfe-pve:~/bpf-developer-tutorial/src/31-goroutine# ecc goroutine.bpf.c goroutine.h
|
||||
INFO [ecc_rs::bpf_compiler] Compiling bpf object...
|
||||
INFO [ecc_rs::bpf_compiler] Generating export types...
|
||||
INFO [ecc_rs::bpf_compiler] Generating package json..
|
||||
INFO [ecc_rs::bpf_compiler] Packing ebpf object and config into package.json...
|
||||
root@mnfe-pve:~/bpf-developer-tutorial/src/31-goroutine# ecli-rs run package.json
|
||||
INFO [faerie::elf] strtab: 0x6fb symtab 0x738 relocs 0x780 sh_offset 0x780
|
||||
INFO [bpf_loader_lib::skeleton::preload::section_loader] User didn't specify custom value for variable __eunomia_dummy_goroutine_execute_data_ptr, use the default one in ELF
|
||||
TIME STATE GOID PID TGID
|
||||
INFO [bpf_loader_lib::skeleton] Running ebpf program...
|
||||
21:00:47 DEAD(6) 0 2542844 2542844
|
||||
21:00:47 RUNNABLE(1) 0 2542844 2542844
|
||||
21:00:47 DEAD(6) 0 2542844 2542844
|
||||
21:00:47 RUNNING(2) 1 2542844 2542844
|
||||
21:00:47 DEAD(6) 0 2542844 2542844
|
||||
21:00:47 RUNNABLE(1) 0 2542844 2542844
|
||||
21:00:47 RUNNABLE(1) 1 2542844 2542844
|
||||
21:00:47 RUNNING(2) 2 2542847 2542844
|
||||
21:00:47 WAITING(4) 2 2542847 2542844
|
||||
....
|
||||
### 协程
|
||||
|
||||
协程是 Go 语言的核心特性之一，它提供了一种简单而高效的并发处理方式。与传统的线程不同，协程由 Go 运行时管理，而不是由操作系统管理，因此更加轻量化。协程可以在以下几种状态之间进行转换：
|
||||
|
||||
- **RUNNABLE（可运行）**：协程已准备好运行。
|
||||
- **RUNNING（运行中）**：协程正在执行中。
|
||||
- **WAITING（等待）**：协程正在等待某个事件（如 I/O 或定时器）。
|
||||
- **DEAD（终止）**：协程执行完毕并已终止。
|
||||
|
||||
理解这些状态以及协程之间的状态转换对于诊断性能问题、确保 Go 程序的高效运行至关重要。
|
||||
|
||||
### eBPF
|
||||
|
||||
eBPF 是一种强大的技术，它允许开发人员在不修改内核源代码或加载内核模块的情况下，在 Linux 内核中运行自定义程序。eBPF 最初用于数据包过滤，但现在已扩展为一种多功能工具，广泛应用于性能监控、安全和调试。
|
||||
|
||||
通过编写 eBPF 程序，开发人员可以跟踪各种系统事件，包括系统调用、网络事件和进程执行。在本文中，我们将重点介绍如何使用 eBPF 跟踪 Go 程序中协程的状态转换。
|
||||
|
||||
## eBPF 内核代码
|
||||
|
||||
现在，让我们深入探讨实现该跟踪功能的 eBPF 内核代码。
|
||||
|
||||
```c
|
||||
#include <vmlinux.h>
|
||||
#include "goroutine.h"
|
||||
#include <bpf/bpf_core_read.h>
|
||||
#include <bpf/bpf_helpers.h>
|
||||
#include <bpf/bpf_tracing.h>
|
||||
|
||||
#define GOID_OFFSET 0x98
|
||||
|
||||
struct {
|
||||
__uint(type, BPF_MAP_TYPE_RINGBUF);
|
||||
__uint(max_entries, 256 * 1024);
|
||||
} rb SEC(".maps");
|
||||
|
||||
SEC("uprobe/./go-server-http/main:runtime.casgstatus")
|
||||
int uprobe_runtime_casgstatus(struct pt_regs *ctx) {
|
||||
int newval = ctx->cx;
|
||||
void *gp = ctx->ax;
|
||||
struct goroutine_execute_data *data;
|
||||
u64 goid;
|
||||
if (bpf_probe_read_user(&goid, sizeof(goid), gp + GOID_OFFSET) == 0) {
|
||||
data = bpf_ringbuf_reserve(&rb, sizeof(*data), 0);
|
||||
if (data) {
|
||||
u64 pid_tgid = bpf_get_current_pid_tgid();
|
||||
data->pid = pid_tgid;
|
||||
data->tgid = pid_tgid >> 32;
|
||||
data->goid = goid;
|
||||
data->state = newval;
|
||||
bpf_ringbuf_submit(data, 0);
|
||||
}
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
|
||||
char LICENSE[] SEC("license") = "GPL";
|
||||
```
|
||||
|
||||
1. **头文件**：代码首先包含了必要的头文件，如 `vmlinux.h`（提供内核定义）和 `bpf_helpers.h`（提供 eBPF 程序的辅助函数）。
|
||||
2. **GOID_OFFSET**：`goid` 字段的偏移量被硬编码为 `0x98`，这是特定于所跟踪的 Go 版本和程序的。此偏移量在不同的 Go 版本或程序中可能有所不同。
|
||||
3. **环形缓冲区映射**：定义了一个 BPF 环形缓冲区映射，用于存储协程的执行数据。这个缓冲区允许内核高效地将信息传递到用户空间。
|
||||
4. **Uprobe**：该 eBPF 程序的核心是一个附加到 Go 程序中 `runtime.casgstatus` 函数的 uprobe（用户级探针）。该函数负责改变协程的状态，因此非常适合用来拦截和跟踪状态转换。
|
||||
5. **读取协程 ID**：`bpf_probe_read_user` 函数从用户空间内存中读取协程 ID（`goid`），使用的是预定义的偏移量。
|
||||
6. **提交数据**：如果成功读取了协程 ID，则数据会与进程 ID、线程组 ID 以及协程的新状态一起存储在环形缓冲区中。随后，这些数据会提交到用户空间以供分析。
|
||||
|
||||
This example is provided as GPL license
|
||||
## 运行程序
|
||||
|
||||
要运行此跟踪程序，请按照以下步骤操作：
|
||||
|
||||
1. **编译 eBPF 代码**：使用类似 `ecc`（eBPF 编译集合）这样的编译器编译 eBPF 程序，并生成一个可以由 eBPF 加载器加载的包。
|
||||
|
||||
```bash
|
||||
ecc goroutine.bpf.c goroutine.h
|
||||
```
|
||||
|
||||
2. **运行 eBPF 程序**：使用 eBPF 加载器运行编译后的 eBPF 程序。
|
||||
|
||||
```bash
|
||||
ecli-rs run package.json
|
||||
```
|
||||
|
||||
3. **输出**：程序将输出协程的状态转换及其 `goid`、`pid` 和 `tgid`。以下是一个示例输出：
|
||||
|
||||
```console
|
||||
TIME STATE GOID PID TGID
|
||||
21:00:47 DEAD(6) 0 2542844 2542844
|
||||
21:00:47 RUNNABLE(1) 0 2542844 2542844
|
||||
21:00:47 RUNNING(2) 1 2542844 2542844
|
||||
21:00:47 WAITING(4) 2 2542847 2542844
|
||||
```
|
||||
|
||||
完整代码可以在 <https://github.com/eunomia-bpf/bpf-developer-tutorial/tree/main/src/31-goroutine> 找到。
|
||||
|
||||
如果你想了解更多关于 eBPF 的知识和实践，你可以访问我们的教程代码库 <https://github.com/eunomia-bpf/bpf-developer-tutorial> 或网站 <https://eunomia.dev/tutorials/> 获取更多示例和完整教程。
|
||||
|
||||
内核模式 eBPF 运行时的 `Uprobe` 可能会带来较大的性能开销。在这种情况下，你也可以考虑使用用户模式的 eBPF 运行时，例如 [bpftime](https://github.com/eunomia-bpf/bpftime)。bpftime 是基于 LLVM JIT/AOT 的用户模式 eBPF 运行时，它可以在用户模式下运行 eBPF 程序，并且在处理 `uprobe` 时比内核模式 eBPF 更快。
|
||||
|
||||
### 结论
|
||||
|
||||
使用 eBPF 跟踪协程状态可以深入了解 Go 程序的执行情况，尤其是在传统调试工具可能无法胜任的生产环境中。通过利用 eBPF，开发人员可以监控和诊断性能问题，确保 Go 应用程序高效运行。
|
||||
|
||||
请注意，本 eBPF 程序中使用的偏移量是特定于所跟踪的 Go 版本和程序的。随着 Go 的发展，这些偏移量可能会发生变化，需要对 eBPF 代码进行更新。
|
||||
|
||||
在未来的探索中，我们可以将这种方法扩展到跟踪 Go 程序或其他语言的其他方面，展示 eBPF 在现代软件开发中的多功能性和强大作用。
|
||||
|
||||
112
src/31-goroutine/README_en.md
Normal file
112
src/31-goroutine/README_en.md
Normal file
@@ -0,0 +1,112 @@
|
||||
# eBPF Practical Tutorial: Using eBPF to Trace Go Routine States
|
||||
|
||||
Go, the popular programming language created by Google, is known for its powerful concurrency model. One of the key features that makes Go stand out is the use of goroutines—lightweight, managed threads that make it easy to write concurrent programs. However, understanding and tracing the execution states of these goroutines in real time can be challenging, especially when debugging complex systems.
|
||||
|
||||
Enter eBPF (Extended Berkeley Packet Filter), a technology originally designed for network packet filtering, but which has since evolved into a powerful tool for tracing and monitoring system behavior. By leveraging eBPF, we can tap into the kernel and gather insights about the runtime behavior of Go programs, including the states of goroutines. This blog post explores how to use eBPF to trace the state transitions of goroutines in a Go program.
|
||||
|
||||
## Background: Goroutines and eBPF
|
||||
|
||||
### Goroutines
|
||||
|
||||
Goroutines are a core feature of Go, providing a simple and efficient way to handle concurrency. Unlike traditional threads, goroutines are managed by the Go runtime rather than the operating system, making them much more lightweight. Goroutines can switch states, such as:
|
||||
|
||||
- **RUNNABLE**: The goroutine is ready to run.
|
||||
- **RUNNING**: The goroutine is currently executing.
|
||||
- **WAITING**: The goroutine is waiting for some event (e.g., I/O, timers).
|
||||
- **DEAD**: The goroutine has finished executing and is terminated.
|
||||
|
||||
Understanding these states and how goroutines transition between them is crucial for diagnosing performance issues and ensuring that your Go programs are running efficiently.
|
||||
|
||||
### eBPF
|
||||
|
||||
eBPF is a powerful technology that allows developers to run custom programs inside the Linux kernel without changing the kernel source code or loading kernel modules. Initially designed for packet filtering, eBPF has grown into a versatile tool used for performance monitoring, security, and debugging.
|
||||
|
||||
By writing eBPF programs, developers can trace various system events, including system calls, network events, and process execution. In this blog, we'll focus on how eBPF can be used to trace the state transitions of goroutines in a Go program.
|
||||
|
||||
## The eBPF Kernel Code
|
||||
|
||||
Let's dive into the eBPF kernel code that makes this tracing possible.
|
||||
|
||||
```c
|
||||
#include <vmlinux.h>
|
||||
#include "goroutine.h"
|
||||
#include <bpf/bpf_core_read.h>
|
||||
#include <bpf/bpf_helpers.h>
|
||||
#include <bpf/bpf_tracing.h>
|
||||
|
||||
#define GOID_OFFSET 0x98
|
||||
|
||||
struct {
|
||||
__uint(type, BPF_MAP_TYPE_RINGBUF);
|
||||
__uint(max_entries, 256 * 1024);
|
||||
} rb SEC(".maps");
|
||||
|
||||
SEC("uprobe/./go-server-http/main:runtime.casgstatus")
|
||||
int uprobe_runtime_casgstatus(struct pt_regs *ctx) {
|
||||
int newval = ctx->cx;
|
||||
void *gp = ctx->ax;
|
||||
struct goroutine_execute_data *data;
|
||||
u64 goid;
|
||||
if (bpf_probe_read_user(&goid, sizeof(goid), gp + GOID_OFFSET) == 0) {
|
||||
data = bpf_ringbuf_reserve(&rb, sizeof(*data), 0);
|
||||
if (data) {
|
||||
u64 pid_tgid = bpf_get_current_pid_tgid();
|
||||
data->pid = pid_tgid;
|
||||
data->tgid = pid_tgid >> 32;
|
||||
data->goid = goid;
|
||||
data->state = newval;
|
||||
bpf_ringbuf_submit(data, 0);
|
||||
}
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
|
||||
char LICENSE[] SEC("license") = "GPL";
|
||||
```
|
||||
|
||||
1. **Header Files**: The code begins by including necessary header files, such as `vmlinux.h`, which provides kernel definitions, and `bpf_helpers.h`, which offers helper functions for eBPF programs.
|
||||
2. **GOID_OFFSET**: The offset of the `goid` field is hardcoded to `0x98`, which is specific to the Go version and the program being traced. This offset may vary between different Go versions or programs.
|
||||
3. **Ring Buffer Map**: A BPF ring buffer map is defined to store the goroutine execution data. This buffer allows the kernel to pass information to user space efficiently.
|
||||
4. **Uprobe**: The core of this eBPF program is an uprobes (user-level probe) attached to the `runtime.casgstatus` function in the Go program. This function is responsible for changing the state of a goroutine, making it an ideal place to intercept and trace state transitions.
|
||||
5. **Reading Goroutine ID**: The `bpf_probe_read_user` function reads the goroutine ID (`goid`) from the user space memory, using the predefined offset.
|
||||
6. **Submitting Data**: If the goroutine ID is successfully read, the data is stored in the ring buffer along with the process ID, thread group ID, and the new state of the goroutine. This data is then submitted to the user space for analysis.
|
||||
|
||||
## Running the Program
|
||||
|
||||
To run this tracing program, follow these steps:
|
||||
|
||||
1. **Compile the eBPF Code**: Compile the eBPF program using a compiler like `ecc` (eBPF Compiler Collection) and generate a package that can be loaded by an eBPF loader.
|
||||
|
||||
```bash
|
||||
ecc goroutine.bpf.c goroutine.h
|
||||
```
|
||||
|
||||
2. **Run the eBPF Program**: Use an eBPF loader to run the compiled eBPF program.
|
||||
|
||||
```bash
|
||||
ecli-rs run package.json
|
||||
```
|
||||
|
||||
3. **Output**: The program will output the state transitions of goroutines along with their `goid`, `pid`, and `tgid`. Here’s an example of the output:
|
||||
|
||||
```console
|
||||
TIME STATE GOID PID TGID
|
||||
21:00:47 DEAD(6) 0 2542844 2542844
|
||||
21:00:47 RUNNABLE(1) 0 2542844 2542844
|
||||
21:00:47 RUNNING(2) 1 2542844 2542844
|
||||
21:00:47 WAITING(4) 2 2542847 2542844
|
||||
```
|
||||
|
||||
You can find the complete code in <https://github.com/eunomia-bpf/bpf-developer-tutorial/tree/main/src/31-goroutine>
|
||||
|
||||
If you want to learn more about eBPF knowledge and practices, you can visit our tutorial code repository <https://github.com/eunomia-bpf/bpf-developer-tutorial> or website <https://eunomia.dev/tutorials/> to get more examples and complete tutorials.
|
||||
|
||||
`Uprobe` in kernel mode eBPF runtime may also cause relatively large performance overhead. In this case, you can also consider using user mode eBPF runtime, such as [bpftime](https://github.com/eunomia-bpf/bpftime). bpftime is a user mode eBPF runtime based on LLVM JIT/AOT. It can run eBPF programs in user mode, compatible with kernel mode eBPF and can be faster for `uprobe`.
|
||||
|
||||
### Conclusion
|
||||
|
||||
Tracing goroutine states using eBPF provides deep insights into the execution of Go programs, especially in production environments where traditional debugging tools may fall short. By leveraging eBPF, developers can monitor and diagnose performance issues, ensuring their Go applications run efficiently.
|
||||
|
||||
Keep in mind that the offsets used in this eBPF program are specific to the Go version and the program being traced. As Go evolves, these offsets may change, requiring updates to the eBPF code.
|
||||
|
||||
In future explorations, we can extend this approach to trace other aspects of Go programs or even other languages, demonstrating the versatility and power of eBPF in modern software development.
|
||||
@@ -28,8 +28,6 @@
|
||||
#include <bpf/bpf_helpers.h>
|
||||
#include <bpf/bpf_tracing.h>
|
||||
|
||||
|
||||
|
||||
#define GOID_OFFSET 0x98
|
||||
|
||||
struct {
|
||||
@@ -40,7 +38,7 @@ struct {
|
||||
SEC("uprobe/./go-server-http/main:runtime.casgstatus")
|
||||
int uprobe_runtime_casgstatus(struct pt_regs *ctx) {
|
||||
int newval = ctx->cx;
|
||||
void *gp = ctx->ax;
|
||||
void *gp = (void*)ctx->ax;
|
||||
struct goroutine_execute_data *data;
|
||||
u64 goid;
|
||||
if (bpf_probe_read_user(&goid, sizeof(goid), gp + GOID_OFFSET) == 0) {
|
||||
|
||||
@@ -1,172 +0,0 @@
|
||||
// SPDX-License-Identifier: GPL-2.0
|
||||
#include <argp.h>
|
||||
#include <signal.h>
|
||||
#include <stdio.h>
|
||||
#include <time.h>
|
||||
#include <sys/resource.h>
|
||||
#include <bpf/libbpf.h>
|
||||
#include "goroutine.h"
|
||||
#include "goroutine.skel.h"
|
||||
|
||||
static struct env {
|
||||
bool verbose;
|
||||
long min_duration_ms;
|
||||
} env;
|
||||
|
||||
const char *argp_program_version = "goroutine 0.0";
|
||||
const char *argp_program_bug_address = "<bpf@vger.kernel.org>";
|
||||
const char argp_program_doc[] =
|
||||
"BPF goroutine demo application.\n"
|
||||
"\n"
|
||||
"It traces process start and exits and shows associated \n"
|
||||
"information (filename, process duration, PID and PPID, etc).\n"
|
||||
"\n"
|
||||
"USAGE: ./goroutine [-d <min-duration-ms>] [-v]\n";
|
||||
|
||||
static const struct argp_option opts[] = {
|
||||
{ "verbose", 'v', NULL, 0, "Verbose debug output" },
|
||||
{ "duration", 'd', "DURATION-MS", 0, "Minimum process duration (ms) to report" },
|
||||
{},
|
||||
};
|
||||
|
||||
static error_t parse_arg(int key, char *arg, struct argp_state *state)
|
||||
{
|
||||
switch (key) {
|
||||
case 'v':
|
||||
env.verbose = true;
|
||||
break;
|
||||
case 'd':
|
||||
errno = 0;
|
||||
env.min_duration_ms = strtol(arg, NULL, 10);
|
||||
if (errno || env.min_duration_ms <= 0) {
|
||||
fprintf(stderr, "Invalid duration: %s\n", arg);
|
||||
argp_usage(state);
|
||||
}
|
||||
break;
|
||||
case ARGP_KEY_ARG:
|
||||
argp_usage(state);
|
||||
break;
|
||||
default:
|
||||
return ARGP_ERR_UNKNOWN;
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
|
||||
static const struct argp argp = {
|
||||
.options = opts,
|
||||
.parser = parse_arg,
|
||||
.doc = argp_program_doc,
|
||||
};
|
||||
|
||||
static int libbpf_print_fn(enum libbpf_print_level level, const char *format, va_list args)
|
||||
{
|
||||
if (level == LIBBPF_DEBUG && !env.verbose)
|
||||
return 0;
|
||||
return vfprintf(stderr, format, args);
|
||||
}
|
||||
|
||||
static volatile bool exiting = false;
|
||||
|
||||
static void sig_handler(int sig)
|
||||
{
|
||||
exiting = true;
|
||||
}
|
||||
|
||||
static int handle_event(void *ctx, void *data, size_t data_sz)
|
||||
{
|
||||
const struct event *e = data;
|
||||
struct tm *tm;
|
||||
char ts[32];
|
||||
time_t t;
|
||||
|
||||
time(&t);
|
||||
tm = localtime(&t);
|
||||
strftime(ts, sizeof(ts), "%H:%M:%S", tm);
|
||||
|
||||
if (e->exit_event) {
|
||||
printf("%-8s %-5s %-16s %-7d %-7d [%u]",
|
||||
ts, "EXIT", e->comm, e->pid, e->ppid, e->exit_code);
|
||||
if (e->duration_ns)
|
||||
printf(" (%llums)", e->duration_ns / 1000000);
|
||||
printf("\n");
|
||||
} else {
|
||||
printf("%-8s %-5s %-16s %-7d %-7d %s\n",
|
||||
ts, "EXEC", e->comm, e->pid, e->ppid, e->filename);
|
||||
}
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
int main(int argc, char **argv)
|
||||
{
|
||||
struct ring_buffer *rb = NULL;
|
||||
struct goroutine_bpf *skel;
|
||||
int err;
|
||||
|
||||
/* Parse command line arguments */
|
||||
err = argp_parse(&argp, argc, argv, 0, NULL, NULL);
|
||||
if (err)
|
||||
return err;
|
||||
|
||||
/* Set up libbpf errors and debug info callback */
|
||||
libbpf_set_print(libbpf_print_fn);
|
||||
|
||||
/* Cleaner handling of Ctrl-C */
|
||||
signal(SIGINT, sig_handler);
|
||||
signal(SIGTERM, sig_handler);
|
||||
|
||||
/* Load and verify BPF application */
|
||||
skel = goroutine_bpf__open();
|
||||
if (!skel) {
|
||||
fprintf(stderr, "Failed to open and load BPF skeleton\n");
|
||||
return 1;
|
||||
}
|
||||
|
||||
/* Parameterize BPF code with minimum duration parameter */
|
||||
skel->rodata->min_duration_ns = env.min_duration_ms * 1000000ULL;
|
||||
|
||||
/* Load & verify BPF programs */
|
||||
err = goroutine_bpf__load(skel);
|
||||
if (err) {
|
||||
fprintf(stderr, "Failed to load and verify BPF skeleton\n");
|
||||
goto cleanup;
|
||||
}
|
||||
|
||||
/* Attach tracepoints */
|
||||
err = goroutine_bpf__attach(skel);
|
||||
if (err) {
|
||||
fprintf(stderr, "Failed to attach BPF skeleton\n");
|
||||
goto cleanup;
|
||||
}
|
||||
|
||||
/* Set up ring buffer polling */
|
||||
rb = ring_buffer__new(bpf_map__fd(skel->maps.rb), handle_event, NULL, NULL);
|
||||
if (!rb) {
|
||||
err = -1;
|
||||
fprintf(stderr, "Failed to create ring buffer\n");
|
||||
goto cleanup;
|
||||
}
|
||||
|
||||
/* Process events */
|
||||
printf("%-8s %-5s %-16s %-7s %-7s %s\n",
|
||||
"TIME", "EVENT", "COMM", "PID", "PPID", "FILENAME/EXIT CODE");
|
||||
while (!exiting) {
|
||||
err = ring_buffer__poll(rb, 100 /* timeout, ms */);
|
||||
/* Ctrl-C will cause -EINTR */
|
||||
if (err == -EINTR) {
|
||||
err = 0;
|
||||
break;
|
||||
}
|
||||
if (err < 0) {
|
||||
printf("Error polling perf buffer: %d\n", err);
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
cleanup:
|
||||
/* Clean up */
|
||||
ring_buffer__free(rb);
|
||||
goroutine_bpf__destroy(skel);
|
||||
|
||||
return err < 0 ? -err : 0;
|
||||
}
|
||||
@@ -1,3 +1,5 @@
|
||||
# http2 from go server
|
||||
# func latency
|
||||
|
||||
TODO: make it work
|
||||
|
||||
from <https://github.com/iovisor/bcc/blob/master/libbpf-tools/funclatency.c>.
|
||||
|
||||
@@ -1,5 +0,0 @@
|
||||
# func latency
|
||||
|
||||
TODO: make it work
|
||||
|
||||
from <https://github.com/iovisor/bcc/blob/master/libbpf-tools/funclatency.c>.
|
||||
@@ -5,7 +5,7 @@ LIBBPF_SRC := $(abspath ../third_party/libbpf/src)
|
||||
BPFTOOL_SRC := $(abspath ../third_party/bpftool/src)
|
||||
LIBBPF_OBJ := $(abspath $(OUTPUT)/libbpf.a)
|
||||
BPFTOOL_OUTPUT ?= $(abspath $(OUTPUT)/bpftool)
|
||||
BPFTOOL ?= $(BPFTOOL_OUTPUT)/funclatency/bpftool
|
||||
BPFTOOL ?= $(BPFTOOL_OUTPUT)/bootstrap/bpftool
|
||||
LIBBLAZESYM_SRC := $(abspath ../third_party/blazesym/)
|
||||
LIBBLAZESYM_OBJ := $(abspath $(OUTPUT)/libblazesym.a)
|
||||
LIBBLAZESYM_HEADER := $(abspath $(OUTPUT)/blazesym.h)
|
||||
@@ -24,13 +24,13 @@ INCLUDES := -I$(OUTPUT) -I../third_party/libbpf/include/uapi -I$(dir $(VMLINUX))
|
||||
CFLAGS := -g -Wall
|
||||
ALL_LDFLAGS := $(LDFLAGS) $(EXTRA_LDFLAGS)
|
||||
|
||||
APPS = funclatency # minimal minimal_legacy uprobe kprobe fentry usdt sockfilter tc ksyscall
|
||||
APPS = funclatency
|
||||
|
||||
CARGO ?= $(shell which cargo)
|
||||
ifeq ($(strip $(CARGO)),)
|
||||
BZS_APPS :=
|
||||
else
|
||||
BZS_APPS := # profile
|
||||
BZS_APPS :=
|
||||
APPS += $(BZS_APPS)
|
||||
# Required by libblazesym
|
||||
ALL_LDFLAGS += -lrt -ldl -lpthread -lm
|
||||
@@ -91,7 +91,7 @@ $(LIBBPF_OBJ): $(wildcard $(LIBBPF_SRC)/*.[ch] $(LIBBPF_SRC)/Makefile) | $(OUTPU
|
||||
# Build bpftool
|
||||
$(BPFTOOL): | $(BPFTOOL_OUTPUT)
|
||||
$(call msg,BPFTOOL,$@)
|
||||
$(Q)$(MAKE) ARCH= CROSS_COMPILE= OUTPUT=$(BPFTOOL_OUTPUT)/ -C $(BPFTOOL_SRC) funclatency
|
||||
$(Q)$(MAKE) ARCH= CROSS_COMPILE= OUTPUT=$(BPFTOOL_OUTPUT)/ -C $(BPFTOOL_SRC) bootstrap
|
||||
|
||||
|
||||
$(LIBBLAZESYM_SRC)/target/release/libblazesym.a::
|
||||
189
src/33-funclatency/README.md
Normal file
189
src/33-funclatency/README.md
Normal file
@@ -0,0 +1,189 @@
|
||||
# 使用 eBPF 测量函数延迟
|
||||
|
||||
在现代软件系统中，了解函数的性能特性，尤其是那些对应用程序运行至关重要的函数的性能特性，是至关重要的。性能分析中的一个关键指标是**函数延迟**，即函数从开始到完成所花费的时间。通过分析函数延迟，开发人员可以识别瓶颈、优化性能，并确保系统在各种条件下高效运行。
|
||||
|
||||
本文将深入探讨如何使用 eBPF 这一强大的工具来测量函数延迟，并展示如何在内核和用户空间中进行跟踪和监控。
|
||||
|
||||
## 什么是 eBPF？
|
||||
|
||||
eBPF（扩展伯克利包过滤器）是一项革命性的技术，它允许开发人员编写小型程序在 Linux 内核中运行。eBPF 最初是为数据包过滤设计的，但它已经发展成为一个多功能工具，用于跟踪、监控和分析系统行为。通过 eBPF，您几乎可以对 Linux 内核或用户空间的任何部分进行插桩，从而收集性能数据、执行安全策略，甚至实时调试系统——这一切都无需修改内核源码或重启系统。
|
||||
|
||||
eBPF 程序在内核的沙盒环境中执行，确保了安全性和稳定性。这些程序可以附加到内核中的各种钩子上，如系统调用、网络事件和跟踪点，甚至可以通过 uprobes（用户级探针）附加到用户空间的函数。eBPF 程序收集的数据可以导出到用户空间进行分析，使其成为系统可观测性的重要工具。内核模式 eBPF 运行时的 `Uprobe` 可能会带来较大的性能开销。在这种情况下，你也可以考虑使用用户模式的 eBPF 运行时，例如 [bpftime](https://github.com/eunomia-bpf/bpftime)。
|
||||
|
||||
## 为什么函数延迟很重要？
|
||||
|
||||
函数延迟是内核和用户空间应用程序性能分析中的一个关键指标。它提供了关于特定函数执行时间的洞察，这对以下方面至关重要：
|
||||
|
||||
- **识别性能瓶颈**：高函数延迟可能表明代码中存在需要优化的低效或问题。
|
||||
- **确保系统响应能力**：在实时系统或对延迟敏感的应用程序中，理解和最小化函数延迟对于保持响应能力至关重要。
|
||||
- **性能分析和基准测试**：通过测量各种函数的延迟，开发人员可以对系统进行基准测试，并比较不同实现或配置的性能。
|
||||
- **调试和诊断**：当系统表现出意外行为或性能下降时，测量函数延迟可以帮助定位问题的根源。
|
||||
|
||||
内核空间（如系统调用、文件操作）和用户空间（如库函数）中的函数都可以进行延迟分析，从而提供系统性能的全面视图。
|
||||
|
||||
## 用于函数延迟的 eBPF 内核代码
|
||||
|
||||
以下是一个设计用于测量函数延迟的 eBPF 程序，它通过挂钩函数的入口和出口点来实现。该程序使用 kprobes 和 kretprobes（用于内核函数）或 uprobes 和 uretprobes（用于用户空间函数）来捕获函数执行的开始和结束时间。
|
||||
|
||||
```c
|
||||
// SPDX-License-Identifier: GPL-2.0
|
||||
/* Copyright (c) 2021 Google LLC. */
|
||||
#include "vmlinux.h"
|
||||
#include <bpf/bpf_core_read.h>
|
||||
#include <bpf/bpf_helpers.h>
|
||||
#include <bpf/bpf_tracing.h>
|
||||
#include "funclatency.h"
|
||||
#include "bits.bpf.h"
|
||||
|
||||
const volatile pid_t targ_tgid = 0;
|
||||
const volatile int units = 0;
|
||||
|
||||
/* key: pid. value: start time */
|
||||
struct {
|
||||
__uint(type, BPF_MAP_TYPE_HASH);
|
||||
__uint(max_entries, MAX_PIDS);
|
||||
__type(key, u32);
|
||||
__type(value, u64);
|
||||
} starts SEC(".maps");
|
||||
|
||||
__u32 hist[MAX_SLOTS] = {};
|
||||
|
||||
static void entry(void)
|
||||
{
|
||||
u64 id = bpf_get_current_pid_tgid();
|
||||
u32 tgid = id >> 32;
|
||||
u32 pid = id;
|
||||
u64 nsec;
|
||||
|
||||
if (targ_tgid && targ_tgid != tgid)
|
||||
return;
|
||||
nsec = bpf_ktime_get_ns();
|
||||
bpf_map_update_elem(&starts, &pid, &nsec, BPF_ANY);
|
||||
}
|
||||
|
||||
SEC("kprobe/dummy_kprobe")
|
||||
int BPF_KPROBE(dummy_kprobe)
|
||||
{
|
||||
entry();
|
||||
return 0;
|
||||
}
|
||||
|
||||
static void exit(void)
|
||||
{
|
||||
u64 *start;
|
||||
u64 nsec = bpf_ktime_get_ns();
|
||||
u64 id = bpf_get_current_pid_tgid();
|
||||
u32 pid = id;
|
||||
u64 slot, delta;
|
||||
|
||||
start = bpf_map_lookup_elem(&starts, &pid);
|
||||
if (!start)
|
||||
return;
|
||||
|
||||
delta = nsec - *start;
|
||||
|
||||
switch (units) {
|
||||
case USEC:
|
||||
delta /= 1000;
|
||||
break;
|
||||
case MSEC:
|
||||
delta /= 1000000;
|
||||
break;
|
||||
}
|
||||
|
||||
slot = log2l(delta);
|
||||
if (slot >= MAX_SLOTS)
|
||||
slot = MAX_SLOTS - 1;
|
||||
__sync_fetch_and_add(&hist[slot], 1);
|
||||
}
|
||||
|
||||
SEC("kretprobe/dummy_kretprobe")
|
||||
int BPF_KRETPROBE(dummy_kretprobe)
|
||||
{
|
||||
exit();
|
||||
return 0;
|
||||
}
|
||||
|
||||
char LICENSE[] SEC("license") = "GPL";
|
||||
```
|
||||
|
||||
### 代码解释
|
||||
|
||||
1. **头文件**：代码首先包含了必要的头文件，如 `vmlinux.h`（提供内核定义）和 `bpf_helpers.h`（提供 eBPF 程序的辅助函数）。
|
||||
|
||||
2. **全局变量**：`targ_tgid` 是目标进程 ID（或线程组 ID），`units` 确定延迟测量的时间单位（如微秒或毫秒）。
|
||||
|
||||
3. **BPF 映射**：定义了一个哈希映射（`starts`），用于存储每个进程 ID 的函数执行开始时间。另一个数组（`hist`）用于存储延迟分布。
|
||||
|
||||
4. **入口函数**：`entry()` 函数在函数进入时捕获当前时间戳，并将其存储在以进程 ID 为键的 `starts` 映射中。
|
||||
|
||||
5. **出口函数**：`exit()` 函数通过将存储的开始时间与当前时间相减来计算延迟。然后将结果分类到直方图槽中，并增加该槽的计数以记录该延迟范围的发生次数。
|
||||
|
||||
6. **探针**：`kprobe` 和 `kretprobe` 用于附加到函数的入口和出口点。这些探针触发 `entry()` 和 `exit()` 函数来测量延迟。
|
||||
|
||||
7. **许可证**：该程序根据 GPL 许可证发布，以确保符合内核的许可要求。
|
||||
|
||||
## 运行函数延迟工具
|
||||
|
||||
### 用户空间函数延迟
|
||||
|
||||
要跟踪用户空间函数（例如 `libc` 库中的 `read` 函数）的延迟，可以运行以下命令：
|
||||
|
||||
```console
|
||||
# ./funclatency /usr/lib/x86_64-linux-gnu/libc.so.6:read
|
||||
tracing /usr/lib/x86_64-linux-gnu/libc.so.6:read...
|
||||
tracing func read in /usr/lib/x86_64-linux-gnu/libc.so.6...
|
||||
Tracing /usr/lib/x86_64-linux-gnu/libc.so.6:read. Hit Ctrl-C to exit
|
||||
^C
|
||||
nsec : count distribution
|
||||
0 -> 1 : 0 | |
|
||||
2 -> 3 : 0 | |
|
||||
4 -> 7 : 0 | |
|
||||
8 -> 15 : 0 | |
|
||||
16 -> 31 : 0 | |
|
||||
32 -> 63 : 0 | |
|
||||
128 -> 255 : 0 | |
|
||||
512 -> 1023 : 0 | |
|
||||
65536 -> 131071 : 651 |****************************************+|
|
||||
131072 -> 262143 : 107 |****** |
|
||||
262144 -> 524287 : 36 |** |
|
||||
524288 -> 1048575 : 8 | |
|
||||
8388608 -> 16777215 : 2 | |
|
||||
Exiting trace of /usr/lib/x86_64-linux-gnu/libc.so.6:read
|
||||
```
|
||||
|
||||
### 内核空间函数延迟
|
||||
|
||||
要跟踪内核空间函数（例如 `vfs_read`）的延迟，可以运行以下命令：
|
||||
|
||||
```console
|
||||
# sudo ./funclatency -u vfs_read
|
||||
Tracing vfs_read. Hit Ctrl-C to exit
|
||||
^C
|
||||
usec : count distribution
|
||||
0 -> 1 : 0 | |
|
||||
8 -> 15 : 0 | |
|
||||
16 -> 31 : 3397 |****************************************|
|
||||
32 -> 63 : 2175 |************************* |
|
||||
64 -> 127 : 184 |** |
|
||||
1024 -> 2047 : 0 | |
|
||||
4096 -> 8191 : 5 | |
|
||||
2097152 ->
|
||||
|
||||
4194303 : 2 | |
|
||||
Exiting trace of vfs_read
|
||||
```
|
||||
|
||||
这些命令会跟踪指定函数（无论是在用户空间还是内核空间）的执行，并打印出观察到的延迟的直方图，显示函数执行时间的分布。
|
||||
|
||||
<https://github.com/eunomia-bpf/bpf-developer-tutorial/blob/main/src/33-funclatency>
|
||||
|
||||
## 结论
|
||||
|
||||
使用 eBPF 测量函数延迟可以深入了解用户空间和内核空间代码的性能。通过了解函数延迟，开发人员可以识别性能瓶颈、提高系统响应能力，并确保其应用程序的顺畅运行。
|
||||
|
||||
本文介绍了使用 eBPF 跟踪函数延迟的基本知识，包括实现该跟踪功能的 eBPF 内核代码概述。文中提供的示例展示了如何运行工具以跟踪用户空间和内核空间函数的延迟。
|
||||
|
||||
如果您有兴趣了解更多关于 eBPF 的知识，包括更多高级示例和教程，请访问我们的[教程代码库](https://github.com/eunomia-bpf/bpf-developer-tutorial)或我们的网站 [Eunomia](https://eunomia.dev/tutorials/)。
|
||||
|
||||
如果您正在寻找一个用于函数延迟测量的生产就绪工具，您可能想查看 BCC 仓库中的完整实现：[BCC 仓库](https://github.com/iovisor/bcc/blob/master/libbpf-tools/funclatency.c)。
|
||||
189
src/33-funclatency/README_en.md
Normal file
189
src/33-funclatency/README_en.md
Normal file
@@ -0,0 +1,189 @@
|
||||
# Measuring Function Latency with eBPF
|
||||
|
||||
In modern software systems, understanding the performance characteristics of functions—especially those critical to the operation of your application—is paramount. One key metric in performance analysis is **function latency**, which is the time taken by a function to execute from start to finish. By analyzing function latency, developers can identify bottlenecks, optimize performance, and ensure that their systems operate efficiently under various conditions.
|
||||
|
||||
This blog post will dive into how to measure function latency using eBPF, an incredibly powerful tool for tracing and monitoring both kernel and user-space programs.
|
||||
|
||||
## What is eBPF?
|
||||
|
||||
eBPF (Extended Berkeley Packet Filter) is a revolutionary technology that allows developers to write small programs that run in the Linux kernel. Originally designed for packet filtering, eBPF has evolved into a versatile tool for tracing, monitoring, and profiling system behavior. With eBPF, you can instrument almost any part of the Linux kernel or user-space programs to collect performance data, enforce security policies, or even debug systems in real time—all without the need to modify the kernel source code or restart the system.
|
||||
|
||||
eBPF programs are executed in a sandboxed environment within the kernel, ensuring safety and stability. These programs can attach to various hooks within the kernel, such as system calls, network events, and tracepoints, or even user-space functions using uprobes (user-level probes). The data collected by eBPF programs can then be exported to user space for analysis, making it an invaluable tool for system observability. `Uprobe` in kernel mode eBPF runtime may also cause relatively large performance overhead. In this case, you can also consider using user mode eBPF runtime, such as [bpftime](https://github.com/eunomia-bpf/bpftime).
|
||||
|
||||
## Why is Function Latency Important?
|
||||
|
||||
Function latency is a critical metric in performance analysis for both kernel and user-space applications. It provides insights into how long a particular function takes to execute, which is crucial for:
|
||||
|
||||
- **Identifying Performance Bottlenecks**: High function latency may indicate inefficiencies or issues within the code that need optimization.
|
||||
- **Ensuring System Responsiveness**: In real-time systems or latency-sensitive applications, understanding and minimizing function latency is essential to maintain responsiveness.
|
||||
- **Profiling and Benchmarking**: By measuring the latency of various functions, developers can benchmark their systems and compare the performance of different implementations or configurations.
|
||||
- **Debugging and Diagnostics**: When a system exhibits unexpected behavior or performance degradation, measuring function latency can help pinpoint the source of the problem.
|
||||
|
||||
Both kernel-space (e.g., system calls, file operations) and user-space (e.g., library functions) functions can be profiled for latency, providing a comprehensive view of system performance.
|
||||
|
||||
## eBPF Kernel Code for Function Latency
|
||||
|
||||
Below is an eBPF program designed to measure the latency of a function by hooking into its entry and exit points. The program uses kprobes and kretprobes (for kernel functions) or uprobes and uretprobes (for user-space functions) to capture the start and end times of the function execution.
|
||||
|
||||
```c
|
||||
// SPDX-License-Identifier: GPL-2.0
|
||||
/* Copyright (c) 2021 Google LLC. */
|
||||
#include "vmlinux.h"
|
||||
#include <bpf/bpf_core_read.h>
|
||||
#include <bpf/bpf_helpers.h>
|
||||
#include <bpf/bpf_tracing.h>
|
||||
#include "funclatency.h"
|
||||
#include "bits.bpf.h"
|
||||
|
||||
const volatile pid_t targ_tgid = 0;
|
||||
const volatile int units = 0;
|
||||
|
||||
/* key: pid. value: start time */
|
||||
struct {
|
||||
__uint(type, BPF_MAP_TYPE_HASH);
|
||||
__uint(max_entries, MAX_PIDS);
|
||||
__type(key, u32);
|
||||
__type(value, u64);
|
||||
} starts SEC(".maps");
|
||||
|
||||
__u32 hist[MAX_SLOTS] = {};
|
||||
|
||||
static void entry(void)
|
||||
{
|
||||
u64 id = bpf_get_current_pid_tgid();
|
||||
u32 tgid = id >> 32;
|
||||
u32 pid = id;
|
||||
u64 nsec;
|
||||
|
||||
if (targ_tgid && targ_tgid != tgid)
|
||||
return;
|
||||
nsec = bpf_ktime_get_ns();
|
||||
bpf_map_update_elem(&starts, &pid, &nsec, BPF_ANY);
|
||||
}
|
||||
|
||||
SEC("kprobe/dummy_kprobe")
|
||||
int BPF_KPROBE(dummy_kprobe)
|
||||
{
|
||||
entry();
|
||||
return 0;
|
||||
}
|
||||
|
||||
static void exit(void)
|
||||
{
|
||||
u64 *start;
|
||||
u64 nsec = bpf_ktime_get_ns();
|
||||
u64 id = bpf_get_current_pid_tgid();
|
||||
u32 pid = id;
|
||||
u64 slot, delta;
|
||||
|
||||
start = bpf_map_lookup_elem(&starts, &pid);
|
||||
if (!start)
|
||||
return;
|
||||
|
||||
delta = nsec - *start;
|
||||
|
||||
switch (units) {
|
||||
case USEC:
|
||||
delta /= 1000;
|
||||
break;
|
||||
case MSEC:
|
||||
delta /= 1000000;
|
||||
break;
|
||||
}
|
||||
|
||||
slot = log2l(delta);
|
||||
if (slot >= MAX_SLOTS)
|
||||
slot = MAX_SLOTS - 1;
|
||||
__sync_fetch_and_add(&hist[slot], 1);
|
||||
}
|
||||
|
||||
SEC("kretprobe/dummy_kretprobe")
|
||||
int BPF_KRETPROBE(dummy_kretprobe)
|
||||
{
|
||||
exit();
|
||||
return 0;
|
||||
}
|
||||
|
||||
char LICENSE[] SEC("license") = "GPL";
|
||||
```
|
||||
|
||||
### Explanation of the Code
|
||||
|
||||
1. **Header Files**: The code begins by including the necessary headers like `vmlinux.h` (which provides kernel definitions) and `bpf_helpers.h` (which offers helper functions for eBPF programs).
|
||||
|
||||
2. **Global Variables**: `targ_tgid` is a target process ID (or thread group ID), and `units` determines the time unit for latency measurement (e.g., microseconds or milliseconds).
|
||||
|
||||
3. **BPF Maps**: A hash map (`starts`) is defined to store the start time of function executions for each process ID. Another array (`hist`) is used to store the latency distribution.
|
||||
|
||||
4. **Entry Function**: The `entry()` function captures the current timestamp when the function is entered and stores it in the `starts` map keyed by the process ID.
|
||||
|
||||
5. **Exit Function**: The `exit()` function calculates the latency by subtracting the stored start time from the current time. The result is then categorized into a histogram slot, which is incremented to record the occurrence of that latency range.
|
||||
|
||||
6. **Probes**: The `kprobe` and `kretprobe` are used to attach to the entry and exit points of the function, respectively. These probes trigger the `entry()` and `exit()` functions to measure the latency.
|
||||
|
||||
7. **License**: The program is licensed under GPL to ensure compliance with kernel licensing requirements.
|
||||
|
||||
## Running the Function Latency Tool
|
||||
|
||||
### User-Space Function Latency
|
||||
|
||||
To trace the latency of a user-space function, such as the `read` function in the `libc` library, you can run the following command:
|
||||
|
||||
```console
|
||||
# ./funclatency /usr/lib/x86_64-linux-gnu/libc.so.6:read
|
||||
tracing /usr/lib/x86_64-linux-gnu/libc.so.6:read...
|
||||
tracing func read in /usr/lib/x86_64-linux-gnu/libc.so.6...
|
||||
Tracing /usr/lib/x86_64-linux-gnu/libc.so.6:read. Hit Ctrl-C to exit
|
||||
^C
|
||||
nsec : count distribution
|
||||
0 -> 1 : 0 | |
|
||||
2 -> 3 : 0 | |
|
||||
4 -> 7 : 0 | |
|
||||
8 -> 15 : 0 | |
|
||||
16 -> 31 : 0 | |
|
||||
32 -> 63 : 0 | |
|
||||
128 -> 255 : 0 | |
|
||||
512 -> 1023 : 0 | |
|
||||
65536 -> 131071 : 651 |****************************************+|
|
||||
131072 -> 262143 : 107 |****** |
|
||||
262144 -> 524287 : 36 |** |
|
||||
524288 -> 1048575 : 8 | |
|
||||
8388608 -> 16777215 : 2 | |
|
||||
Exiting trace of /usr/lib/x86_64-linux-gnu/libc.so.6:read
|
||||
```
|
||||
|
||||
### Kernel-Space Function Latency
|
||||
|
||||
To trace the latency of a kernel-space function, such as `vfs_read`, run the following command:
|
||||
|
||||
```console
|
||||
# sudo ./funclatency -u vfs_read
|
||||
Tracing vfs_read. Hit Ctrl-C to exit
|
||||
^C
|
||||
usec : count distribution
|
||||
0 -> 1 : 0 | |
|
||||
8 -> 15 : 0 | |
|
||||
16 -> 31 : 3397 |****************************************|
|
||||
32 -> 63 : 2175 |************************* |
|
||||
64 -> 127 : 184 |** |
|
||||
1024 -> 2047 : 0 | |
|
||||
4096 -> 8191 : 5 | |
|
||||
2097152 -> 4194303 : 2 | |
|
||||
Exiting trace of vfs_read
|
||||
```
|
||||
|
||||
These commands trace the execution of the specified function, either in user-space or kernel-space, and print a histogram of the observed latencies, showing the distribution of function execution times.
|
||||
|
||||
You can find the source code in <https://github.com/eunomia-bpf/bpf-developer-tutorial/blob/main/src/33-funclatency>
|
||||
|
||||
## Conclusion
|
||||
|
||||
Measuring function latency with eBPF offers deep insights into the performance of both user-space and kernel-space code. By understanding function latency, developers can identify performance bottlenecks, improve system responsiveness, and ensure the smooth operation of their applications.
|
||||
|
||||
This
|
||||
|
||||
blog post covered the basics of using eBPF to trace function latency, including an overview of the eBPF kernel code used to perform the tracing. The examples provided demonstrated how to run the tool to trace both user-space and kernel-space functions.
|
||||
|
||||
For those interested in learning more about eBPF, including more advanced examples and tutorials, please visit our [https://github.com/eunomia-bpf/bpf-developer-tutorial](https://github.com/eunomia-bpf/bpf-developer-tutorial) or our website [https://eunomia.dev/tutorials/](https://eunomia.dev/tutorials/).
|
||||
|
||||
If you are looking for a production-ready tool for function latency measurement, you might want to check out the full implementation available in the [BCC repository](https://github.com/iovisor/bcc/blob/master/libbpf-tools/funclatency.c).
|
||||
31
src/33-funclatency/bits.bpf.h
Normal file
31
src/33-funclatency/bits.bpf.h
Normal file
@@ -0,0 +1,31 @@
|
||||
/* SPDX-License-Identifier: (LGPL-2.1 OR BSD-2-Clause) */
|
||||
#ifndef __BITS_BPF_H
|
||||
#define __BITS_BPF_H
|
||||
|
||||
#define READ_ONCE(x) (*(volatile typeof(x) *)&(x))
|
||||
#define WRITE_ONCE(x, val) ((*(volatile typeof(x) *)&(x)) = val)
|
||||
|
||||
static __always_inline u64 log2(u32 v)
|
||||
{
|
||||
u32 shift, r;
|
||||
|
||||
r = (v > 0xFFFF) << 4; v >>= r;
|
||||
shift = (v > 0xFF) << 3; v >>= shift; r |= shift;
|
||||
shift = (v > 0xF) << 2; v >>= shift; r |= shift;
|
||||
shift = (v > 0x3) << 1; v >>= shift; r |= shift;
|
||||
r |= (v >> 1);
|
||||
|
||||
return r;
|
||||
}
|
||||
|
||||
static __always_inline u64 log2l(u64 v)
|
||||
{
|
||||
u32 hi = v >> 32;
|
||||
|
||||
if (hi)
|
||||
return log2(hi) + 32;
|
||||
else
|
||||
return log2(v);
|
||||
}
|
||||
|
||||
#endif /* __BITS_BPF_H */
|
||||
@@ -33,13 +33,6 @@ static void entry(void)
|
||||
bpf_map_update_elem(&starts, &pid, &nsec, BPF_ANY);
|
||||
}
|
||||
|
||||
SEC("fentry/dummy_fentry")
|
||||
int BPF_PROG(dummy_fentry)
|
||||
{
|
||||
entry();
|
||||
return 0;
|
||||
}
|
||||
|
||||
SEC("kprobe/dummy_kprobe")
|
||||
int BPF_KPROBE(dummy_kprobe)
|
||||
{
|
||||
@@ -76,13 +69,6 @@ static void exit(void)
|
||||
__sync_fetch_and_add(&hist[slot], 1);
|
||||
}
|
||||
|
||||
SEC("fexit/dummy_fexit")
|
||||
int BPF_PROG(dummy_fexit)
|
||||
{
|
||||
exit();
|
||||
return 0;
|
||||
}
|
||||
|
||||
SEC("kretprobe/dummy_kretprobe")
|
||||
int BPF_KRETPROBE(dummy_kretprobe)
|
||||
{
|
||||
@@ -22,8 +22,6 @@
|
||||
#include <bpf/bpf.h>
|
||||
#include "funclatency.h"
|
||||
#include "funclatency.skel.h"
|
||||
#include "trace_helpers.h"
|
||||
#include "uprobe_helpers.h"
|
||||
|
||||
#define warn(...) fprintf(stderr, __VA_ARGS__)
|
||||
|
||||
@@ -63,7 +61,7 @@ static const char program_doc[] =
|
||||
" ./funclatency -m do_nanosleep # time do_nanosleep(), in milliseconds\n"
|
||||
" ./funclatency -u vfs_read # time vfs_read(), in microseconds\n"
|
||||
" ./funclatency -p 181 vfs_read # time process 181 only\n"
|
||||
" ./funclatency -p 181 c:read # time the read() C library function\n"
|
||||
" ./funclatency -p 181 /usr/lib/x86_64-linux-gnu/libc.so.6:read # time the read() C library function\n"
|
||||
" ./funclatency -p 181 :foo # time foo() from pid 181's userspace\n"
|
||||
" ./funclatency -i 2 -d 10 vfs_read # output every 2 seconds, for 10s\n"
|
||||
" ./funclatency -mTi 5 vfs_read # output every 5 seconds, with timestamps\n";
|
||||
@@ -223,8 +221,6 @@ static int attach_kprobes(struct funclatency_bpf *obj)
|
||||
static int attach_uprobes(struct funclatency_bpf *obj)
|
||||
{
|
||||
char *binary, *function;
|
||||
char bin_path[PATH_MAX];
|
||||
off_t func_off;
|
||||
int ret = -1;
|
||||
long err;
|
||||
|
||||
@@ -234,6 +230,7 @@ static int attach_uprobes(struct funclatency_bpf *obj)
|
||||
warn("strdup failed");
|
||||
return -1;
|
||||
}
|
||||
printf("tracing %s...\n", binary);
|
||||
function = strchr(binary, ':');
|
||||
if (!function)
|
||||
{
|
||||
@@ -242,20 +239,15 @@ static int attach_uprobes(struct funclatency_bpf *obj)
|
||||
}
|
||||
*function = '\0';
|
||||
function++;
|
||||
printf("tracing func %s in %s...\n", function, binary);
|
||||
|
||||
if (resolve_binary_path(binary, env.pid, bin_path, sizeof(bin_path)))
|
||||
goto out_binary;
|
||||
|
||||
func_off = get_elf_func_offset(bin_path, function);
|
||||
if (func_off < 0)
|
||||
{
|
||||
warn("Could not find %s in %s\n", function, bin_path);
|
||||
goto out_binary;
|
||||
}
|
||||
LIBBPF_OPTS(bpf_uprobe_opts, opts);
|
||||
opts.func_name = function;
|
||||
opts.retprobe = false;
|
||||
|
||||
obj->links.dummy_kprobe =
|
||||
bpf_program__attach_uprobe(obj->progs.dummy_kprobe, false,
|
||||
env.pid ?: -1, bin_path, func_off);
|
||||
bpf_program__attach_uprobe_opts(obj->progs.dummy_kprobe,
|
||||
env.pid ?: -1, binary, 0, &opts);
|
||||
if (!obj->links.dummy_kprobe)
|
||||
{
|
||||
err = -errno;
|
||||
@@ -263,9 +255,11 @@ static int attach_uprobes(struct funclatency_bpf *obj)
|
||||
goto out_binary;
|
||||
}
|
||||
|
||||
opts.retprobe = true;
|
||||
|
||||
obj->links.dummy_kretprobe =
|
||||
bpf_program__attach_uprobe(obj->progs.dummy_kretprobe, true,
|
||||
env.pid ?: -1, bin_path, func_off);
|
||||
bpf_program__attach_uprobe_opts(obj->progs.dummy_kretprobe,
|
||||
env.pid ?: -1, binary, 0, &opts);
|
||||
if (!obj->links.dummy_kretprobe)
|
||||
{
|
||||
err = -errno;
|
||||
@@ -290,6 +284,10 @@ static void sig_hand(int signr)
|
||||
|
||||
static struct sigaction sigact = {.sa_handler = sig_hand};
|
||||
|
||||
#ifndef min
|
||||
#define min(a,b) ((a) < (b) ? (a) : (b))
|
||||
#endif
|
||||
|
||||
static void print_stars(unsigned int val, unsigned int val_max, int width)
|
||||
{
|
||||
int num_stars, num_spaces, i;
|
||||
@@ -362,9 +360,7 @@ int main(int argc, char **argv)
|
||||
struct tm *tm;
|
||||
char ts[32];
|
||||
time_t t;
|
||||
int idx, cg_map_fd;
|
||||
int cgfd = -1;
|
||||
bool used_fentry = false;
|
||||
|
||||
err = argp_parse(&argp, argc, argv, 0, NULL, &env);
|
||||
if (err)
|
||||
@@ -439,7 +435,6 @@ int main(int argc, char **argv)
|
||||
|
||||
cleanup:
|
||||
funclatency_bpf__destroy(obj);
|
||||
cleanup_core_btf(&open_opts);
|
||||
if (cgfd > 0)
|
||||
close(cgfd);
|
||||
|
||||
149
src/39-nginx/README.md
Normal file
149
src/39-nginx/README.md
Normal file
@@ -0,0 +1,149 @@
|
||||
# 使用 eBPF 跟踪 Nginx 请求
|
||||
|
||||
## 引言
|
||||
|
||||
Nginx 是世界上最流行的 Web 服务器和反向代理之一，以其高性能、稳定性和低资源消耗而闻名。它广泛用于提供静态内容、负载均衡以及作为动态应用的反向代理。为了保持其性能优势，监控和优化 Nginx 的运行尤为重要，尤其是在处理大量请求时。利用 eBPF（扩展的伯克利包过滤器），可以深入了解 Nginx 的性能表现，识别瓶颈并进行优化，而无需修改源代码或重启服务。
|
||||
|
||||
eBPF 是一项革命性技术，允许开发人员在 Linux 内核中运行自定义程序。最初设计用于网络数据包过滤，但 eBPF 现已发展为一个多功能工具，广泛应用于跟踪、监控和分析系统行为。通过利用 eBPF，您可以跟踪 Nginx 的关键函数，测量延迟，识别瓶颈，进而优化系统性能。
|
||||
|
||||
## 背景：Nginx 和 eBPF
|
||||
|
||||
### Nginx
|
||||
|
||||
Nginx 采用事件驱动架构，使其在资源占用极少的情况下能够高效处理成千上万的并发连接。这种高效性依赖于其请求处理、响应生成和事件处理等多个性能关键函数。了解这些函数在不同负载下的表现对于优化 Nginx 的使用至关重要。
|
||||
|
||||
### eBPF
|
||||
|
||||
eBPF 程序在 Linux 内核的安全沙盒环境中运行。这些程序可以附加到各种钩子上，如系统调用、跟踪点，甚至可以通过 uprobes（用户级探针）附加到用户空间的函数。这使得 eBPF 成为一个强大的系统可观测性工具，可以收集详细的性能数据并实时执行策略。
|
||||
|
||||
eBPF 的一个常见用例是跟踪函数执行时间，以测量延迟。这对于了解 Nginx 中特定函数的执行时间特别有用，有助于诊断性能问题、优化资源使用，并提高 Nginx 部署的整体效率。
|
||||
|
||||
### Uprobes
|
||||
|
||||
Uprobes 是一种用于跟踪用户空间应用程序函数的探针，它通过附加到特定用户空间函数的入口和出口点，可以捕获精确的时间信息。然而，需要注意的是，在内核模式 eBPF 运行时使用 uprobes 可能会带来一定的性能开销。为此，您可以考虑使用基于 LLVM JIT/AOT 的用户模式 eBPF 运行时 [bpftime](https://github.com/eunomia-bpf/bpftime)。这种运行时可以在用户空间中运行 eBPF 程序，与内核模式 eBPF 兼容，并有可能降低开销。
|
||||
|
||||
## Nginx 的性能关键函数
|
||||
|
||||
以下是 Nginx 中一些性能关键的函数，可以通过 eBPF 进行监控：
|
||||
|
||||
- **ngx_http_process_request**：负责处理传入的 HTTP 请求。监控此函数有助于跟踪请求处理的开始。
|
||||
- **ngx_http_upstream_send_request**：当 Nginx 作为反向代理时，负责向上游服务器发送请求。
|
||||
- **ngx_http_finalize_request**：完成 HTTP 请求的处理，包括发送响应。跟踪此函数可以衡量整个请求处理的时间。
|
||||
- **ngx_event_process_posted**：处理事件循环中的队列事件。
|
||||
- **ngx_handle_read_event**：负责处理来自套接字的读取事件，对监控网络 I/O 性能至关重要。
|
||||
- **ngx_writev_chain**：负责将响应发送回客户端，通常与写事件循环结合使用。
|
||||
|
||||
## 使用 bpftrace 跟踪 Nginx 函数
|
||||
|
||||
为了监控这些函数，我们可以使用 `bpftrace`，一种 eBPF 的高级跟踪语言。以下是一个用于跟踪几个关键 Nginx 函数执行时间的脚本：
|
||||
|
||||
```bt
|
||||
#!/usr/sbin/bpftrace
|
||||
|
||||
// 监控 HTTP 请求处理的开始
|
||||
uprobe:/usr/sbin/nginx:ngx_http_process_request
|
||||
{
|
||||
printf("HTTP 请求处理开始 (tid: %d)\n", tid);
|
||||
@start[tid] = nsecs;
|
||||
}
|
||||
|
||||
// 监控 HTTP 请求的完成
|
||||
uretprobe:/usr/sbin/nginx:ngx_http_finalize_request
|
||||
/@start[tid]/
|
||||
{
|
||||
$elapsed = nsecs - @start[tid];
|
||||
printf("HTTP 请求处理时间: %d ns (tid: %d)\n", $elapsed, tid);
|
||||
delete(@start[tid]);
|
||||
}
|
||||
|
||||
// 监控向上游服务器发送请求的开始
|
||||
uprobe:/usr/sbin/nginx:ngx_http_upstream_send_request
|
||||
{
|
||||
printf("开始向上游服务器发送请求 (tid: %d)\n", tid);
|
||||
@upstream_start[tid] = nsecs;
|
||||
}
|
||||
|
||||
// 监控上游请求发送完成
|
||||
uretprobe:/usr/sbin/nginx:ngx_http_upstream_send_request
|
||||
/@upstream_start[tid]/
|
||||
{
|
||||
$elapsed = nsecs - @upstream_start[tid];
|
||||
printf("上游请求发送完成时间: %d ns (tid: %d)\n", $elapsed, tid);
|
||||
delete(@upstream_start[tid]);
|
||||
}
|
||||
|
||||
// 监控事件处理的开始
|
||||
uprobe:/usr/sbin/nginx:ngx_event_process_posted
|
||||
{
|
||||
printf("事件处理开始 (tid: %d)\n", tid);
|
||||
@event_start[tid] = nsecs;
|
||||
}
|
||||
|
||||
// 监控事件处理的完成
|
||||
uretprobe:/usr/sbin/nginx:ngx_event_process_posted
|
||||
/@event_start[tid]/
|
||||
{
|
||||
$elapsed = nsecs - @event_start[tid];
|
||||
printf("事件处理时间: %d ns (tid: %d)\n", $elapsed, tid);
|
||||
delete(@event_start[tid]);
|
||||
}
|
||||
```
|
||||
|
||||
### 运行脚本
|
||||
|
||||
要运行上述脚本，先启动 Nginx，然后使用 `curl` 等工具生成 HTTP 请求：
|
||||
|
||||
```bt
|
||||
# bpftrace /home/yunwei37/bpf-developer-tutorial/src/39-nginx/trace.bt
|
||||
Attaching 4 probes...
|
||||
事件处理开始 (tid: 1071)
|
||||
事件处理时间: 166396 ns (tid: 1071)
|
||||
事件处理开始 (tid: 1071)
|
||||
事件处理时间: 87998 ns (tid: 1071)
|
||||
HTTP 请求处理开始 (tid: 1071)
|
||||
HTTP 请求处理时间: 1083969 ns (tid: 1071)
|
||||
事件处理开始 (tid: 1071)
|
||||
事件处理时间: 92597 ns (tid: 1071)
|
||||
```
|
||||
|
||||
该脚本监控了几个 Nginx 函数的开始和结束时间，并打印了每个函数的执行时间。这些数据可以用来分析和优化 Nginx 服务器的性能。
|
||||
|
||||
## 测试 Nginx 的函数延迟
|
||||
|
||||
为了更详细地分析函数延迟，您可以使用 `funclatency` 工具，该工具可以测量 Nginx 函数的延迟分布。以下是如何测试 `ngx_http_process_request` 函数的延迟：
|
||||
|
||||
```console
|
||||
# sudo ./funclatency /usr/sbin/nginx:ngx_http_process_request
|
||||
tracing /usr/sbin/nginx:ngx_http_process_request...
|
||||
tracing func ngx_http_process_request in /usr/sbin/nginx...
|
||||
Tracing /usr/sbin/nginx:ngx_http_process_request. Hit Ctrl-C to exit
|
||||
^C
|
||||
nsec : count distribution
|
||||
0 -> 1 : 0 | |
|
||||
524288 -> 1048575 : 16546 |****************************************|
|
||||
1048576 -> 2097151 : 2296 |***** |
|
||||
2097152 -> 4194303 : 1264 |*** |
|
||||
4194304 -> 8388607 : 293 | |
|
||||
8388608 -> 16777215 : 37 | |
|
||||
Exiting trace of /usr/sbin/nginx:ngx_http_process_request
|
||||
```
|
||||
|
||||
### 结果总结
|
||||
|
||||
上述结果显示了 `ngx_http_process_request` 函数的延迟分布。大多数请求在 524,288 至 1,048,575 纳秒内处理完成，少部分请求处理时间更长。这些信息对于识别性能瓶颈和优化 Nginx 请求处理至关重要。
|
||||
|
||||
通过使用 `funclatency`，您可以：
|
||||
|
||||
- **识别性能瓶颈**：了解哪些函数执行时间最长，并将优化工作重点放在这些函数上。
|
||||
- **监控系统性能**：定期监控函数延迟，确保在高负载下 Nginx 服务器的最佳性能。
|
||||
- **优化 Nginx 配置**：利用延迟测量得出的洞察调整 Nginx 设置或修改应用程序，以提高整体性能。
|
||||
|
||||
您可以在 [bpf-developer-tutorial 仓库](https://github.com/eunomia-bpf/bpf-developer-tutorial/blob/main/src/33-funclatency) 中找到 `funclatency` 工具。
|
||||
|
||||
## 结论
|
||||
|
||||
通过 eBPF 跟踪 Nginx 请求可以为您的 Web 服务器提供宝贵的性能洞察，使您能够监控、分析和优化其操作。使用 `bpftrace` 和 `funclatency`
|
||||
|
||||
等工具，您可以测量函数执行时间、识别瓶颈，并根据数据做出决策来改进 Nginx 部署。
|
||||
|
||||
如果您有兴趣了解更多关于 eBPF 的知识，包括更多高级示例和教程，请访问我们的 [https://github.com/eunomia-bpf/bpf-developer-tutorial](https://github.com/eunomia-bpf/bpf-developer-tutorial) 或查看我们的网站 [https://eunomia.dev/tutorials/](https://eunomia.dev/tutorials/)。
|
||||
145
src/39-nginx/README_en.md
Normal file
145
src/39-nginx/README_en.md
Normal file
@@ -0,0 +1,145 @@
|
||||
# Using eBPF to Trace Nginx Requests
|
||||
|
||||
Nginx is one of the most popular web servers and reverse proxies in the world, known for its high performance, stability, and low resource consumption. It is widely used for serving static content, load balancing, and acting as a reverse proxy for dynamic applications. To maintain its performance edge, it's crucial to monitor and optimize Nginx's operations, especially when handling a large number of requests. One powerful way to gain insights into Nginx's performance is by using eBPF (Extended Berkeley Packet Filter).
|
||||
|
||||
eBPF is a revolutionary technology that allows developers to run custom programs in the Linux kernel. Originally designed for network packet filtering, eBPF has evolved into a versatile tool for tracing, monitoring, and profiling system behavior in both kernel and user space. By leveraging eBPF, you can trace Nginx's critical functions, measure latency, and identify bottlenecks without modifying the source code or restarting the service.
|
||||
|
||||
## Background: Nginx and eBPF
|
||||
|
||||
### Nginx
|
||||
|
||||
Nginx operates on an event-driven architecture, making it highly efficient and capable of handling thousands of simultaneous connections with minimal resources. This efficiency is achieved through various performance-critical functions involved in request processing, response generation, and event handling. Understanding how these functions behave under different loads is key to optimizing Nginx for your specific use case.
|
||||
|
||||
### eBPF
|
||||
|
||||
eBPF programs run in a secure, sandboxed environment within the Linux kernel. These programs can attach to various hooks, such as system calls, tracepoints, and even user-space functions via uprobes (user-level probes). This capability allows you to collect detailed performance data and enforce policies in real time, making eBPF an invaluable tool for system observability.
|
||||
|
||||
One common use case of eBPF is tracing function execution to measure latency, which is particularly useful for understanding how long specific Nginx functions take to execute. This information can help in diagnosing performance issues, optimizing resource usage, and improving the overall efficiency of your Nginx deployment.
|
||||
|
||||
### Uprobes
|
||||
|
||||
Uprobes are a type of probe that can be used to trace functions in user-space applications, such as Nginx. They work by attaching to specific user-space function entry and exit points, allowing you to capture precise timing information. However, it’s important to note that using uprobes in the kernel mode eBPF runtime may cause some performance overhead. To mitigate this, you can consider using a user-mode eBPF runtime like [bpftime](https://github.com/eunomia-bpf/bpftime), which is based on LLVM JIT/AOT. This runtime can run eBPF programs in user space, offering compatibility with kernel mode eBPF while potentially reducing overhead.
|
||||
|
||||
## Performance-Critical Functions in Nginx
|
||||
|
||||
Here are some key Nginx functions that are performance-critical and can be monitored using eBPF:
|
||||
|
||||
- **ngx_http_process_request**: Processes incoming HTTP requests. Monitoring this function helps track the start of request handling.
|
||||
- **ngx_http_upstream_send_request**: Handles sending requests to upstream servers when Nginx is acting as a reverse proxy.
|
||||
- **ngx_http_finalize_request**: Finalizes HTTP request processing, including sending the response. Tracing this can measure total request handling time.
|
||||
- **ngx_event_process_posted**: Processes queued events as part of the Nginx event loop.
|
||||
- **ngx_handle_read_event**: Handles read events from sockets, crucial for monitoring network I/O performance.
|
||||
- **ngx_writev_chain**: Sends responses back to the client, typically used in conjunction with the write event loop.
|
||||
|
||||
## Using bpftrace to Trace Nginx Functions
|
||||
|
||||
To monitor these functions, we can use `bpftrace`, a high-level tracing language for eBPF. Below is a script that traces the execution time of several critical Nginx functions:
|
||||
|
||||
```bt
|
||||
#!/usr/sbin/bpftrace
|
||||
|
||||
// Monitor the start of HTTP request processing
|
||||
uprobe:/usr/sbin/nginx:ngx_http_process_request
|
||||
{
|
||||
printf("HTTP request processing started (tid: %d)\n", tid);
|
||||
@start[tid] = nsecs;
|
||||
}
|
||||
|
||||
// Monitor when an HTTP request is finalized
|
||||
uretprobe:/usr/sbin/nginx:ngx_http_finalize_request
|
||||
/@start[tid]/
|
||||
{
|
||||
$elapsed = nsecs - @start[tid];
|
||||
printf("HTTP request processed in %d ns (tid: %d)\n", $elapsed, tid);
|
||||
delete(@start[tid]);
|
||||
}
|
||||
|
||||
// Monitor the start of sending a request to an upstream server
|
||||
uprobe:/usr/sbin/nginx:ngx_http_upstream_send_request
|
||||
{
|
||||
printf("Upstream request sending started (tid: %d)\n", tid);
|
||||
@upstream_start[tid] = nsecs;
|
||||
}
|
||||
|
||||
// Monitor when the upstream request is sent
|
||||
uretprobe:/usr/sbin/nginx:ngx_http_upstream_send_request
|
||||
/@upstream_start[tid]/
|
||||
{
|
||||
$elapsed = nsecs - @upstream_start[tid];
|
||||
printf("Upstream request sent in %d ns (tid: %d)\n", $elapsed, tid);
|
||||
delete(@upstream_start[tid]);
|
||||
}
|
||||
|
||||
// Monitor the start of event processing
|
||||
uprobe:/usr/sbin/nginx:ngx_event_process_posted
|
||||
{
|
||||
printf("Event processing started (tid: %d)\n", tid);
|
||||
@event_start[tid] = nsecs;
|
||||
}
|
||||
|
||||
// Monitor when event processing is completed
|
||||
uretprobe:/usr/sbin/nginx:ngx_event_process_posted
|
||||
/@event_start[tid]/
|
||||
{
|
||||
$elapsed = nsecs - @event_start[tid];
|
||||
printf("Event processed in %d ns (tid: %d)\n", $elapsed, tid);
|
||||
delete(@event_start[tid]);
|
||||
}
|
||||
```
|
||||
|
||||
### Running the Program
|
||||
|
||||
To run the above script, start Nginx and use a tool like `curl` to generate HTTP requests:
|
||||
|
||||
```bt
|
||||
# bpftrace /home/yunwei37/bpf-developer-tutorial/src/39-nginx/trace.bt
|
||||
Attaching 4 probes...
|
||||
Event processing started (tid: 1071)
|
||||
Event processed in 166396 ns (tid: 1071)
|
||||
Event processing started (tid: 1071)
|
||||
Event processed in 87998 ns (tid: 1071)
|
||||
HTTP request processing started (tid: 1071)
|
||||
HTTP request processed in 1083969 ns (tid: 1071)
|
||||
Event processing started (tid: 1071)
|
||||
Event processed in 92597 ns (tid: 1071)
|
||||
```
|
||||
|
||||
The script monitors the start and end times of various Nginx functions and prints the elapsed time for each. This data can be used to analyze and optimize the performance of your Nginx server.
|
||||
|
||||
## Testing Function Latency in Nginx
|
||||
|
||||
For a more detailed analysis of function latency, you can use the `funclatency` tool, which measures the latency distribution of Nginx functions. Here’s how to test the latency of the `ngx_http_process_request` function:
|
||||
|
||||
```console
|
||||
# sudo ./funclatency /usr/sbin/nginx:ngx_http_process_request
|
||||
tracing /usr/sbin/nginx:ngx_http_process_request...
|
||||
tracing func ngx_http_process_request in /usr/sbin/nginx...
|
||||
Tracing /usr/sbin/nginx:ngx_http_process_request. Hit Ctrl-C to exit
|
||||
^C
|
||||
nsec : count distribution
|
||||
0 -> 1 : 0 | |
|
||||
524288 -> 1048575 : 16546 |****************************************|
|
||||
1048576 -> 2097151 : 2296 |***** |
|
||||
2097152 -> 4194303 : 1264 |*** |
|
||||
4194304 -> 8388607 : 293 | |
|
||||
8388608 -> 16777215 : 37 | |
|
||||
Exiting trace of /usr/sbin/nginx:ngx_http_process_request
|
||||
```
|
||||
|
||||
### Summary of Results
|
||||
|
||||
The results above show the distribution of latency for the `ngx_http_process_request` function. The majority of requests were processed within 524,288 to 1,048,575 nanoseconds, with a smaller percentage taking longer. This information can be crucial in identifying performance bottlenecks and optimizing request handling in Nginx.
|
||||
|
||||
By using `funclatency`, you can:
|
||||
|
||||
- **Identify Performance Bottlenecks**: Understand which functions are taking the most time to execute and focus your optimization efforts there.
|
||||
- **Monitor System Performance**: Regularly monitor function latency to ensure your Nginx server is performing optimally, especially under heavy load.
|
||||
- **Optimize Nginx Configuration**: Use the insights gained from latency measurements to tweak Nginx settings or modify your application to improve overall performance.
|
||||
|
||||
You can find the `funclatency` tool in the [bpf-developer-tutorial repository](https://github.com/eunomia-bpf/bpf-developer-tutorial/blob/main/src/33-funclatency).
|
||||
|
||||
## Conclusion
|
||||
|
||||
Tracing Nginx requests with eBPF provides valuable insights into the performance of your web server, allowing you to monitor, analyze, and optimize its operation. By using tools like `bpftrace` and `funclatency`, you can measure function execution times, identify bottlenecks, and make data-driven decisions to improve your Nginx deployment.
|
||||
|
||||
For those interested in learning more about eBPF, including more advanced examples and tutorials, please visit our [https://github.com/eunomia-bpf/bpf-developer-tutorial](https://github.com/eunomia-bpf/bpf-developer-tutorial) or check out our [https://eunomia.dev/tutorials/](https://eunomia.dev/tutorials/).
|
||||
49
src/39-nginx/trace.bt
Normal file
49
src/39-nginx/trace.bt
Normal file
@@ -0,0 +1,49 @@
|
||||
#!/usr/sbin/bpftrace
|
||||
|
||||
// Monitor the start of HTTP request processing
|
||||
uprobe:/usr/sbin/nginx:ngx_http_process_request
|
||||
{
|
||||
printf("HTTP request processing started (tid: %d)\n", tid);
|
||||
@start[tid] = nsecs;
|
||||
}
|
||||
|
||||
// Monitor when an HTTP request is finalized
|
||||
uretprobe:/usr/sbin/nginx:ngx_http_finalize_request
|
||||
/@start[tid]/
|
||||
{
|
||||
$elapsed = nsecs - @start[tid];
|
||||
printf("HTTP request processed in %d ns (tid: %d)\n", $elapsed, tid);
|
||||
delete(@start[tid]);
|
||||
}
|
||||
|
||||
// Monitor the start of sending a request to an upstream server
|
||||
uprobe:/usr/sbin/nginx:ngx_http_upstream_send_request
|
||||
{
|
||||
printf("Upstream request sending started (tid: %d)\n", tid);
|
||||
@upstream_start[tid] = nsecs;
|
||||
}
|
||||
|
||||
// Monitor when the upstream request is sent
|
||||
uretprobe:/usr/sbin/nginx:ngx_http_upstream_send_request
|
||||
/@upstream_start[tid]/
|
||||
{
|
||||
$elapsed = nsecs - @upstream_start[tid];
|
||||
printf("Upstream request sent in %d ns (tid: %d)\n", $elapsed, tid);
|
||||
delete(@upstream_start[tid]);
|
||||
}
|
||||
|
||||
// Monitor the start of event processing
|
||||
uprobe:/usr/sbin/nginx:ngx_event_process_posted
|
||||
{
|
||||
printf("Event processing started (tid: %d)\n", tid);
|
||||
@event_start[tid] = nsecs;
|
||||
}
|
||||
|
||||
// Monitor when event processing is completed
|
||||
uretprobe:/usr/sbin/nginx:ngx_event_process_posted
|
||||
/@event_start[tid]/
|
||||
{
|
||||
$elapsed = nsecs - @event_start[tid];
|
||||
printf("Event processed in %d ns (tid: %d)\n", $elapsed, tid);
|
||||
delete(@event_start[tid]);
|
||||
}
|
||||
@@ -52,7 +52,10 @@ Networking and tracing:
|
||||
- [Tracing HTTP requests or other layer-7 protocols using eBPF socket filter or syscall trace](23-http/README.md)
|
||||
- [Accelerating network request forwarding using sockops](29-sockops/README.md)
|
||||
- [Capturing Plain Text Data of Various Libraries' SSL/TLS Using uprobe](30-sslsniff/README.md)
|
||||
- [Using eBPF to Trace Go Routine States](31-goroutine/README.md)
|
||||
- [Measuring Function Latency with eBPF](33-funclatency/README.md)
|
||||
- [Use uprobe to trace Rust programs](37-uprobe-rust/README.md)
|
||||
- [Using eBPF to Trace Nginx Requests](39-nginx/README.md)
|
||||
|
||||
Security:
|
||||
|
||||
|
||||
Reference in New Issue
Block a user