Add wall clock analysis (#184)

* Add combined on-CPU and off-CPU profiler script - Implemented a new profiling tool that captures both on-CPU and off-CPU activity for a specified process. - The script runs 'oncputime' and 'offcputime' tools simultaneously and combines their results into a unified flamegraph. - Added functionality to discover threads and profile them individually if the application is multi-threaded. - Included error handling for tool execution and output processing. - Created methods for generating flamegraph data and SVG visualizations. - Added command-line argument parsing for user-defined profiling parameters. - Implemented detailed analysis reports for both individual threads and overall profiling results. * feat: integrate blazesym v0.2.0 for improved symbol resolution and add test program for memory leak detection * docs: update README to enhance wall clock profiling tutorial with detailed explanations and examples * feat: add wallclock-profiler tests to CI workflow for tool validation * fix: rename combined_profiler.py to wallclock_profiler.py in test scripts and usage examples
2026-02-09 13:15:14 +08:00 · 2025-09-30 22:11:52 -07:00
parent da67bfcf24
commit 70451702f0
31 changed files with 8584 additions and 53 deletions
--- a/src/16-memleak/.gitignore
+++ b/src/16-memleak/.gitignore
@@ -6,3 +6,5 @@ package.json
 package.yaml
 ecli
 memleak
+test_memleak
+.output/
--- a/src/16-memleak/Makefile
+++ b/src/16-memleak/Makefile
@@ -25,6 +25,7 @@ CFLAGS := -g -Wall
 ALL_LDFLAGS := $(LDFLAGS) $(EXTRA_LDFLAGS)

 APPS = # minimal minimal_legacy bootstrap uprobe kprobe fentry usdt sockfilter tc ksyscall
+TEST_APPS = test_memleak

 CARGO ?= $(shell which cargo)
 ifeq ($(strip $(CARGO)),)
@@ -69,12 +70,12 @@ $(call allow-override,CC,$(CROSS_COMPILE)cc)
 $(call allow-override,LD,$(CROSS_COMPILE)ld)

 .PHONY: all
-all: $(APPS)
+all: $(APPS) $(TEST_APPS)

 .PHONY: clean
 clean:
 	$(call msg,CLEAN)
-	$(Q)rm -rf $(OUTPUT) $(APPS)
+	$(Q)rm -rf $(OUTPUT) $(APPS) $(TEST_APPS)

 $(OUTPUT) $(OUTPUT)/libbpf $(BPFTOOL_OUTPUT):
 	$(call msg,MKDIR,$@)
@@ -94,16 +95,16 @@ $(BPFTOOL): | $(BPFTOOL_OUTPUT)
 	$(Q)$(MAKE) ARCH= CROSS_COMPILE= OUTPUT=$(BPFTOOL_OUTPUT)/ -C $(BPFTOOL_SRC) bootstrap


-$(LIBBLAZESYM_SRC)/target/release/libblazesym.a::
-	$(Q)cd $(LIBBLAZESYM_SRC) && RUSTFLAGS="-A dangerous_implicit_autorefs" $(CARGO) build --features=cheader,dont-generate-test-files --release
+$(LIBBLAZESYM_SRC)/target/release/libblazesym_c.a::
+	$(Q)cd $(LIBBLAZESYM_SRC)/capi && $(CARGO) build --release

-$(LIBBLAZESYM_OBJ): $(LIBBLAZESYM_SRC)/target/release/libblazesym.a | $(OUTPUT)
+$(LIBBLAZESYM_OBJ): $(LIBBLAZESYM_SRC)/target/release/libblazesym_c.a | $(OUTPUT)
 	$(call msg,LIB, $@)
-	$(Q)cp $(LIBBLAZESYM_SRC)/target/release/libblazesym.a $@
+	$(Q)cp $(LIBBLAZESYM_SRC)/target/release/libblazesym_c.a $@

-$(LIBBLAZESYM_HEADER): $(LIBBLAZESYM_SRC)/target/release/libblazesym.a | $(OUTPUT)
+$(LIBBLAZESYM_HEADER): $(LIBBLAZESYM_SRC)/target/release/libblazesym_c.a | $(OUTPUT)
 	$(call msg,LIB,$@)
-	$(Q)cp $(LIBBLAZESYM_SRC)/target/release/blazesym.h $@
+	$(Q)cp $(LIBBLAZESYM_SRC)/capi/include/blazesym.h $@

 # Build BPF code
 $(OUTPUT)/%.bpf.o: %.bpf.c $(LIBBPF_OBJ) $(wildcard %.h) $(VMLINUX) | $(OUTPUT) $(BPFTOOL)
@@ -137,5 +138,10 @@ $(APPS): %: $(OUTPUT)/%.o $(LIBBPF_OBJ) | $(OUTPUT)
 # delete failed targets
 .DELETE_ON_ERROR:

+# Test program (no BPF, just plain C)
+test_memleak: test_memleak.c
+	$(call msg,TEST,$@)
+	$(Q)$(CC) $(CFLAGS) $< -o $@
+
 # keep intermediate (.skel.h, .bpf.o, etc) targets
 .SECONDARY:
--- a/src/16-memleak/README.md
+++ b/src/16-memleak/README.md
@@ -419,6 +419,8 @@ Reference: <https://github.com/iovisor/bcc/blob/master/libbpf-tools/memleak.c>

 ## Compile and Run

+This implementation uses the latest **blazesym v0.2.0** library for symbol resolution, which provides improved performance and a modern C API through the `capi` package.
+
 ```console
 $ make
 $ sudo ./memleak
@@ -438,6 +440,35 @@ Tracing outstanding memory allocs...  Hit Ctrl-C to end
 ...
 ```

+## Testing memleak
+
+This repository includes a test program (`test_memleak.c`) that intentionally leaks memory for testing purposes. You can build and run it to verify that memleak correctly detects memory leaks:
+
+```console
+$ make test_memleak
+$ ./test_memleak &
+$ sudo ./memleak -p $(pidof test_memleak)
+using default object: libc.so.6
+using page size: 4096
+tracing kernel: false
+Tracing outstanding memory allocs...  Hit Ctrl-C to end
+[19:31:49] Top 1 stacks with outstanding allocations:
+10240 bytes in 5 allocations from stack
+	0 [<00005a7ea0a34212>] leak_with_loop+0x1f
+	1 [<00005a7ea0a3428e>] main+0x6e
+	2 [<00007b4ea482a1ca>] <null sym>
+	3 [<00007b4ea482a28b>] __libc_start_main+0x8b
+	4 [<00005a7ea0a34105>] _start+0x25
+```
+
+As shown above, memleak successfully:
+- Detected **10240 bytes in 5 allocations** (5 × 2048 bytes) that were leaked
+- Identified the leaking function **leak_with_loop+0x1f** with the correct offset
+- Provided the complete call stack showing the leak originated from `main` function
+- Used the new **blazesym v0.2.0** C API for fast and accurate symbol resolution
+
+The test demonstrates that memleak with the updated blazesym library can effectively trace memory allocations and pinpoint the exact functions responsible for memory leaks.
+
 ## Summary

 Through this eBPF introductory tutorial, you have learned how to write a Memleak eBPF monitoring program to monitor memory leaks in real time. You have also learned about the application of eBPF in memory monitoring, how to write eBPF programs using the BPF API, create and use eBPF maps, and how to use eBPF tools to monitor and analyze memory leak issues. We have provided a detailed example to help you understand the execution flow and principles of eBPF code.
--- a/src/16-memleak/README.zh.md
+++ b/src/16-memleak/README.zh.md
@@ -459,6 +459,8 @@ int attach_uprobes(struct memleak_bpf *skel)

 ## 编译运行

+本实现使用最新的 **blazesym v0.2.0** 库进行符号解析，该库通过 `capi` 包提供了改进的性能和现代化的 C API。
+
 ```console
 $ make
 $ sudo ./memleak 
@@ -478,6 +480,35 @@ Tracing outstanding memory allocs...  Hit Ctrl-C to end
 ...
 ```

+## 测试 memleak
+
+本仓库包含一个测试程序（`test_memleak.c`），该程序会故意泄漏内存以供测试使用。您可以编译并运行它来验证 memleak 是否能正确检测内存泄漏：
+
+```console
+$ make test_memleak
+$ ./test_memleak &
+$ sudo ./memleak -p $(pidof test_memleak)
+using default object: libc.so.6
+using page size: 4096
+tracing kernel: false
+Tracing outstanding memory allocs...  Hit Ctrl-C to end
+[19:31:49] Top 1 stacks with outstanding allocations:
+10240 bytes in 5 allocations from stack
+	0 [<00005a7ea0a34212>] leak_with_loop+0x1f
+	1 [<00005a7ea0a3428e>] main+0x6e
+	2 [<00007b4ea482a1ca>] <null sym>
+	3 [<00007b4ea482a28b>] __libc_start_main+0x8b
+	4 [<00005a7ea0a34105>] _start+0x25
+```
+
+如上所示，memleak 成功地：
+- 检测到 **10240 字节的 5 次分配**（5 × 2048 字节）发生了泄漏
+- 识别出泄漏函数 **leak_with_loop+0x1f** 及其正确的偏移量
+- 提供了完整的调用栈，显示泄漏源自 `main` 函数
+- 使用新的 **blazesym v0.2.0** C API 进行快速准确的符号解析
+
+该测试演示了更新 blazesym 库后的 memleak 可以有效地跟踪内存分配，并精确定位导致内存泄漏的函数。
+
 ## 总结

 通过本篇 eBPF 入门实践教程，您已经学习了如何编写 Memleak eBPF 监控程序，以实时监控程序的内存泄漏。您已经了解了 eBPF 在内存监控方面的应用，学会了使用 BPF API 编写 eBPF 程序，创建和使用 eBPF maps，并且明白了如何用 eBPF 工具监测和分析内存泄漏问题。我们展示了一个详细的例子，帮助您理解 eBPF 代码的运行流程和原理。
--- a/src/16-memleak/memleak.c
+++ b/src/16-memleak/memleak.c
@@ -133,7 +133,7 @@ static int event_notify(int fd, uint64_t event);

 static pid_t fork_sync_exec(const char *command, int fd);

-static void print_stack_frame_by_blazesym(size_t frame, uint64_t addr, const blazesym_csym *sym);
+static void print_stack_frame_by_blazesym(size_t frame, uint64_t addr, const struct blaze_sym *sym);
 static void print_stack_frames_by_blazesym();
 static int print_stack_frames(struct allocation *allocs, size_t nr_allocs, int stack_traces_fd);

@@ -205,8 +205,7 @@ static struct sigaction sig_action = {

 static int child_exec_event_fd = -1;

-static blazesym *symbolizer;
-static sym_src_cfg src_cfg;
+static blaze_symbolizer *symbolizer;
 static void (*print_stack_frames_func)();

 static uint64_t *stack;
@@ -302,14 +301,8 @@ int main(int argc, char *argv[])
 		goto cleanup;
 	}

-	if (env.pid < 0) {
-		src_cfg.src_type = SRC_T_KERNEL;
-		src_cfg.params.kernel.kallsyms = NULL;
-		src_cfg.params.kernel.kernel_image = NULL;
-	} else {
-		src_cfg.src_type = SRC_T_PROCESS;
-		src_cfg.params.process.pid = env.pid;
-	}
+	// Symbolization source is now set up in print_stack_frames_by_blazesym()
+	// based on env.kernel_trace and env.pid

 	// allocate space for storing "allocation" structs
 	if (env.combined_only)
@@ -395,9 +388,9 @@ int main(int argc, char *argv[])
 		}
 	}

-	symbolizer = blazesym_new();
+	symbolizer = blaze_symbolizer_new();
 	if (!symbolizer) {
-		fprintf(stderr, "Failed to load blazesym\n");
+		fprintf(stderr, "Failed to create blazesym symbolizer\n");
 		ret = -ENOMEM;

 		goto cleanup;
@@ -439,7 +432,7 @@ int main(int argc, char *argv[])
 	}

 cleanup:
-	blazesym_free(symbolizer);
+	blaze_symbolizer_free(symbolizer);
 	memleak_bpf__destroy(skel);

 	free(allocs);
@@ -636,54 +629,67 @@ pid_t fork_sync_exec(const char *command, int fd)
 	return pid;
 }

-void print_stack_frame_by_blazesym(size_t frame, uint64_t addr, const blazesym_csym *sym)
+void print_stack_frame_by_blazesym(size_t frame, uint64_t addr, const struct blaze_sym *sym)
 {
-	if (!sym)
+	if (!sym || !sym->name)
 		printf("\t%zu [<%016lx>] <%s>\n", frame, addr, "null sym");
-	else if (sym->path && strlen(sym->path))
-		printf("\t%zu [<%016lx>] %s+0x%lx %s:%ld\n", frame, addr, sym->symbol, addr - sym->start_address, sym->path, sym->line_no);
+	else if (sym->code_info.file && strlen(sym->code_info.file))
+		printf("\t%zu [<%016lx>] %s+0x%zx %s:%u\n", frame, addr, sym->name, sym->offset, sym->code_info.file, sym->code_info.line);
 	else
-		printf("\t%zu [<%016lx>] %s+0x%lx\n", frame, addr, sym->symbol, addr - sym->start_address);
+		printf("\t%zu [<%016lx>] %s+0x%zx\n", frame, addr, sym->name, sym->offset);
 }

 void print_stack_frames_by_blazesym()
 {
-	const blazesym_result *result = blazesym_symbolize(symbolizer, &src_cfg, 1, stack, env.perf_max_stack_depth);
+	const struct blaze_syms *syms;

-	for (size_t j = 0; j < result->size; ++j) {
+	// Choose symbolization source based on kernel_trace flag
+	if (env.kernel_trace) {
+		const struct blaze_symbolize_src_kernel src = {
+			.type_size = sizeof(src),
+			.kallsyms = NULL,
+			.vmlinux = NULL,
+			.debug_syms = false,
+			.reserved = {0},
+		};
+		syms = blaze_symbolize_kernel_abs_addrs(symbolizer, &src, stack, env.perf_max_stack_depth);
+	} else {
+		const struct blaze_symbolize_src_process src = {
+			.type_size = sizeof(src),
+			.pid = env.pid,
+		};
+		syms = blaze_symbolize_process_abs_addrs(symbolizer, &src, stack, env.perf_max_stack_depth);
+	}
+
+	if (!syms) {
+		fprintf(stderr, "Failed to symbolize addresses\n");
+		return;
+	}
+
+	for (size_t j = 0; j < syms->cnt; ++j) {
 		const uint64_t addr = stack[j];

 		if (addr == 0)
 			break;

-		// no symbol found
-		if (!result || j >= result->size || result->entries[j].size == 0) {
-			print_stack_frame_by_blazesym(j, addr, NULL);
+		const struct blaze_sym *sym = &syms->syms[j];

-			continue;
-		}
+		// Print main symbol
+		print_stack_frame_by_blazesym(j, addr, sym);

-		// single symbol found
-		if (result->entries[j].size == 1) {
-			const blazesym_csym *sym = &result->entries[j].syms[0];
-			print_stack_frame_by_blazesym(j, addr, sym);
-
-			continue;
-		}
-
-		// multi symbol found
-		printf("\t%zu [<%016lx>] (%lu entries)\n", j, addr, result->entries[j].size);
-
-		for (size_t k = 0; k < result->entries[j].size; ++k) {
-			const blazesym_csym *sym = &result->entries[j].syms[k];
-			if (sym->path && strlen(sym->path))
-				printf("\t\t%s@0x%lx %s:%ld\n", sym->symbol, sym->start_address, sym->path, sym->line_no);
-			else
-				printf("\t\t%s@0x%lx\n", sym->symbol, sym->start_address);
+		// Print inlined function calls if any
+		if (sym->inlined_cnt > 0) {
+			for (size_t k = 0; k < sym->inlined_cnt; ++k) {
+				const struct blaze_symbolize_inlined_fn *inlined = &sym->inlined[k];
+				if (inlined->code_info.file && strlen(inlined->code_info.file))
+					printf("\t\t[inlined] %s %s:%u\n", inlined->name, inlined->code_info.file, inlined->code_info.line);
+				else
+					printf("\t\t[inlined] %s\n", inlined->name);
+			}
 		}
 	}

-	blazesym_result_free(result);
+	blaze_syms_free(syms);
 }

 int print_stack_frames(struct allocation *allocs, size_t nr_allocs, int stack_traces_fd)
--- a/src/16-memleak/test_memleak.c
+++ b/src/16-memleak/test_memleak.c
@@ -0,0 +1,44 @@
+// SPDX-License-Identifier: (LGPL-2.1 OR BSD-2-Clause)
+// Test program for memleak - intentionally leaks memory for testing
+#include <stdlib.h>
+#include <unistd.h>
+#include <stdio.h>
+
+void leak_small() {
+    malloc(1024);
+}
+
+void leak_large() {
+    malloc(8192);
+}
+
+void leak_with_loop() {
+    for (int i = 0; i < 5; i++) {
+        malloc(2048);
+    }
+}
+
+int main() {
+    printf("Memory leak test starting (PID: %d)\n", getpid());
+    printf("This program intentionally leaks memory for testing memleak\n");
+
+    // Wait a bit for memleak to attach
+    sleep(2);
+
+    // Create various leaks
+    leak_small();
+    sleep(1);
+
+    leak_large();
+    sleep(1);
+
+    leak_with_loop();
+    sleep(1);
+
+    // Keep running so memleak can observe
+    printf("Leaks created, sleeping for 30 seconds...\n");
+    sleep(30);
+
+    printf("Test complete\n");
+    return 0;
+}