add new topics (#132)

* add goroutine text * update * fix funclatency * update doc * update about nginx * update nginx * u[date co * fix compile
2026-02-11 06:05:19 +08:00 · 2024-09-02 06:18:57 +00:00
parent a2d54b1f67
commit 9c2bba3e16
20 changed files with 1004 additions and 246 deletions
--- a/src/33-funclatency/.gitignore
+++ b/src/33-funclatency/.gitignore
@@ -0,0 +1,8 @@
+.vscode
+package.json
+*.o
+*.skel.json
+*.skel.yaml
+package.yaml
+ecli
+funclatency
--- a/src/33-funclatency/Makefile
+++ b/src/33-funclatency/Makefile
@@ -0,0 +1,141 @@
+# SPDX-License-Identifier: (LGPL-2.1 OR BSD-2-Clause)
+OUTPUT := .output
+CLANG ?= clang
+LIBBPF_SRC := $(abspath ../third_party/libbpf/src)
+BPFTOOL_SRC := $(abspath ../third_party/bpftool/src)
+LIBBPF_OBJ := $(abspath $(OUTPUT)/libbpf.a)
+BPFTOOL_OUTPUT ?= $(abspath $(OUTPUT)/bpftool)
+BPFTOOL ?= $(BPFTOOL_OUTPUT)/bootstrap/bpftool
+LIBBLAZESYM_SRC := $(abspath ../third_party/blazesym/)
+LIBBLAZESYM_OBJ := $(abspath $(OUTPUT)/libblazesym.a)
+LIBBLAZESYM_HEADER := $(abspath $(OUTPUT)/blazesym.h)
+ARCH ?= $(shell uname -m | sed 's/x86_64/x86/' \
+			 | sed 's/arm.*/arm/' \
+			 | sed 's/aarch64/arm64/' \
+			 | sed 's/ppc64le/powerpc/' \
+			 | sed 's/mips.*/mips/' \
+			 | sed 's/riscv64/riscv/' \
+			 | sed 's/loongarch64/loongarch/')
+VMLINUX := ../third_party/vmlinux/$(ARCH)/vmlinux.h
+# Use our own libbpf API headers and Linux UAPI headers distributed with
+# libbpf to avoid dependency on system-wide headers, which could be missing or
+# outdated
+INCLUDES := -I$(OUTPUT) -I../third_party/libbpf/include/uapi -I$(dir $(VMLINUX))
+CFLAGS := -g -Wall
+ALL_LDFLAGS := $(LDFLAGS) $(EXTRA_LDFLAGS)
+
+APPS = funclatency
+
+CARGO ?= $(shell which cargo)
+ifeq ($(strip $(CARGO)),)
+BZS_APPS :=
+else
+BZS_APPS := 
+APPS += $(BZS_APPS)
+# Required by libblazesym
+ALL_LDFLAGS += -lrt -ldl -lpthread -lm
+endif
+
+# Get Clang's default includes on this system. We'll explicitly add these dirs
+# to the includes list when compiling with `-target bpf` because otherwise some
+# architecture-specific dirs will be "missing" on some architectures/distros -
+# headers such as asm/types.h, asm/byteorder.h, asm/socket.h, asm/sockios.h,
+# sys/cdefs.h etc. might be missing.
+#
+# Use '-idirafter': Don't interfere with include mechanics except where the
+# build would have failed anyways.
+CLANG_BPF_SYS_INCLUDES ?= $(shell $(CLANG) -v -E - </dev/null 2>&1 \
+	| sed -n '/<...> search starts here:/,/End of search list./{ s| \(/.*\)|-idirafter \1|p }')
+
+ifeq ($(V),1)
+	Q =
+	msg =
+else
+	Q = @
+	msg = @printf '  %-8s %s%s\n'					\
+		      "$(1)"						\
+		      "$(patsubst $(abspath $(OUTPUT))/%,%,$(2))"	\
+		      "$(if $(3), $(3))";
+	MAKEFLAGS += --no-print-directory
+endif
+
+define allow-override
+  $(if $(or $(findstring environment,$(origin $(1))),\
+            $(findstring command line,$(origin $(1)))),,\
+    $(eval $(1) = $(2)))
+endef
+
+$(call allow-override,CC,$(CROSS_COMPILE)cc)
+$(call allow-override,LD,$(CROSS_COMPILE)ld)
+
+.PHONY: all
+all: $(APPS)
+
+.PHONY: clean
+clean:
+	$(call msg,CLEAN)
+	$(Q)rm -rf $(OUTPUT) $(APPS)
+
+$(OUTPUT) $(OUTPUT)/libbpf $(BPFTOOL_OUTPUT):
+	$(call msg,MKDIR,$@)
+	$(Q)mkdir -p $@
+
+# Build libbpf
+$(LIBBPF_OBJ): $(wildcard $(LIBBPF_SRC)/*.[ch] $(LIBBPF_SRC)/Makefile) | $(OUTPUT)/libbpf
+	$(call msg,LIB,$@)
+	$(Q)$(MAKE) -C $(LIBBPF_SRC) BUILD_STATIC_ONLY=1		      \
+		    OBJDIR=$(dir $@)/libbpf DESTDIR=$(dir $@)		      \
+		    INCLUDEDIR= LIBDIR= UAPIDIR=			      \
+		    install
+
+# Build bpftool
+$(BPFTOOL): | $(BPFTOOL_OUTPUT)
+	$(call msg,BPFTOOL,$@)
+	$(Q)$(MAKE) ARCH= CROSS_COMPILE= OUTPUT=$(BPFTOOL_OUTPUT)/ -C $(BPFTOOL_SRC) bootstrap
+
+
+$(LIBBLAZESYM_SRC)/target/release/libblazesym.a::
+	$(Q)cd $(LIBBLAZESYM_SRC) && $(CARGO) build --features=cheader,dont-generate-test-files --release
+
+$(LIBBLAZESYM_OBJ): $(LIBBLAZESYM_SRC)/target/release/libblazesym.a | $(OUTPUT)
+	$(call msg,LIB, $@)
+	$(Q)cp $(LIBBLAZESYM_SRC)/target/release/libblazesym.a $@
+
+$(LIBBLAZESYM_HEADER): $(LIBBLAZESYM_SRC)/target/release/libblazesym.a | $(OUTPUT)
+	$(call msg,LIB,$@)
+	$(Q)cp $(LIBBLAZESYM_SRC)/target/release/blazesym.h $@
+
+# Build BPF code
+$(OUTPUT)/%.bpf.o: %.bpf.c $(LIBBPF_OBJ) $(wildcard %.h) $(VMLINUX) | $(OUTPUT) $(BPFTOOL)
+	$(call msg,BPF,$@)
+	$(Q)$(CLANG) -g -O2 -target bpf -D__TARGET_ARCH_$(ARCH)		      \
+		     $(INCLUDES) $(CLANG_BPF_SYS_INCLUDES)		      \
+		     -c $(filter %.c,$^) -o $(patsubst %.bpf.o,%.tmp.bpf.o,$@)
+	$(Q)$(BPFTOOL) gen object $@ $(patsubst %.bpf.o,%.tmp.bpf.o,$@)
+
+# Generate BPF skeletons
+$(OUTPUT)/%.skel.h: $(OUTPUT)/%.bpf.o | $(OUTPUT) $(BPFTOOL)
+	$(call msg,GEN-SKEL,$@)
+	$(Q)$(BPFTOOL) gen skeleton $< > $@
+
+# Build user-space code
+$(patsubst %,$(OUTPUT)/%.o,$(APPS)): %.o: %.skel.h
+
+$(OUTPUT)/%.o: %.c $(wildcard %.h) | $(OUTPUT)
+	$(call msg,CC,$@)
+	$(Q)$(CC) $(CFLAGS) $(INCLUDES) -c $(filter %.c,$^) -o $@
+
+$(patsubst %,$(OUTPUT)/%.o,$(BZS_APPS)): $(LIBBLAZESYM_HEADER)
+
+$(BZS_APPS): $(LIBBLAZESYM_OBJ)
+
+# Build application binary
+$(APPS): %: $(OUTPUT)/%.o $(LIBBPF_OBJ) | $(OUTPUT)
+	$(call msg,BINARY,$@)
+	$(Q)$(CC) $(CFLAGS) $^ $(ALL_LDFLAGS) -lelf -lz -o $@
+
+# delete failed targets
+.DELETE_ON_ERROR:
+
+# keep intermediate (.skel.h, .bpf.o, etc) targets
+.SECONDARY:
--- a/src/33-funclatency/README.md
+++ b/src/33-funclatency/README.md
@@ -0,0 +1,189 @@
+# 使用 eBPF 测量函数延迟
+
+在现代软件系统中，了解函数的性能特性，尤其是那些对应用程序运行至关重要的函数的性能特性，是至关重要的。性能分析中的一个关键指标是**函数延迟**，即函数从开始到完成所花费的时间。通过分析函数延迟，开发人员可以识别瓶颈、优化性能，并确保系统在各种条件下高效运行。
+
+本文将深入探讨如何使用 eBPF 这一强大的工具来测量函数延迟，并展示如何在内核和用户空间中进行跟踪和监控。
+
+## 什么是 eBPF？
+
+eBPF（扩展伯克利包过滤器）是一项革命性的技术，它允许开发人员编写小型程序在 Linux 内核中运行。eBPF 最初是为数据包过滤设计的，但它已经发展成为一个多功能工具，用于跟踪、监控和分析系统行为。通过 eBPF，您几乎可以对 Linux 内核或用户空间的任何部分进行插桩，从而收集性能数据、执行安全策略，甚至实时调试系统——这一切都无需修改内核源码或重启系统。
+
+eBPF 程序在内核的沙盒环境中执行，确保了安全性和稳定性。这些程序可以附加到内核中的各种钩子上，如系统调用、网络事件和跟踪点，甚至可以通过 uprobes（用户级探针）附加到用户空间的函数。eBPF 程序收集的数据可以导出到用户空间进行分析，使其成为系统可观测性的重要工具。内核模式 eBPF 运行时的 `Uprobe` 可能会带来较大的性能开销。在这种情况下，你也可以考虑使用用户模式的 eBPF 运行时，例如 [bpftime](https://github.com/eunomia-bpf/bpftime)。
+
+## 为什么函数延迟很重要？
+
+函数延迟是内核和用户空间应用程序性能分析中的一个关键指标。它提供了关于特定函数执行时间的洞察，这对以下方面至关重要：
+
+- **识别性能瓶颈**：高函数延迟可能表明代码中存在需要优化的低效或问题。
+- **确保系统响应能力**：在实时系统或对延迟敏感的应用程序中，理解和最小化函数延迟对于保持响应能力至关重要。
+- **性能分析和基准测试**：通过测量各种函数的延迟，开发人员可以对系统进行基准测试，并比较不同实现或配置的性能。
+- **调试和诊断**：当系统表现出意外行为或性能下降时，测量函数延迟可以帮助定位问题的根源。
+
+内核空间（如系统调用、文件操作）和用户空间（如库函数）中的函数都可以进行延迟分析，从而提供系统性能的全面视图。
+
+## 用于函数延迟的 eBPF 内核代码
+
+以下是一个设计用于测量函数延迟的 eBPF 程序，它通过挂钩函数的入口和出口点来实现。该程序使用 kprobes 和 kretprobes（用于内核函数）或 uprobes 和 uretprobes（用于用户空间函数）来捕获函数执行的开始和结束时间。
+
+```c
+// SPDX-License-Identifier: GPL-2.0
+/* Copyright (c) 2021 Google LLC. */
+#include "vmlinux.h"
+#include <bpf/bpf_core_read.h>
+#include <bpf/bpf_helpers.h>
+#include <bpf/bpf_tracing.h>
+#include "funclatency.h"
+#include "bits.bpf.h"
+
+const volatile pid_t targ_tgid = 0;
+const volatile int units = 0;
+
+/* key: pid.  value: start time */
+struct {
+    __uint(type, BPF_MAP_TYPE_HASH);
+    __uint(max_entries, MAX_PIDS);
+    __type(key, u32);
+    __type(value, u64);
+} starts SEC(".maps");
+
+__u32 hist[MAX_SLOTS] = {};
+
+static void entry(void)
+{
+    u64 id = bpf_get_current_pid_tgid();
+    u32 tgid = id >> 32;
+    u32 pid = id;
+    u64 nsec;
+
+    if (targ_tgid && targ_tgid != tgid)
+        return;
+    nsec = bpf_ktime_get_ns();
+    bpf_map_update_elem(&starts, &pid, &nsec, BPF_ANY);
+}
+
+SEC("kprobe/dummy_kprobe")
+int BPF_KPROBE(dummy_kprobe)
+{
+    entry();
+    return 0;
+}
+
+static void exit(void)
+{
+    u64 *start;
+    u64 nsec = bpf_ktime_get_ns();
+    u64 id = bpf_get_current_pid_tgid();
+    u32 pid = id;
+    u64 slot, delta;
+
+    start = bpf_map_lookup_elem(&starts, &pid);
+    if (!start)
+        return;
+
+    delta = nsec - *start;
+
+    switch (units) {
+    case USEC:
+        delta /= 1000;
+        break;
+    case MSEC:
+        delta /= 1000000;
+        break;
+    }
+
+    slot = log2l(delta);
+    if (slot >= MAX_SLOTS)
+        slot = MAX_SLOTS - 1;
+    __sync_fetch_and_add(&hist[slot], 1);
+}
+
+SEC("kretprobe/dummy_kretprobe")
+int BPF_KRETPROBE(dummy_kretprobe)
+{
+    exit();
+    return 0;
+}
+
+char LICENSE[] SEC("license") = "GPL";
+```
+
+### 代码解释
+
+1. **头文件**：代码首先包含了必要的头文件，如 `vmlinux.h`（提供内核定义）和 `bpf_helpers.h`（提供 eBPF 程序的辅助函数）。
+
+2. **全局变量**：`targ_tgid` 是目标进程 ID（或线程组 ID），`units` 确定延迟测量的时间单位（如微秒或毫秒）。
+
+3. **BPF 映射**：定义了一个哈希映射（`starts`），用于存储每个进程 ID 的函数执行开始时间。另一个数组（`hist`）用于存储延迟分布。
+
+4. **入口函数**：`entry()` 函数在函数进入时捕获当前时间戳，并将其存储在以进程 ID 为键的 `starts` 映射中。
+
+5. **出口函数**：`exit()` 函数通过将存储的开始时间与当前时间相减来计算延迟。然后将结果分类到直方图槽中，并增加该槽的计数以记录该延迟范围的发生次数。
+
+6. **探针**：`kprobe` 和 `kretprobe` 用于附加到函数的入口和出口点。这些探针触发 `entry()` 和 `exit()` 函数来测量延迟。
+
+7. **许可证**：该程序根据 GPL 许可证发布，以确保符合内核的许可要求。
+
+## 运行函数延迟工具
+
+### 用户空间函数延迟
+
+要跟踪用户空间函数（例如 `libc` 库中的 `read` 函数）的延迟，可以运行以下命令：
+
+```console
+# ./funclatency /usr/lib/x86_64-linux-gnu/libc.so.6:read    
+tracing /usr/lib/x86_64-linux-gnu/libc.so.6:read...
+tracing func read in /usr/lib/x86_64-linux-gnu/libc.so.6...
+Tracing /usr/lib/x86_64-linux-gnu/libc.so.6:read.  Hit Ctrl-C to exit
+^C
+     nsec                : count    distribution
+         0 -> 1          : 0        |                                        |
+         2 -> 3          : 0        |                                        |
+         4 -> 7          : 0        |                                        |
+         8 -> 15         : 0        |                                        |
+        16 -> 31         : 0        |                                        |
+        32 -> 63         : 0        |                                        |
+       128 -> 255        : 0        |                                        |
+       512 -> 1023       : 0        |                                        |
+      65536 -> 131071     : 651      |****************************************+|
+    131072 -> 262143     : 107      |******                                  |
+    262144 -> 524287     : 36       |**                                      |
+    524288 -> 1048575    : 8        |                                        |
+   8388608 -> 16777215   : 2        |                                        |
+Exiting trace of /usr/lib/x86_64-linux-gnu/libc.so.6:read
+```
+
+### 内核空间函数延迟
+
+要跟踪内核空间函数（例如 `vfs_read`）的延迟，可以运行以下命令：
+
+```console
+# sudo ./funclatency -u vfs_read
+Tracing vfs_read.  Hit Ctrl-C to exit
+^C
+     usec                : count    distribution
+         0 -> 1          : 0        |                                        |
+         8 -> 15         : 0        |                                        |
+        16 -> 31         : 3397     |****************************************|
+        32 -> 63         : 2175     |*************************               |
+        64 -> 127        : 184      |**                                      |
+       1024 -> 2047       : 0        |                                        |
+       4096 -> 8191       : 5        |                                        |
+   2097152 -> 
+
+4194303    : 2        |                                        |
+Exiting trace of vfs_read
+```
+
+这些命令会跟踪指定函数（无论是在用户空间还是内核空间）的执行，并打印出观察到的延迟的直方图，显示函数执行时间的分布。
+
+<https://github.com/eunomia-bpf/bpf-developer-tutorial/blob/main/src/33-funclatency>
+
+## 结论
+
+使用 eBPF 测量函数延迟可以深入了解用户空间和内核空间代码的性能。通过了解函数延迟，开发人员可以识别性能瓶颈、提高系统响应能力，并确保其应用程序的顺畅运行。
+
+本文介绍了使用 eBPF 跟踪函数延迟的基本知识，包括实现该跟踪功能的 eBPF 内核代码概述。文中提供的示例展示了如何运行工具以跟踪用户空间和内核空间函数的延迟。
+
+如果您有兴趣了解更多关于 eBPF 的知识，包括更多高级示例和教程，请访问我们的[教程代码库](https://github.com/eunomia-bpf/bpf-developer-tutorial)或我们的网站 [Eunomia](https://eunomia.dev/tutorials/)。
+
+如果您正在寻找一个用于函数延迟测量的生产就绪工具，您可能想查看 BCC 仓库中的完整实现：[BCC 仓库](https://github.com/iovisor/bcc/blob/master/libbpf-tools/funclatency.c)。
--- a/src/33-funclatency/README_en.md
+++ b/src/33-funclatency/README_en.md
@@ -0,0 +1,189 @@
+# Measuring Function Latency with eBPF
+
+In modern software systems, understanding the performance characteristics of functions—especially those critical to the operation of your application—is paramount. One key metric in performance analysis is **function latency**, which is the time taken by a function to execute from start to finish. By analyzing function latency, developers can identify bottlenecks, optimize performance, and ensure that their systems operate efficiently under various conditions.
+
+This blog post will dive into how to measure function latency using eBPF, an incredibly powerful tool for tracing and monitoring both kernel and user-space programs.
+
+## What is eBPF?
+
+eBPF (Extended Berkeley Packet Filter) is a revolutionary technology that allows developers to write small programs that run in the Linux kernel. Originally designed for packet filtering, eBPF has evolved into a versatile tool for tracing, monitoring, and profiling system behavior. With eBPF, you can instrument almost any part of the Linux kernel or user-space programs to collect performance data, enforce security policies, or even debug systems in real time—all without the need to modify the kernel source code or restart the system.
+
+eBPF programs are executed in a sandboxed environment within the kernel, ensuring safety and stability. These programs can attach to various hooks within the kernel, such as system calls, network events, and tracepoints, or even user-space functions using uprobes (user-level probes). The data collected by eBPF programs can then be exported to user space for analysis, making it an invaluable tool for system observability. `Uprobe` in kernel mode eBPF runtime may also cause relatively large performance overhead. In this case, you can also consider using user mode eBPF runtime, such as [bpftime](https://github.com/eunomia-bpf/bpftime).
+
+## Why is Function Latency Important?
+
+Function latency is a critical metric in performance analysis for both kernel and user-space applications. It provides insights into how long a particular function takes to execute, which is crucial for:
+
+- **Identifying Performance Bottlenecks**: High function latency may indicate inefficiencies or issues within the code that need optimization.
+- **Ensuring System Responsiveness**: In real-time systems or latency-sensitive applications, understanding and minimizing function latency is essential to maintain responsiveness.
+- **Profiling and Benchmarking**: By measuring the latency of various functions, developers can benchmark their systems and compare the performance of different implementations or configurations.
+- **Debugging and Diagnostics**: When a system exhibits unexpected behavior or performance degradation, measuring function latency can help pinpoint the source of the problem.
+
+Both kernel-space (e.g., system calls, file operations) and user-space (e.g., library functions) functions can be profiled for latency, providing a comprehensive view of system performance.
+
+## eBPF Kernel Code for Function Latency
+
+Below is an eBPF program designed to measure the latency of a function by hooking into its entry and exit points. The program uses kprobes and kretprobes (for kernel functions) or uprobes and uretprobes (for user-space functions) to capture the start and end times of the function execution.
+
+```c
+// SPDX-License-Identifier: GPL-2.0
+/* Copyright (c) 2021 Google LLC. */
+#include "vmlinux.h"
+#include <bpf/bpf_core_read.h>
+#include <bpf/bpf_helpers.h>
+#include <bpf/bpf_tracing.h>
+#include "funclatency.h"
+#include "bits.bpf.h"
+
+const volatile pid_t targ_tgid = 0;
+const volatile int units = 0;
+
+/* key: pid.  value: start time */
+struct {
+    __uint(type, BPF_MAP_TYPE_HASH);
+    __uint(max_entries, MAX_PIDS);
+    __type(key, u32);
+    __type(value, u64);
+} starts SEC(".maps");
+
+__u32 hist[MAX_SLOTS] = {};
+
+static void entry(void)
+{
+    u64 id = bpf_get_current_pid_tgid();
+    u32 tgid = id >> 32;
+    u32 pid = id;
+    u64 nsec;
+
+    if (targ_tgid && targ_tgid != tgid)
+        return;
+    nsec = bpf_ktime_get_ns();
+    bpf_map_update_elem(&starts, &pid, &nsec, BPF_ANY);
+}
+
+SEC("kprobe/dummy_kprobe")
+int BPF_KPROBE(dummy_kprobe)
+{
+    entry();
+    return 0;
+}
+
+static void exit(void)
+{
+    u64 *start;
+    u64 nsec = bpf_ktime_get_ns();
+    u64 id = bpf_get_current_pid_tgid();
+    u32 pid = id;
+    u64 slot, delta;
+
+    start = bpf_map_lookup_elem(&starts, &pid);
+    if (!start)
+        return;
+
+    delta = nsec - *start;
+
+    switch (units) {
+    case USEC:
+        delta /= 1000;
+        break;
+    case MSEC:
+        delta /= 1000000;
+        break;
+    }
+
+    slot = log2l(delta);
+    if (slot >= MAX_SLOTS)
+        slot = MAX_SLOTS - 1;
+    __sync_fetch_and_add(&hist[slot], 1);
+}
+
+SEC("kretprobe/dummy_kretprobe")
+int BPF_KRETPROBE(dummy_kretprobe)
+{
+    exit();
+    return 0;
+}
+
+char LICENSE[] SEC("license") = "GPL";
+```
+
+### Explanation of the Code
+
+1. **Header Files**: The code begins by including the necessary headers like `vmlinux.h` (which provides kernel definitions) and `bpf_helpers.h` (which offers helper functions for eBPF programs).
+
+2. **Global Variables**: `targ_tgid` is a target process ID (or thread group ID), and `units` determines the time unit for latency measurement (e.g., microseconds or milliseconds).
+
+3. **BPF Maps**: A hash map (`starts`) is defined to store the start time of function executions for each process ID. Another array (`hist`) is used to store the latency distribution.
+
+4. **Entry Function**: The `entry()` function captures the current timestamp when the function is entered and stores it in the `starts` map keyed by the process ID.
+
+5. **Exit Function**: The `exit()` function calculates the latency by subtracting the stored start time from the current time. The result is then categorized into a histogram slot, which is incremented to record the occurrence of that latency range.
+
+6. **Probes**: The `kprobe` and `kretprobe` are used to attach to the entry and exit points of the function, respectively. These probes trigger the `entry()` and `exit()` functions to measure the latency.
+
+7. **License**: The program is licensed under GPL to ensure compliance with kernel licensing requirements.
+
+## Running the Function Latency Tool
+
+### User-Space Function Latency
+
+To trace the latency of a user-space function, such as the `read` function in the `libc` library, you can run the following command:
+
+```console
+# ./funclatency /usr/lib/x86_64-linux-gnu/libc.so.6:read    
+tracing /usr/lib/x86_64-linux-gnu/libc.so.6:read...
+tracing func read in /usr/lib/x86_64-linux-gnu/libc.so.6...
+Tracing /usr/lib/x86_64-linux-gnu/libc.so.6:read.  Hit Ctrl-C to exit
+^C
+     nsec                : count    distribution
+         0 -> 1          : 0        |                                        |
+         2 -> 3          : 0        |                                        |
+         4 -> 7          : 0        |                                        |
+         8 -> 15         : 0        |                                        |
+        16 -> 31         : 0        |                                        |
+        32 -> 63         : 0        |                                        |
+       128 -> 255        : 0        |                                        |
+       512 -> 1023       : 0        |                                        |
+      65536 -> 131071     : 651      |****************************************+|
+    131072 -> 262143     : 107      |******                                  |
+    262144 -> 524287     : 36       |**                                      |
+    524288 -> 1048575    : 8        |                                        |
+   8388608 -> 16777215   : 2        |                                        |
+Exiting trace of /usr/lib/x86_64-linux-gnu/libc.so.6:read
+```
+
+### Kernel-Space Function Latency
+
+To trace the latency of a kernel-space function, such as `vfs_read`, run the following command:
+
+```console
+# sudo ./funclatency -u vfs_read
+Tracing vfs_read.  Hit Ctrl-C to exit
+^C
+     usec                : count    distribution
+         0 -> 1          : 0        |                                        |
+         8 -> 15         : 0        |                                        |
+        16 -> 31         : 3397     |****************************************|
+        32 -> 63         : 2175     |*************************               |
+        64 -> 127        : 184      |**                                      |
+       1024 -> 2047       : 0        |                                        |
+       4096 -> 8191       : 5        |                                        |
+   2097152 -> 4194303    : 2        |                                        |
+Exiting trace of vfs_read
+```
+
+These commands trace the execution of the specified function, either in user-space or kernel-space, and print a histogram of the observed latencies, showing the distribution of function execution times.
+
+You can find the source code in <https://github.com/eunomia-bpf/bpf-developer-tutorial/blob/main/src/33-funclatency>
+
+## Conclusion
+
+Measuring function latency with eBPF offers deep insights into the performance of both user-space and kernel-space code. By understanding function latency, developers can identify performance bottlenecks, improve system responsiveness, and ensure the smooth operation of their applications.
+
+This
+
+ blog post covered the basics of using eBPF to trace function latency, including an overview of the eBPF kernel code used to perform the tracing. The examples provided demonstrated how to run the tool to trace both user-space and kernel-space functions.
+
+For those interested in learning more about eBPF, including more advanced examples and tutorials, please visit our [https://github.com/eunomia-bpf/bpf-developer-tutorial](https://github.com/eunomia-bpf/bpf-developer-tutorial) or our website [https://eunomia.dev/tutorials/](https://eunomia.dev/tutorials/).
+
+If you are looking for a production-ready tool for function latency measurement, you might want to check out the full implementation available in the [BCC repository](https://github.com/iovisor/bcc/blob/master/libbpf-tools/funclatency.c).
--- a/src/33-funclatency/bits.bpf.h
+++ b/src/33-funclatency/bits.bpf.h
@@ -0,0 +1,31 @@
+/* SPDX-License-Identifier: (LGPL-2.1 OR BSD-2-Clause) */
+#ifndef __BITS_BPF_H
+#define __BITS_BPF_H
+
+#define READ_ONCE(x) (*(volatile typeof(x) *)&(x))
+#define WRITE_ONCE(x, val) ((*(volatile typeof(x) *)&(x)) = val)
+
+static __always_inline u64 log2(u32 v)
+{
+	u32 shift, r;
+
+	r = (v > 0xFFFF) << 4; v >>= r;
+	shift = (v > 0xFF) << 3; v >>= shift; r |= shift;
+	shift = (v > 0xF) << 2; v >>= shift; r |= shift;
+	shift = (v > 0x3) << 1; v >>= shift; r |= shift;
+	r |= (v >> 1);
+
+	return r;
+}
+
+static __always_inline u64 log2l(u64 v)
+{
+	u32 hi = v >> 32;
+
+	if (hi)
+		return log2(hi) + 32;
+	else
+		return log2(v);
+}
+
+#endif /* __BITS_BPF_H */
--- a/src/33-funclatency/funclatency.bpf.c
+++ b/src/33-funclatency/funclatency.bpf.c
@@ -0,0 +1,79 @@
+// SPDX-License-Identifier: GPL-2.0
+/* Copyright (c) 2021 Google LLC. */
+#include "vmlinux.h"
+#include <bpf/bpf_core_read.h>
+#include <bpf/bpf_helpers.h>
+#include <bpf/bpf_tracing.h>
+#include "funclatency.h"
+#include "bits.bpf.h"
+
+const volatile pid_t targ_tgid = 0;
+const volatile int units = 0;
+
+/* key: pid.  value: start time */
+struct {
+	__uint(type, BPF_MAP_TYPE_HASH);
+	__uint(max_entries, MAX_PIDS);
+	__type(key, u32);
+	__type(value, u64);
+} starts SEC(".maps");
+
+__u32 hist[MAX_SLOTS] = {};
+
+static void entry(void)
+{
+	u64 id = bpf_get_current_pid_tgid();
+	u32 tgid = id >> 32;
+	u32 pid = id;
+	u64 nsec;
+
+	if (targ_tgid && targ_tgid != tgid)
+		return;
+	nsec = bpf_ktime_get_ns();
+	bpf_map_update_elem(&starts, &pid, &nsec, BPF_ANY);
+}
+
+SEC("kprobe/dummy_kprobe")
+int BPF_KPROBE(dummy_kprobe)
+{
+	entry();
+	return 0;
+}
+
+static void exit(void)
+{
+	u64 *start;
+	u64 nsec = bpf_ktime_get_ns();
+	u64 id = bpf_get_current_pid_tgid();
+	u32 pid = id;
+	u64 slot, delta;
+
+	start = bpf_map_lookup_elem(&starts, &pid);
+	if (!start)
+		return;
+
+	delta = nsec - *start;
+
+	switch (units) {
+	case USEC:
+		delta /= 1000;
+		break;
+	case MSEC:
+		delta /= 1000000;
+		break;
+	}
+
+	slot = log2l(delta);
+	if (slot >= MAX_SLOTS)
+		slot = MAX_SLOTS - 1;
+	__sync_fetch_and_add(&hist[slot], 1);
+}
+
+SEC("kretprobe/dummy_kretprobe")
+int BPF_KRETPROBE(dummy_kretprobe)
+{
+	exit();
+	return 0;
+}
+
+char LICENSE[] SEC("license") = "GPL";
--- a/src/33-funclatency/funclatency.c
+++ b/src/33-funclatency/funclatency.c
@@ -0,0 +1,442 @@
+// SPDX-License-Identifier: (LGPL-2.1 OR BSD-2-Clause)
+/* Copyright (c) 2021 Google LLC.
+ *
+ * Based on funclatency from BCC by Brendan Gregg and others
+ * 2021-02-26   Barret Rhoden   Created this.
+ *
+ * TODO:
+ * - support uprobes on libraries without -p PID. (parse ld.so.cache)
+ * - support regexp pattern matching and per-function histograms
+ */
+#include <argp.h>
+#include <errno.h>
+#include <signal.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <time.h>
+#include <unistd.h>
+#include <fcntl.h>
+
+#include <bpf/libbpf.h>
+#include <bpf/bpf.h>
+#include "funclatency.h"
+#include "funclatency.skel.h"
+
+#define warn(...) fprintf(stderr, __VA_ARGS__)
+
+static struct prog_env
+{
+	int units;
+	pid_t pid;
+	unsigned int duration;
+	unsigned int interval;
+	unsigned int iterations;
+	bool timestamp;
+	char *funcname;
+	bool verbose;
+	bool kprobes;
+	bool is_kernel_func;
+} env = {
+	.interval = 99999999,
+	.iterations = 99999999,
+};
+
+const char *argp_program_version = "funclatency 0.1";
+const char *argp_program_bug_address =
+	"https://github.com/iovisor/bcc/tree/master/libbpf-tools";
+static const char args_doc[] = "FUNCTION";
+static const char program_doc[] =
+	"Time functions and print latency as a histogram\n"
+	"\n"
+	"Usage: funclatency [-h] [-m|-u] [-p PID] [-d DURATION] [ -i INTERVAL ]\n"
+	"                   [-T] FUNCTION\n"
+	"       Choices for FUNCTION: FUNCTION         (kprobe)\n"
+	"                             LIBRARY:FUNCTION (uprobe a library in -p PID)\n"
+	"                             :FUNCTION        (uprobe the binary of -p PID)\n"
+	"                             PROGRAM:FUNCTION (uprobe the binary PROGRAM)\n"
+	"\v"
+	"Examples:\n"
+	"  ./funclatency do_sys_open         # time the do_sys_open() kernel function\n"
+	"  ./funclatency -m do_nanosleep     # time do_nanosleep(), in milliseconds\n"
+	"  ./funclatency -u vfs_read         # time vfs_read(), in microseconds\n"
+	"  ./funclatency -p 181 vfs_read     # time process 181 only\n"
+	"  ./funclatency -p 181 /usr/lib/x86_64-linux-gnu/libc.so.6:read       # time the read() C library function\n"
+	"  ./funclatency -p 181 :foo         # time foo() from pid 181's userspace\n"
+	"  ./funclatency -i 2 -d 10 vfs_read # output every 2 seconds, for 10s\n"
+	"  ./funclatency -mTi 5 vfs_read     # output every 5 seconds, with timestamps\n";
+
+static const struct argp_option opts[] = {
+	{"milliseconds", 'm', NULL, 0, "Output in milliseconds"},
+	{"microseconds", 'u', NULL, 0, "Output in microseconds"},
+	{0, 0, 0, 0, ""},
+	{"pid", 'p', "PID", 0, "Process ID to trace"},
+	{0, 0, 0, 0, ""},
+	{"interval", 'i', "INTERVAL", 0, "Summary interval in seconds"},
+	{"duration", 'd', "DURATION", 0, "Duration to trace"},
+	{"timestamp", 'T', NULL, 0, "Print timestamp"},
+	{"verbose", 'v', NULL, 0, "Verbose debug output"},
+	{"kprobes", 'k', NULL, 0, "Use kprobes instead of fentry"},
+	{NULL, 'h', NULL, OPTION_HIDDEN, "Show the full help"},
+	{},
+};
+
+static error_t parse_arg(int key, char *arg, struct argp_state *state)
+{
+	struct prog_env *env = state->input;
+	long duration, interval, pid;
+
+	switch (key)
+	{
+	case 'p':
+		errno = 0;
+		pid = strtol(arg, NULL, 10);
+		if (errno || pid <= 0)
+		{
+			warn("Invalid PID: %s\n", arg);
+			argp_usage(state);
+		}
+		env->pid = pid;
+		break;
+	case 'm':
+		if (env->units != NSEC)
+		{
+			warn("only set one of -m or -u\n");
+			argp_usage(state);
+		}
+		env->units = MSEC;
+		break;
+	case 'u':
+		if (env->units != NSEC)
+		{
+			warn("only set one of -m or -u\n");
+			argp_usage(state);
+		}
+		env->units = USEC;
+		break;
+	case 'd':
+		errno = 0;
+		duration = strtol(arg, NULL, 10);
+		if (errno || duration <= 0)
+		{
+			warn("Invalid duration: %s\n", arg);
+			argp_usage(state);
+		}
+		env->duration = duration;
+		break;
+	case 'i':
+		errno = 0;
+		interval = strtol(arg, NULL, 10);
+		if (errno || interval <= 0)
+		{
+			warn("Invalid interval: %s\n", arg);
+			argp_usage(state);
+		}
+		env->interval = interval;
+		break;
+	case 'T':
+		env->timestamp = true;
+		break;
+	case 'k':
+		env->kprobes = true;
+		break;
+	case 'v':
+		env->verbose = true;
+		break;
+	case 'h':
+		argp_state_help(state, stderr, ARGP_HELP_STD_HELP);
+		break;
+	case ARGP_KEY_ARG:
+		if (env->funcname)
+		{
+			warn("Too many function names: %s\n", arg);
+			argp_usage(state);
+		}
+		env->funcname = arg;
+		break;
+	case ARGP_KEY_END:
+		if (!env->funcname)
+		{
+			warn("Need a function to trace\n");
+			argp_usage(state);
+		}
+		if (env->duration)
+		{
+			if (env->interval > env->duration)
+				env->interval = env->duration;
+			env->iterations = env->duration / env->interval;
+		}
+		break;
+	default:
+		return ARGP_ERR_UNKNOWN;
+	}
+	return 0;
+}
+
+static int libbpf_print_fn(enum libbpf_print_level level, const char *format, va_list args)
+{
+	if (level == LIBBPF_DEBUG && !env.verbose)
+		return 0;
+	return vfprintf(stderr, format, args);
+}
+
+static const char *unit_str(void)
+{
+	switch (env.units)
+	{
+	case NSEC:
+		return "nsec";
+	case USEC:
+		return "usec";
+	case MSEC:
+		return "msec";
+	};
+
+	return "bad units";
+}
+
+static int attach_kprobes(struct funclatency_bpf *obj)
+{
+	obj->links.dummy_kprobe =
+		bpf_program__attach_kprobe(obj->progs.dummy_kprobe, false,
+								   env.funcname);
+	if (!obj->links.dummy_kprobe)
+	{
+		warn("failed to attach kprobe: %d\n", -errno);
+		return -1;
+	}
+
+	obj->links.dummy_kretprobe =
+		bpf_program__attach_kprobe(obj->progs.dummy_kretprobe, true,
+								   env.funcname);
+	if (!obj->links.dummy_kretprobe)
+	{
+		warn("failed to attach kretprobe: %d\n", -errno);
+		return -1;
+	}
+
+	return 0;
+}
+
+static int attach_uprobes(struct funclatency_bpf *obj)
+{
+	char *binary, *function;
+	int ret = -1;
+	long err;
+
+	binary = strdup(env.funcname);
+	if (!binary)
+	{
+		warn("strdup failed");
+		return -1;
+	}
+	printf("tracing %s...\n", binary);
+	function = strchr(binary, ':');
+	if (!function)
+	{
+		warn("Binary should have contained ':' (internal bug!)\n");
+		return -1;
+	}
+	*function = '\0';
+	function++;
+	printf("tracing func %s in %s...\n", function, binary);
+
+	LIBBPF_OPTS(bpf_uprobe_opts, opts);
+	opts.func_name = function;
+	opts.retprobe = false;
+
+	obj->links.dummy_kprobe =
+		bpf_program__attach_uprobe_opts(obj->progs.dummy_kprobe, 
+								   env.pid ?: -1, binary, 0, &opts);
+	if (!obj->links.dummy_kprobe)
+	{
+		err = -errno;
+		warn("Failed to attach uprobe: %ld\n", err);
+		goto out_binary;
+	}
+
+	opts.retprobe = true;
+
+	obj->links.dummy_kretprobe =
+		bpf_program__attach_uprobe_opts(obj->progs.dummy_kretprobe, 
+								   env.pid ?: -1, binary, 0, &opts);
+	if (!obj->links.dummy_kretprobe)
+	{
+		err = -errno;
+		warn("Failed to attach uretprobe: %ld\n", err);
+		goto out_binary;
+	}
+
+	ret = 0;
+
+out_binary:
+	free(binary);
+
+	return ret;
+}
+
+static volatile bool exiting;
+
+static void sig_hand(int signr)
+{
+	exiting = true;
+}
+
+static struct sigaction sigact = {.sa_handler = sig_hand};
+
+#ifndef min
+#define min(a,b) ((a) < (b) ? (a) : (b))
+#endif
+
+static void print_stars(unsigned int val, unsigned int val_max, int width)
+{
+	int num_stars, num_spaces, i;
+	bool need_plus;
+
+	num_stars = min(val, val_max) * width / val_max;
+	num_spaces = width - num_stars;
+	need_plus = val > val_max;
+
+	for (i = 0; i < num_stars; i++)
+		printf("*");
+	for (i = 0; i < num_spaces; i++)
+		printf(" ");
+	if (need_plus)
+		printf("+");
+}
+
+void print_log2_hist(unsigned int *vals, int vals_size, const char *val_type)
+{
+	int stars_max = 40, idx_max = -1;
+	unsigned int val, val_max = 0;
+	unsigned long long low, high;
+	int stars, width, i;
+
+	for (i = 0; i < vals_size; i++)
+	{
+		val = vals[i];
+		if (val > 0)
+			idx_max = i;
+		if (val > val_max)
+			val_max = val;
+	}
+
+	if (idx_max < 0)
+		return;
+
+	printf("%*s%-*s : count    distribution\n", idx_max <= 32 ? 5 : 15, "",
+		   idx_max <= 32 ? 19 : 29, val_type);
+
+	if (idx_max <= 32)
+		stars = stars_max;
+	else
+		stars = stars_max / 2;
+
+	for (i = 0; i <= idx_max; i++)
+	{
+		low = (1ULL << (i + 1)) >> 1;
+		high = (1ULL << (i + 1)) - 1;
+		if (low == high)
+			low -= 1;
+		val = vals[i];
+		width = idx_max <= 32 ? 10 : 20;
+		printf("%*lld -> %-*lld : %-8d |", width, low, width, high, val);
+		print_stars(val, val_max, stars);
+		printf("|\n");
+	}
+}
+
+int main(int argc, char **argv)
+{
+	LIBBPF_OPTS(bpf_object_open_opts, open_opts);
+	static const struct argp argp = {
+		.options = opts,
+		.parser = parse_arg,
+		.args_doc = args_doc,
+		.doc = program_doc,
+	};
+	struct funclatency_bpf *obj;
+	int i, err;
+	struct tm *tm;
+	char ts[32];
+	time_t t;
+	int cgfd = -1;
+
+	err = argp_parse(&argp, argc, argv, 0, NULL, &env);
+	if (err)
+		return err;
+
+	env.is_kernel_func = !strchr(env.funcname, ':');
+
+	sigaction(SIGINT, &sigact, 0);
+
+	libbpf_set_print(libbpf_print_fn);
+
+	obj = funclatency_bpf__open_opts(&open_opts);
+	if (!obj)
+	{
+		warn("failed to open BPF object\n");
+		return 1;
+	}
+
+	obj->rodata->units = env.units;
+	obj->rodata->targ_tgid = env.pid;
+
+	err = funclatency_bpf__load(obj);
+	if (err)
+	{
+		warn("failed to load BPF object\n");
+		return 1;
+	}
+
+	if (!obj->bss)
+	{
+		warn("Memory-mapping BPF maps is supported starting from Linux 5.7, please upgrade.\n");
+		goto cleanup;
+	}
+
+	if (env.is_kernel_func)
+		err = attach_kprobes(obj);
+	else
+		err = attach_uprobes(obj);
+	if (err)
+		goto cleanup;
+
+	err = funclatency_bpf__attach(obj);
+	if (err)
+	{
+		fprintf(stderr, "failed to attach BPF programs: %s\n",
+				strerror(-err));
+		goto cleanup;
+	}
+
+	printf("Tracing %s.  Hit Ctrl-C to exit\n", env.funcname);
+
+	for (i = 0; i < env.iterations && !exiting; i++)
+	{
+		sleep(env.interval);
+
+		printf("\n");
+		if (env.timestamp)
+		{
+			time(&t);
+			tm = localtime(&t);
+			strftime(ts, sizeof(ts), "%H:%M:%S", tm);
+			printf("%-8s\n", ts);
+		}
+
+		print_log2_hist(obj->bss->hist, MAX_SLOTS, unit_str());
+
+		/* Cleanup histograms for interval output */
+		memset(obj->bss->hist, 0, sizeof(obj->bss->hist));
+	}
+
+	printf("Exiting trace of %s\n", env.funcname);
+
+cleanup:
+	funclatency_bpf__destroy(obj);
+	if (cgfd > 0)
+		close(cgfd);
+
+	return err != 0;
+}
--- a/src/33-funclatency/funclatency.h
+++ b/src/33-funclatency/funclatency.h
@@ -0,0 +1,18 @@
+/* SPDX-License-Identifier: (LGPL-2.1 OR BSD-2-Clause) */
+/* Copyright (c) 2020 Facebook */
+#ifndef __BOOTSTRAP_H
+#define __BOOTSTRAP_H
+
+/* SPDX-License-Identifier: (LGPL-2.1 OR BSD-2-Clause) */
+#pragma once
+
+#define MAX_PIDS 102400
+#define MAX_SLOTS 25
+
+enum units {
+	NSEC,
+	USEC,
+	MSEC,
+};
+
+#endif /* __BOOTSTRAP_H */