From db659aaa1c40ebd6d790e4d570b424113103a549 Mon Sep 17 00:00:00 2001 From: officeyutong <9004058+Officeyutong@users.noreply.github.com> Date: Sat, 21 Jan 2023 20:10:54 +0800 Subject: [PATCH] update tutorial 13 --- 13-tcpconnlat/README.md | 4 + .../libbpf-bootstrap/tcpconnlat.bpf.c | 131 ++++++++ 13-tcpconnlat/libbpf-bootstrap/tcpconnlat.c | 298 ++++++++++++++++++ 13-tcpconnlat/libbpf-bootstrap/tcpconnlat.h | 31 ++ 13-tcpconnlat/tcpconnlat-libbpf-bootstrap.md | 27 ++ 5 files changed, 491 insertions(+) create mode 100644 13-tcpconnlat/libbpf-bootstrap/tcpconnlat.bpf.c create mode 100644 13-tcpconnlat/libbpf-bootstrap/tcpconnlat.c create mode 100644 13-tcpconnlat/libbpf-bootstrap/tcpconnlat.h create mode 100644 13-tcpconnlat/tcpconnlat-libbpf-bootstrap.md diff --git a/13-tcpconnlat/README.md b/13-tcpconnlat/README.md index 6f78269..989e6b6 100644 --- a/13-tcpconnlat/README.md +++ b/13-tcpconnlat/README.md @@ -1,5 +1,9 @@ ## eBPF 入门实践教程： +## 备注 + +对于使用 `libbpf-bootstrap` 的开发，具体见 [tcpconnlat-libbpf-bootstrap.md](tcpconnlat-libbpf-bootstrap.md) + ## origin origin from: diff --git a/13-tcpconnlat/libbpf-bootstrap/tcpconnlat.bpf.c b/13-tcpconnlat/libbpf-bootstrap/tcpconnlat.bpf.c new file mode 100644 index 0000000..4b334f7 --- /dev/null +++ b/13-tcpconnlat/libbpf-bootstrap/tcpconnlat.bpf.c @@ -0,0 +1,131 @@ +// SPDX-License-Identifier: GPL-2.0 +// Copyright (c) 2020 Wenbo Zhang +#include +#include +#include +#include +#include "tcpconnlat.h" + +#define AF_INET 2 +#define AF_INET6 10 + +const volatile __u64 targ_min_us = 0; +const volatile pid_t targ_tgid = 0; + +struct piddata { + char comm[TASK_COMM_LEN]; + u64 ts; + u32 tgid; +}; + +struct { + __uint(type, BPF_MAP_TYPE_HASH); + __uint(max_entries, 4096); + __type(key, struct sock *); + __type(value, struct piddata); +} start SEC(".maps"); + +struct { + __uint(type, BPF_MAP_TYPE_PERF_EVENT_ARRAY); + __uint(key_size, sizeof(u32)); + __uint(value_size, sizeof(u32)); +} events SEC(".maps"); + +static int trace_connect(struct sock *sk) +{ + u32 tgid = bpf_get_current_pid_tgid() >> 32; + struct piddata piddata = {}; + + if (targ_tgid && targ_tgid != tgid) + return 0; + + bpf_get_current_comm(&piddata.comm, sizeof(piddata.comm)); + piddata.ts = bpf_ktime_get_ns(); + piddata.tgid = tgid; + bpf_map_update_elem(&start, &sk, &piddata, 0); + return 0; +} + +static int handle_tcp_rcv_state_process(void *ctx, struct sock *sk) +{ + struct piddata *piddatap; + struct event event = {}; + s64 delta; + u64 ts; + + if (BPF_CORE_READ(sk, __sk_common.skc_state) != TCP_SYN_SENT) + return 0; + + piddatap = bpf_map_lookup_elem(&start, &sk); + if (!piddatap) + return 0; + + ts = bpf_ktime_get_ns(); + delta = (s64)(ts - piddatap->ts); + if (delta < 0) + goto cleanup; + + event.delta_us = delta / 1000U; + if (targ_min_us && event.delta_us < targ_min_us) + goto cleanup; + __builtin_memcpy(&event.comm, piddatap->comm, + sizeof(event.comm)); + event.ts_us = ts / 1000; + event.tgid = piddatap->tgid; + event.lport = BPF_CORE_READ(sk, __sk_common.skc_num); + event.dport = BPF_CORE_READ(sk, __sk_common.skc_dport); + event.af = BPF_CORE_READ(sk, __sk_common.skc_family); + if (event.af == AF_INET) { + event.saddr_v4 = BPF_CORE_READ(sk, __sk_common.skc_rcv_saddr); + event.daddr_v4 = BPF_CORE_READ(sk, __sk_common.skc_daddr); + } else { + BPF_CORE_READ_INTO(&event.saddr_v6, sk, + __sk_common.skc_v6_rcv_saddr.in6_u.u6_addr32); + BPF_CORE_READ_INTO(&event.daddr_v6, sk, + __sk_common.skc_v6_daddr.in6_u.u6_addr32); + } + bpf_perf_event_output(ctx, &events, BPF_F_CURRENT_CPU, + &event, sizeof(event)); + +cleanup: + bpf_map_delete_elem(&start, &sk); + return 0; +} + +SEC("kprobe/tcp_v4_connect") +int BPF_KPROBE(tcp_v4_connect, struct sock *sk) +{ + return trace_connect(sk); +} + +SEC("kprobe/tcp_v6_connect") +int BPF_KPROBE(tcp_v6_connect, struct sock *sk) +{ + return trace_connect(sk); +} + +SEC("kprobe/tcp_rcv_state_process") +int BPF_KPROBE(tcp_rcv_state_process, struct sock *sk) +{ + return handle_tcp_rcv_state_process(ctx, sk); +} + +SEC("fentry/tcp_v4_connect") +int BPF_PROG(fentry_tcp_v4_connect, struct sock *sk) +{ + return trace_connect(sk); +} + +SEC("fentry/tcp_v6_connect") +int BPF_PROG(fentry_tcp_v6_connect, struct sock *sk) +{ + return trace_connect(sk); +} + +SEC("fentry/tcp_rcv_state_process") +int BPF_PROG(fentry_tcp_rcv_state_process, struct sock *sk) +{ + return handle_tcp_rcv_state_process(ctx, sk); +} + +char LICENSE[] SEC("license") = "GPL"; \ No newline at end of file diff --git a/13-tcpconnlat/libbpf-bootstrap/tcpconnlat.c b/13-tcpconnlat/libbpf-bootstrap/tcpconnlat.c new file mode 100644 index 0000000..8fa49a5 --- /dev/null +++ b/13-tcpconnlat/libbpf-bootstrap/tcpconnlat.c @@ -0,0 +1,298 @@ +// SPDX-License-Identifier: (LGPL-2.1 OR BSD-2-Clause) +// Copyright (c) 2020 Wenbo Zhang +// +// Based on tcpconnlat(8) from BCC by Brendan Gregg. +// 11-Jul-2020 Wenbo Zhang Created this. +#include "tcpconnlat.h" +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "tcpconnlat.skel.h" +// #include "trace_helpers.h" + +#define PERF_BUFFER_PAGES 16 +#define PERF_POLL_TIMEOUT_MS 100 + +static volatile sig_atomic_t exiting = 0; + +static struct env { + __u64 min_us; + pid_t pid; + bool timestamp; + bool lport; + bool verbose; +} env; + +const char* argp_program_version = "tcpconnlat 0.1"; +const char* argp_program_bug_address = + "https://github.com/iovisor/bcc/tree/master/libbpf-tools"; +const char argp_program_doc[] = + "\nTrace TCP connects and show connection latency.\n" + "\n" + "USAGE: tcpconnlat [--help] [-t] [-p PID] [-L]\n" + "\n" + "EXAMPLES:\n" + " tcpconnlat # summarize on-CPU time as a histogram\n" + " tcpconnlat 1 # trace connection latency slower than 1 ms\n" + " tcpconnlat 0.1 # trace connection latency slower than 100 " + "us\n" + " tcpconnlat -t # 1s summaries, milliseconds, and timestamps\n" + " tcpconnlat -p 185 # trace PID 185 only\n" + " tcpconnlat -L # include LPORT while printing outputs\n"; + +static const struct argp_option opts[] = { + {"timestamp", 't', NULL, 0, "Include timestamp on output"}, + {"pid", 'p', "PID", 0, "Trace this PID only"}, + {"lport", 'L', NULL, 0, "Include LPORT on output"}, + {"verbose", 'v', NULL, 0, "Verbose debug output"}, + {NULL, 'h', NULL, OPTION_HIDDEN, "Show the full help"}, + {}, +}; + +static error_t parse_arg(int key, char* arg, struct argp_state* state) { + static int pos_args; + + switch (key) { + case 'h': + argp_state_help(state, stderr, ARGP_HELP_STD_HELP); + break; + case 'v': + env.verbose = true; + break; + case 'p': + errno = 0; + env.pid = strtol(arg, NULL, 10); + if (errno) { + fprintf(stderr, "invalid PID: %s\n", arg); + argp_usage(state); + } + break; + case 't': + env.timestamp = true; + break; + case 'L': + env.lport = true; + break; + case ARGP_KEY_ARG: + if (pos_args++) { + fprintf(stderr, "Unrecognized positional argument: %s\n", arg); + argp_usage(state); + } + errno = 0; + env.min_us = strtod(arg, NULL) * 1000; + if (errno || env.min_us <= 0) { + fprintf(stderr, "Invalid delay (in us): %s\n", arg); + argp_usage(state); + } + break; + default: + return ARGP_ERR_UNKNOWN; + } + return 0; +} + +static int libbpf_print_fn(enum libbpf_print_level level, + const char* format, + va_list args) { + if (level == LIBBPF_DEBUG && !env.verbose) + return 0; + return vfprintf(stderr, format, args); +} + +static void sig_int(int signo) { + exiting = 1; +} + +void handle_event(void* ctx, int cpu, void* data, __u32 data_sz) { + const struct event* e = data; + char src[INET6_ADDRSTRLEN]; + char dst[INET6_ADDRSTRLEN]; + union { + struct in_addr x4; + struct in6_addr x6; + } s, d; + static __u64 start_ts; + + if (env.timestamp) { + if (start_ts == 0) + start_ts = e->ts_us; + printf("%-9.3f ", (e->ts_us - start_ts) / 1000000.0); + } + if (e->af == AF_INET) { + s.x4.s_addr = e->saddr_v4; + d.x4.s_addr = e->daddr_v4; + } else if (e->af == AF_INET6) { + memcpy(&s.x6.s6_addr, e->saddr_v6, sizeof(s.x6.s6_addr)); + memcpy(&d.x6.s6_addr, e->daddr_v6, sizeof(d.x6.s6_addr)); + } else { + fprintf(stderr, "broken event: event->af=%d", e->af); + return; + } + + if (env.lport) { + printf("%-6d %-12.12s %-2d %-16s %-6d %-16s %-5d %.2f\n", e->tgid, + e->comm, e->af == AF_INET ? 4 : 6, + inet_ntop(e->af, &s, src, sizeof(src)), e->lport, + inet_ntop(e->af, &d, dst, sizeof(dst)), ntohs(e->dport), + e->delta_us / 1000.0); + } else { + printf("%-6d %-12.12s %-2d %-16s %-16s %-5d %.2f\n", e->tgid, e->comm, + e->af == AF_INET ? 4 : 6, inet_ntop(e->af, &s, src, sizeof(src)), + inet_ntop(e->af, &d, dst, sizeof(dst)), ntohs(e->dport), + e->delta_us / 1000.0); + } +} + +void handle_lost_events(void* ctx, int cpu, __u64 lost_cnt) { + fprintf(stderr, "lost %llu events on CPU #%d\n", lost_cnt, cpu); +} +static bool fentry_try_attach(int id) { + int prog_fd, attach_fd; + char error[4096]; + struct bpf_insn insns[] = { + {.code = BPF_ALU64 | BPF_MOV | BPF_K, .dst_reg = BPF_REG_0, .imm = 0}, + {.code = BPF_JMP | BPF_EXIT}, + }; + LIBBPF_OPTS(bpf_prog_load_opts, opts, + .expected_attach_type = BPF_TRACE_FENTRY, .attach_btf_id = id, + .log_buf = error, .log_size = sizeof(error), ); + + prog_fd = bpf_prog_load(BPF_PROG_TYPE_TRACING, "test", "GPL", insns, + sizeof(insns) / sizeof(struct bpf_insn), &opts); + if (prog_fd < 0) + return false; + + attach_fd = bpf_raw_tracepoint_open(NULL, prog_fd); + if (attach_fd >= 0) + close(attach_fd); + + close(prog_fd); + return attach_fd >= 0; +} +static bool fentry_can_attach(const char* name, const char* mod) { + struct btf *btf, *vmlinux_btf, *module_btf = NULL; + int err, id; + + vmlinux_btf = btf__load_vmlinux_btf(); + err = libbpf_get_error(vmlinux_btf); + if (err) + return false; + + btf = vmlinux_btf; + + if (mod) { + module_btf = btf__load_module_btf(mod, vmlinux_btf); + err = libbpf_get_error(module_btf); + if (!err) + btf = module_btf; + } + + id = btf__find_by_name_kind(btf, name, BTF_KIND_FUNC); + + btf__free(module_btf); + btf__free(vmlinux_btf); + return id > 0 && fentry_try_attach(id); +} + +int main(int argc, char** argv) { + static const struct argp argp = { + .options = opts, + .parser = parse_arg, + .doc = argp_program_doc, + }; + struct perf_buffer* pb = NULL; + struct tcpconnlat_bpf* obj; + int err; + + err = argp_parse(&argp, argc, argv, 0, NULL, NULL); + if (err) + return err; + + libbpf_set_strict_mode(LIBBPF_STRICT_ALL); + libbpf_set_print(libbpf_print_fn); + + obj = tcpconnlat_bpf__open(); + if (!obj) { + fprintf(stderr, "failed to open BPF object\n"); + return 1; + } + + /* initialize global data (filtering options) */ + obj->rodata->targ_min_us = env.min_us; + obj->rodata->targ_tgid = env.pid; + + if (fentry_can_attach("tcp_v4_connect", NULL)) { + bpf_program__set_attach_target(obj->progs.fentry_tcp_v4_connect, 0, + "tcp_v4_connect"); + bpf_program__set_attach_target(obj->progs.fentry_tcp_v6_connect, 0, + "tcp_v6_connect"); + bpf_program__set_attach_target(obj->progs.fentry_tcp_rcv_state_process, + 0, "tcp_rcv_state_process"); + bpf_program__set_autoload(obj->progs.tcp_v4_connect, false); + bpf_program__set_autoload(obj->progs.tcp_v6_connect, false); + bpf_program__set_autoload(obj->progs.tcp_rcv_state_process, false); + } else { + bpf_program__set_autoload(obj->progs.fentry_tcp_v4_connect, false); + bpf_program__set_autoload(obj->progs.fentry_tcp_v6_connect, false); + bpf_program__set_autoload(obj->progs.fentry_tcp_rcv_state_process, + false); + } + + err = tcpconnlat_bpf__load(obj); + if (err) { + fprintf(stderr, "failed to load BPF object: %d\n", err); + goto cleanup; + } + + err = tcpconnlat_bpf__attach(obj); + if (err) { + goto cleanup; + } + + pb = perf_buffer__new(bpf_map__fd(obj->maps.events), PERF_BUFFER_PAGES, + handle_event, handle_lost_events, NULL, NULL); + if (!pb) { + fprintf(stderr, "failed to open perf buffer: %d\n", errno); + goto cleanup; + } + + /* print header */ + if (env.timestamp) + printf("%-9s ", ("TIME(s)")); + if (env.lport) { + printf("%-6s %-12s %-2s %-16s %-6s %-16s %-5s %s\n", "PID", "COMM", + "IP", "SADDR", "LPORT", "DADDR", "DPORT", "LAT(ms)"); + } else { + printf("%-6s %-12s %-2s %-16s %-16s %-5s %s\n", "PID", "COMM", "IP", + "SADDR", "DADDR", "DPORT", "LAT(ms)"); + } + + if (signal(SIGINT, sig_int) == SIG_ERR) { + fprintf(stderr, "can't set signal handler: %s\n", strerror(errno)); + err = 1; + goto cleanup; + } + + /* main: poll */ + while (!exiting) { + err = perf_buffer__poll(pb, PERF_POLL_TIMEOUT_MS); + if (err < 0 && err != -EINTR) { + fprintf(stderr, "error polling perf buffer: %s\n", strerror(-err)); + goto cleanup; + } + /* reset err to return 0 if exiting */ + err = 0; + } + +cleanup: + perf_buffer__free(pb); + tcpconnlat_bpf__destroy(obj); + + return err != 0; +} \ No newline at end of file diff --git a/13-tcpconnlat/libbpf-bootstrap/tcpconnlat.h b/13-tcpconnlat/libbpf-bootstrap/tcpconnlat.h new file mode 100644 index 0000000..7019483 --- /dev/null +++ b/13-tcpconnlat/libbpf-bootstrap/tcpconnlat.h @@ -0,0 +1,31 @@ +/* SPDX-License-Identifier: (LGPL-2.1 OR BSD-2-Clause) */ +#ifndef __TCPCONNLAT_H +#define __TCPCONNLAT_H + +// #include +typedef unsigned char __u8; +typedef unsigned short __u16; +typedef unsigned int __u32; +typedef unsigned long long __u64; + +#define TASK_COMM_LEN 16 + +struct event { + union { + __u32 saddr_v4; + __u8 saddr_v6[16]; + }; + union { + __u32 daddr_v4; + __u8 daddr_v6[16]; + }; + char comm[TASK_COMM_LEN]; + __u64 delta_us; + __u64 ts_us; + __u32 tgid; + int af; + __u16 lport; + __u16 dport; +}; + +#endif /* __TCPCONNLAT_H_ */ \ No newline at end of file diff --git a/13-tcpconnlat/tcpconnlat-libbpf-bootstrap.md b/13-tcpconnlat/tcpconnlat-libbpf-bootstrap.md new file mode 100644 index 0000000..e0c3fb0 --- /dev/null +++ b/13-tcpconnlat/tcpconnlat-libbpf-bootstrap.md @@ -0,0 +1,27 @@ +## eBPF入门实践教程：使用 libbpf-bootstrap 开发程序统计 TCP 连接延时 + +## 来源 + +修改自 https://github.com/iovisor/bcc/blob/master/libbpf-tools/tcpconnlat.bpf.c + +## 编译运行 + +- ```git clone https://github.com/libbpf/libbpf-bootstrap libbpf-bootstrap-cloned``` +- 将 [libbpf-bootstrap](libbpf-bootstrap)目录下的文件复制到 ```libbpf-bootstrap-cloned/examples/c```下 +- 修改 ```libbpf-bootstrap-cloned/examples/c/Makefile``` ，在其 ```APPS``` 项后添加 ```tcpconnlat``` +- 在 ```libbpf-bootstrap-cloned/examples/c``` 下运行 ```make tcpconnlat``` +- ```sudo ./tcpconnlat``` + +## 效果 +``` +root@yutong-VirtualBox:~/libbpf-bootstrap/examples/c# ./tcpconnlat +PID COMM IP SADDR DADDR DPORT LAT(ms) +222564 wget 4 192.168.88.15 110.242.68.3 80 25.29 +222684 wget 4 192.168.88.15 167.179.101.42 443 246.76 +222726 ssh 4 192.168.88.15 167.179.101.42 22 241.17 +222774 ssh 4 192.168.88.15 1.15.149.151 22 25.31 +``` + +对于输出的详细解释，详见 [README.md](README.md) + +对于源代码的详解，具体见 [tcpconnlat.md](tcpconnlat.md)