From 9af603b21a17fafa09c8a8afb705916e9706aa89 Mon Sep 17 00:00:00 2001 From: yunwei37 <1067852565@qq.com> Date: Sun, 7 May 2023 03:54:36 +0800 Subject: [PATCH] update tcpconnlat --- src/13-tcpconnlat/.gitignore | 2 + src/13-tcpconnlat/Makefile | 141 ++++++++++++++++ src/13-tcpconnlat/README.md | 20 +-- .../libbpf-bootstrap/tcpconnlat.bpf.c | 131 --------------- src/13-tcpconnlat/tcpconnlat.bpf.c | 22 ++- src/13-tcpconnlat/tcpconnlat.bpf.h | 26 --- .../{libbpf-bootstrap => }/tcpconnlat.c | 0 .../{libbpf-bootstrap => }/tcpconnlat.h | 0 src/13-tcpconnlat/tcpconnlat.md | 158 ------------------ src/13-tcpconnlat/tcpconnlat_p.png | Bin 41522 -> 0 bytes src/15-tcprtt/README.md | 85 +++++----- 11 files changed, 212 insertions(+), 373 deletions(-) create mode 100644 src/13-tcpconnlat/Makefile delete mode 100644 src/13-tcpconnlat/libbpf-bootstrap/tcpconnlat.bpf.c delete mode 100644 src/13-tcpconnlat/tcpconnlat.bpf.h rename src/13-tcpconnlat/{libbpf-bootstrap => }/tcpconnlat.c (100%) rename src/13-tcpconnlat/{libbpf-bootstrap => }/tcpconnlat.h (100%) delete mode 100644 src/13-tcpconnlat/tcpconnlat.md delete mode 100644 src/13-tcpconnlat/tcpconnlat_p.png diff --git a/src/13-tcpconnlat/.gitignore b/src/13-tcpconnlat/.gitignore index 3e91eef..1d5e532 100644 --- a/src/13-tcpconnlat/.gitignore +++ b/src/13-tcpconnlat/.gitignore @@ -1,2 +1,4 @@ .vscode package.json +tcpconnlat +.output diff --git a/src/13-tcpconnlat/Makefile b/src/13-tcpconnlat/Makefile new file mode 100644 index 0000000..b344853 --- /dev/null +++ b/src/13-tcpconnlat/Makefile @@ -0,0 +1,141 @@ +# SPDX-License-Identifier: (LGPL-2.1 OR BSD-2-Clause) +OUTPUT := .output +CLANG ?= clang +LIBBPF_SRC := $(abspath ../../libbpf/src) +BPFTOOL_SRC := $(abspath ../../bpftool/src) +LIBBPF_OBJ := $(abspath $(OUTPUT)/libbpf.a) +BPFTOOL_OUTPUT ?= $(abspath $(OUTPUT)/bpftool) +BPFTOOL ?= $(BPFTOOL_OUTPUT)/bootstrap/bpftool +LIBBLAZESYM_SRC := $(abspath ../../blazesym/) +LIBBLAZESYM_OBJ := $(abspath $(OUTPUT)/libblazesym.a) +LIBBLAZESYM_HEADER := $(abspath $(OUTPUT)/blazesym.h) +ARCH ?= $(shell uname -m | sed 's/x86_64/x86/' \ + | sed 's/arm.*/arm/' \ + | sed 's/aarch64/arm64/' \ + | sed 's/ppc64le/powerpc/' \ + | sed 's/mips.*/mips/' \ + | sed 's/riscv64/riscv/' \ + | sed 's/loongarch64/loongarch/') +VMLINUX := ../../vmlinux/$(ARCH)/vmlinux.h +# Use our own libbpf API headers and Linux UAPI headers distributed with +# libbpf to avoid dependency on system-wide headers, which could be missing or +# outdated +INCLUDES := -I$(OUTPUT) -I../../libbpf/include/uapi -I$(dir $(VMLINUX)) +CFLAGS := -g -Wall +ALL_LDFLAGS := $(LDFLAGS) $(EXTRA_LDFLAGS) + +APPS = tcpconnlat # minimal minimal_legacy uprobe kprobe fentry usdt sockfilter tc ksyscall + +CARGO ?= $(shell which cargo) +ifeq ($(strip $(CARGO)),) +BZS_APPS := +else +BZS_APPS := # profile +APPS += $(BZS_APPS) +# Required by libblazesym +ALL_LDFLAGS += -lrt -ldl -lpthread -lm +endif + +# Get Clang's default includes on this system. We'll explicitly add these dirs +# to the includes list when compiling with `-target bpf` because otherwise some +# architecture-specific dirs will be "missing" on some architectures/distros - +# headers such as asm/types.h, asm/byteorder.h, asm/socket.h, asm/sockios.h, +# sys/cdefs.h etc. might be missing. +# +# Use '-idirafter': Don't interfere with include mechanics except where the +# build would have failed anyways. +CLANG_BPF_SYS_INCLUDES ?= $(shell $(CLANG) -v -E - &1 \ + | sed -n '/<...> search starts here:/,/End of search list./{ s| \(/.*\)|-idirafter \1|p }') + +ifeq ($(V),1) + Q = + msg = +else + Q = @ + msg = @printf ' %-8s %s%s\n' \ + "$(1)" \ + "$(patsubst $(abspath $(OUTPUT))/%,%,$(2))" \ + "$(if $(3), $(3))"; + MAKEFLAGS += --no-print-directory +endif + +define allow-override + $(if $(or $(findstring environment,$(origin $(1))),\ + $(findstring command line,$(origin $(1)))),,\ + $(eval $(1) = $(2))) +endef + +$(call allow-override,CC,$(CROSS_COMPILE)cc) +$(call allow-override,LD,$(CROSS_COMPILE)ld) + +.PHONY: all +all: $(APPS) + +.PHONY: clean +clean: + $(call msg,CLEAN) + $(Q)rm -rf $(OUTPUT) $(APPS) + +$(OUTPUT) $(OUTPUT)/libbpf $(BPFTOOL_OUTPUT): + $(call msg,MKDIR,$@) + $(Q)mkdir -p $@ + +# Build libbpf +$(LIBBPF_OBJ): $(wildcard $(LIBBPF_SRC)/*.[ch] $(LIBBPF_SRC)/Makefile) | $(OUTPUT)/libbpf + $(call msg,LIB,$@) + $(Q)$(MAKE) -C $(LIBBPF_SRC) BUILD_STATIC_ONLY=1 \ + OBJDIR=$(dir $@)/libbpf DESTDIR=$(dir $@) \ + INCLUDEDIR= LIBDIR= UAPIDIR= \ + install + +# Build bpftool +$(BPFTOOL): | $(BPFTOOL_OUTPUT) + $(call msg,BPFTOOL,$@) + $(Q)$(MAKE) ARCH= CROSS_COMPILE= OUTPUT=$(BPFTOOL_OUTPUT)/ -C $(BPFTOOL_SRC) bootstrap + + +$(LIBBLAZESYM_SRC)/target/release/libblazesym.a:: + $(Q)cd $(LIBBLAZESYM_SRC) && $(CARGO) build --features=cheader,dont-generate-test-files --release + +$(LIBBLAZESYM_OBJ): $(LIBBLAZESYM_SRC)/target/release/libblazesym.a | $(OUTPUT) + $(call msg,LIB, $@) + $(Q)cp $(LIBBLAZESYM_SRC)/target/release/libblazesym.a $@ + +$(LIBBLAZESYM_HEADER): $(LIBBLAZESYM_SRC)/target/release/libblazesym.a | $(OUTPUT) + $(call msg,LIB,$@) + $(Q)cp $(LIBBLAZESYM_SRC)/target/release/blazesym.h $@ + +# Build BPF code +$(OUTPUT)/%.bpf.o: %.bpf.c $(LIBBPF_OBJ) $(wildcard %.h) $(VMLINUX) | $(OUTPUT) $(BPFTOOL) + $(call msg,BPF,$@) + $(Q)$(CLANG) -g -O2 -target bpf -D__TARGET_ARCH_$(ARCH) \ + $(INCLUDES) $(CLANG_BPF_SYS_INCLUDES) \ + -c $(filter %.c,$^) -o $(patsubst %.bpf.o,%.tmp.bpf.o,$@) + $(Q)$(BPFTOOL) gen object $@ $(patsubst %.bpf.o,%.tmp.bpf.o,$@) + +# Generate BPF skeletons +$(OUTPUT)/%.skel.h: $(OUTPUT)/%.bpf.o | $(OUTPUT) $(BPFTOOL) + $(call msg,GEN-SKEL,$@) + $(Q)$(BPFTOOL) gen skeleton $< > $@ + +# Build user-space code +$(patsubst %,$(OUTPUT)/%.o,$(APPS)): %.o: %.skel.h + +$(OUTPUT)/%.o: %.c $(wildcard %.h) | $(OUTPUT) + $(call msg,CC,$@) + $(Q)$(CC) $(CFLAGS) $(INCLUDES) -c $(filter %.c,$^) -o $@ + +$(patsubst %,$(OUTPUT)/%.o,$(BZS_APPS)): $(LIBBLAZESYM_HEADER) + +$(BZS_APPS): $(LIBBLAZESYM_OBJ) + +# Build application binary +$(APPS): %: $(OUTPUT)/%.o $(LIBBPF_OBJ) | $(OUTPUT) + $(call msg,BINARY,$@) + $(Q)$(CC) $(CFLAGS) $^ $(ALL_LDFLAGS) -lelf -lz -o $@ + +# delete failed targets +.DELETE_ON_ERROR: + +# keep intermediate (.skel.h, .bpf.o, etc) targets +.SECONDARY: diff --git a/src/13-tcpconnlat/README.md b/src/13-tcpconnlat/README.md index 2e1ddac..c99cd1f 100644 --- a/src/13-tcpconnlat/README.md +++ b/src/13-tcpconnlat/README.md @@ -8,7 +8,7 @@ tcp 连接延时分析对于网络性能分析优化或者故障排查都能起 ## tcpconnlat 的实现原理 -tcpconnlat 这个工具跟踪执行活动TCP连接的内核函数 (例如，通过connect()系统调用），并显示本地测量的连接的延迟（时间），即从发送 SYN 到响应包的时间。 +tcpconnlat 这个工具跟踪执行活动TCP连接的内核函数（例如，通过connect()系统调用），并显示本地测量的连接的延迟（时间），即从发送 SYN 到响应包的时间。 ### tcp 连接原理 @@ -141,16 +141,14 @@ cleanup: ## 编译运行 -- ```git clone https://github.com/libbpf/libbpf-bootstrap libbpf-bootstrap-cloned``` -- 将 [libbpf-bootstrap](libbpf-bootstrap)目录下的文件复制到 ```libbpf-bootstrap-cloned/examples/c```下 -- 修改 ```libbpf-bootstrap-cloned/examples/c/Makefile``` ，在其 ```APPS``` 项后添加 ```tcpconnlat``` -- 在 ```libbpf-bootstrap-cloned/examples/c``` 下运行 ```make tcpconnlat``` -- ```sudo ./tcpconnlat``` - -## 效果 - -```plain -root@yutong-VirtualBox:~/libbpf-bootstrap/examples/c# ./tcpconnlat +```console +$ make +... + BPF .output/tcpconnlat.bpf.o + GEN-SKEL .output/tcpconnlat.skel.h + CC .output/tcpconnlat.o + BINARY tcpconnlat +$ sudo ./tcpconnlat PID COMM IP SADDR DADDR DPORT LAT(ms) 222564 wget 4 192.168.88.15 110.242.68.3 80 25.29 222684 wget 4 192.168.88.15 167.179.101.42 443 246.76 diff --git a/src/13-tcpconnlat/libbpf-bootstrap/tcpconnlat.bpf.c b/src/13-tcpconnlat/libbpf-bootstrap/tcpconnlat.bpf.c deleted file mode 100644 index 4b334f7..0000000 --- a/src/13-tcpconnlat/libbpf-bootstrap/tcpconnlat.bpf.c +++ /dev/null @@ -1,131 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0 -// Copyright (c) 2020 Wenbo Zhang -#include -#include -#include -#include -#include "tcpconnlat.h" - -#define AF_INET 2 -#define AF_INET6 10 - -const volatile __u64 targ_min_us = 0; -const volatile pid_t targ_tgid = 0; - -struct piddata { - char comm[TASK_COMM_LEN]; - u64 ts; - u32 tgid; -}; - -struct { - __uint(type, BPF_MAP_TYPE_HASH); - __uint(max_entries, 4096); - __type(key, struct sock *); - __type(value, struct piddata); -} start SEC(".maps"); - -struct { - __uint(type, BPF_MAP_TYPE_PERF_EVENT_ARRAY); - __uint(key_size, sizeof(u32)); - __uint(value_size, sizeof(u32)); -} events SEC(".maps"); - -static int trace_connect(struct sock *sk) -{ - u32 tgid = bpf_get_current_pid_tgid() >> 32; - struct piddata piddata = {}; - - if (targ_tgid && targ_tgid != tgid) - return 0; - - bpf_get_current_comm(&piddata.comm, sizeof(piddata.comm)); - piddata.ts = bpf_ktime_get_ns(); - piddata.tgid = tgid; - bpf_map_update_elem(&start, &sk, &piddata, 0); - return 0; -} - -static int handle_tcp_rcv_state_process(void *ctx, struct sock *sk) -{ - struct piddata *piddatap; - struct event event = {}; - s64 delta; - u64 ts; - - if (BPF_CORE_READ(sk, __sk_common.skc_state) != TCP_SYN_SENT) - return 0; - - piddatap = bpf_map_lookup_elem(&start, &sk); - if (!piddatap) - return 0; - - ts = bpf_ktime_get_ns(); - delta = (s64)(ts - piddatap->ts); - if (delta < 0) - goto cleanup; - - event.delta_us = delta / 1000U; - if (targ_min_us && event.delta_us < targ_min_us) - goto cleanup; - __builtin_memcpy(&event.comm, piddatap->comm, - sizeof(event.comm)); - event.ts_us = ts / 1000; - event.tgid = piddatap->tgid; - event.lport = BPF_CORE_READ(sk, __sk_common.skc_num); - event.dport = BPF_CORE_READ(sk, __sk_common.skc_dport); - event.af = BPF_CORE_READ(sk, __sk_common.skc_family); - if (event.af == AF_INET) { - event.saddr_v4 = BPF_CORE_READ(sk, __sk_common.skc_rcv_saddr); - event.daddr_v4 = BPF_CORE_READ(sk, __sk_common.skc_daddr); - } else { - BPF_CORE_READ_INTO(&event.saddr_v6, sk, - __sk_common.skc_v6_rcv_saddr.in6_u.u6_addr32); - BPF_CORE_READ_INTO(&event.daddr_v6, sk, - __sk_common.skc_v6_daddr.in6_u.u6_addr32); - } - bpf_perf_event_output(ctx, &events, BPF_F_CURRENT_CPU, - &event, sizeof(event)); - -cleanup: - bpf_map_delete_elem(&start, &sk); - return 0; -} - -SEC("kprobe/tcp_v4_connect") -int BPF_KPROBE(tcp_v4_connect, struct sock *sk) -{ - return trace_connect(sk); -} - -SEC("kprobe/tcp_v6_connect") -int BPF_KPROBE(tcp_v6_connect, struct sock *sk) -{ - return trace_connect(sk); -} - -SEC("kprobe/tcp_rcv_state_process") -int BPF_KPROBE(tcp_rcv_state_process, struct sock *sk) -{ - return handle_tcp_rcv_state_process(ctx, sk); -} - -SEC("fentry/tcp_v4_connect") -int BPF_PROG(fentry_tcp_v4_connect, struct sock *sk) -{ - return trace_connect(sk); -} - -SEC("fentry/tcp_v6_connect") -int BPF_PROG(fentry_tcp_v6_connect, struct sock *sk) -{ - return trace_connect(sk); -} - -SEC("fentry/tcp_rcv_state_process") -int BPF_PROG(fentry_tcp_rcv_state_process, struct sock *sk) -{ - return handle_tcp_rcv_state_process(ctx, sk); -} - -char LICENSE[] SEC("license") = "GPL"; \ No newline at end of file diff --git a/src/13-tcpconnlat/tcpconnlat.bpf.c b/src/13-tcpconnlat/tcpconnlat.bpf.c index 544701b..4b334f7 100644 --- a/src/13-tcpconnlat/tcpconnlat.bpf.c +++ b/src/13-tcpconnlat/tcpconnlat.bpf.c @@ -4,7 +4,7 @@ #include #include #include -#include "tcpconnlat.bpf.h" +#include "tcpconnlat.h" #define AF_INET 2 #define AF_INET6 10 @@ -110,4 +110,22 @@ int BPF_KPROBE(tcp_rcv_state_process, struct sock *sk) return handle_tcp_rcv_state_process(ctx, sk); } -char LICENSE[] SEC("license") = "GPL"; +SEC("fentry/tcp_v4_connect") +int BPF_PROG(fentry_tcp_v4_connect, struct sock *sk) +{ + return trace_connect(sk); +} + +SEC("fentry/tcp_v6_connect") +int BPF_PROG(fentry_tcp_v6_connect, struct sock *sk) +{ + return trace_connect(sk); +} + +SEC("fentry/tcp_rcv_state_process") +int BPF_PROG(fentry_tcp_rcv_state_process, struct sock *sk) +{ + return handle_tcp_rcv_state_process(ctx, sk); +} + +char LICENSE[] SEC("license") = "GPL"; \ No newline at end of file diff --git a/src/13-tcpconnlat/tcpconnlat.bpf.h b/src/13-tcpconnlat/tcpconnlat.bpf.h deleted file mode 100644 index d6cd930..0000000 --- a/src/13-tcpconnlat/tcpconnlat.bpf.h +++ /dev/null @@ -1,26 +0,0 @@ -/* SPDX-License-Identifier: (LGPL-2.1 OR BSD-2-Clause) */ -#ifndef __TCPCONNLAT_H -#define __TCPCONNLAT_H - -#define TASK_COMM_LEN 16 - -struct event { - // union { - unsigned int saddr_v4; - unsigned char saddr_v6[16]; - // }; - // union { - unsigned int daddr_v4; - unsigned char daddr_v6[16]; - // }; - char comm[TASK_COMM_LEN]; - unsigned long long delta_us; - unsigned long long ts_us; - unsigned int tgid; - int af; - unsigned short lport; - unsigned short dport; -}; - - -#endif /* __TCPCONNLAT_H_ */ diff --git a/src/13-tcpconnlat/libbpf-bootstrap/tcpconnlat.c b/src/13-tcpconnlat/tcpconnlat.c similarity index 100% rename from src/13-tcpconnlat/libbpf-bootstrap/tcpconnlat.c rename to src/13-tcpconnlat/tcpconnlat.c diff --git a/src/13-tcpconnlat/libbpf-bootstrap/tcpconnlat.h b/src/13-tcpconnlat/tcpconnlat.h similarity index 100% rename from src/13-tcpconnlat/libbpf-bootstrap/tcpconnlat.h rename to src/13-tcpconnlat/tcpconnlat.h diff --git a/src/13-tcpconnlat/tcpconnlat.md b/src/13-tcpconnlat/tcpconnlat.md deleted file mode 100644 index 90309fc..0000000 --- a/src/13-tcpconnlat/tcpconnlat.md +++ /dev/null @@ -1,158 +0,0 @@ -# eBPF 入门实践教程：编写 eBPF 程序 tcpconnlat 测量 tcp 连接延时 - -## 代码解释 - -### 背景 - -在互联网后端日常开发接口的时候中，不管你使用的是C、Java、PHP还是Golang，都避免不了需要调用mysql、redis等组件来获取数据，可能还需要执行一些rpc远程调用，或者再调用一些其它restful api。 在这些调用的底层，基本都是在使用TCP协议进行传输。这是因为在传输层协议中，TCP协议具备可靠的连接，错误重传，拥塞控制等优点，所以目前应用比UDP更广泛一些。但相对而言，tcp 连接也有一些缺点，例如建立连接的延时较长等。因此也会出现像 QUIC ，即 快速UDP网络连接 ( Quick UDP Internet Connections )这样的替代方案。 - -tcp 连接延时分析对于网络性能分析优化或者故障排查都能起到不少作用。 - -### tcpconnlat 的实现原理 - -tcpconnlat 这个工具跟踪执行活动TCP连接的内核函数 (例如，通过connect()系统调用），并显示本地测量的连接的延迟（时间），即从发送 SYN 到响应包的时间。 - -### tcp 连接原理 - -tcp 连接的整个过程如图所示： - -![tcpconnlate](tcpconnlat1.png) - -在这个连接过程中，我们来简单分析一下每一步的耗时： - -1. 客户端发出SYNC包：客户端一般是通过connect系统调用来发出 SYN 的，这里牵涉到本机的系统调用和软中断的 CPU 耗时开销 -2. SYN传到服务器：SYN从客户端网卡被发出，这是一次长途远距离的网络传输 -3. 服务器处理SYN包：内核通过软中断来收包，然后放到半连接队列中，然后再发出SYN/ACK响应。主要是 CPU 耗时开销 -4. SYC/ACK传到客户端：长途网络跋涉 -5. 客户端处理 SYN/ACK：客户端内核收包并处理SYN后，经过几us的CPU处理，接着发出 ACK。同样是软中断处理开销 -6. ACK传到服务器：长途网络跋涉 -7. 服务端收到ACK：服务器端内核收到并处理ACK，然后把对应的连接从半连接队列中取出来，然后放到全连接队列中。一次软中断CPU开销 -8. 服务器端用户进程唤醒：正在被accpet系统调用阻塞的用户进程被唤醒，然后从全连接队列中取出来已经建立好的连接。一次上下文切换的CPU开销 - -在客户端视角，在正常情况下一次TCP连接总的耗时也就就大约是一次网络RTT的耗时。但在某些情况下，可能会导致连接时的网络传输耗时上涨、CPU处理开销增加、甚至是连接失败。这种时候在发现延时过长之后，就可以结合其他信息进行分析。 - -### ebpf 实现原理 - -在 TCP 三次握手的时候，Linux 内核会维护两个队列，分别是： - -- 半连接队列，也称 SYN 队列； -- 全连接队列，也称 accepet 队列； - -服务端收到客户端发起的 SYN 请求后，内核会把该连接存储到半连接队列，并向客户端响应 SYN+ACK，接着客户端会返回 ACK，服务端收到第三次握手的 ACK 后，内核会把连接从半连接队列移除，然后创建新的完全的连接，并将其添加到 accept 队列，等待进程调用 accept 函数时把连接取出来。 - -我们的 ebpf 代码实现在 中： - -它主要使用了 trace_tcp_rcv_state_process 和 kprobe/tcp_v4_connect 这样的跟踪点： - -```c - -SEC("kprobe/tcp_v4_connect") -int BPF_KPROBE(tcp_v4_connect, struct sock *sk) -{ - return trace_connect(sk); -} - -SEC("kprobe/tcp_v6_connect") -int BPF_KPROBE(tcp_v6_connect, struct sock *sk) -{ - return trace_connect(sk); -} - -SEC("kprobe/tcp_rcv_state_process") -int BPF_KPROBE(tcp_rcv_state_process, struct sock *sk) -{ - return handle_tcp_rcv_state_process(ctx, sk); -} -``` - -在 trace_connect 中，我们跟踪新的 tcp 连接，记录到达时间，并且把它加入 map 中： - -```c -struct { - __uint(type, BPF_MAP_TYPE_HASH); - __uint(max_entries, 4096); - __type(key, struct sock *); - __type(value, struct piddata); -} start SEC(".maps"); - -static int trace_connect(struct sock *sk) -{ - u32 tgid = bpf_get_current_pid_tgid() >> 32; - struct piddata piddata = {}; - - if (targ_tgid && targ_tgid != tgid) - return 0; - - bpf_get_current_comm(&piddata.comm, sizeof(piddata.comm)); - piddata.ts = bpf_ktime_get_ns(); - piddata.tgid = tgid; - bpf_map_update_elem(&start, &sk, &piddata, 0); - return 0; -} -``` - -在 handle_tcp_rcv_state_process 中，我们跟踪接收到的 tcp 数据包，从 map 从提取出对应的 connect 事件，并且计算延迟： - -```c -static int handle_tcp_rcv_state_process(void *ctx, struct sock *sk) -{ - struct piddata *piddatap; - struct event event = {}; - s64 delta; - u64 ts; - - if (BPF_CORE_READ(sk, __sk_common.skc_state) != TCP_SYN_SENT) - return 0; - - piddatap = bpf_map_lookup_elem(&start, &sk); - if (!piddatap) - return 0; - - ts = bpf_ktime_get_ns(); - delta = (s64)(ts - piddatap->ts); - if (delta < 0) - goto cleanup; - - event.delta_us = delta / 1000U; - if (targ_min_us && event.delta_us < targ_min_us) - goto cleanup; - __builtin_memcpy(&event.comm, piddatap->comm, - sizeof(event.comm)); - event.ts_us = ts / 1000; - event.tgid = piddatap->tgid; - event.lport = BPF_CORE_READ(sk, __sk_common.skc_num); - event.dport = BPF_CORE_READ(sk, __sk_common.skc_dport); - event.af = BPF_CORE_READ(sk, __sk_common.skc_family); - if (event.af == AF_INET) { - event.saddr_v4 = BPF_CORE_READ(sk, __sk_common.skc_rcv_saddr); - event.daddr_v4 = BPF_CORE_READ(sk, __sk_common.skc_daddr); - } else { - BPF_CORE_READ_INTO(&event.saddr_v6, sk, - __sk_common.skc_v6_rcv_saddr.in6_u.u6_addr32); - BPF_CORE_READ_INTO(&event.daddr_v6, sk, - __sk_common.skc_v6_daddr.in6_u.u6_addr32); - } - bpf_perf_event_output(ctx, &events, BPF_F_CURRENT_CPU, - &event, sizeof(event)); - -cleanup: - bpf_map_delete_elem(&start, &sk); - return 0; -} -``` - -### 编译运行 - -TODO -### 总结 - -通过上面的实验，我们可以看到，tcpconnlat 工具的实现原理是基于内核的TCP连接的跟踪，并且可以跟踪到 tcp 连接的延迟时间；除了命令行使用方式之外，还可以将其和容器、k8s 等元信息综合起来，通过 `prometheus` 和 `grafana` 等工具进行网络性能分析。 - -> `Eunomia` 是一个使用 C/C++ 开发的基于 eBPF的轻量级，高性能云原生监控工具，旨在帮助用户了解容器的各项行为、监控可疑的容器安全事件，力求提供覆盖容器全生命周期的轻量级开源监控解决方案。它使用 `Linux` `eBPF` 技术在运行时跟踪您的系统和应用程序，并分析收集的事件以检测可疑的行为模式。目前，它包含性能分析、容器集群网络可视化分析*、容器安全感知告警、一键部署、持久化存储监控等功能，提供了多样化的 ebpf 追踪点。其核心导出器/命令行工具最小仅需要约 4MB 大小的二进制程序，即可在支持的 Linux 内核上启动。 - -项目地址： - -### 参考资料 - -1. -2. diff --git a/src/13-tcpconnlat/tcpconnlat_p.png b/src/13-tcpconnlat/tcpconnlat_p.png deleted file mode 100644 index 74caa9ac9eee3f528e8bb6924562a021afd12491..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 41522 zcmeFZ2UL^U_BU)9bc8{$04h?{5u~dKC=kGkN|Rm!L`6Ud0i_BdL{tRJz>M@70hJay zBvO(npa>C>mH+`FLTE`SfrJv0ybn6#ecwCx>U?W`Ykl9l)<0{x5ni`$o#=)0XROw(6B1vyZvCsDHv#YTGtayP{#zelWqfK~$%nm@z{O9V zC(TZ-TUQ<@vU)`bxZZN(yko$+b=w2i{;ePKkh`^RU3T+@Gbe3sxiWE^Q+Cp_Fa)ZA z;ydAv~9ys$ctp|{zd-fx%L2nZR`=K{P2E$lUCM!;JlVD!()4ufNw_5*I%6?c&hSLi@j6?7X*O+qVnhQ;|P^yAaC$|J7rV zIh zwpW7u=FuN%#}aOteJIk3(_{MRaS}8&m9`Ru*ED|Zm9N^vmxlfCoON`oZbK`TPGh%-!nT@1ylW4+*ZJ@st<&xteah*%T zXBOoDyvN8r+i1W@Nxb9kcLuQ+x=OE`-a=nQ`d?b$OqEp#eRgm2`}B zK|lO<`y1QHThzO8_u0|){NM!+tMEobIYrp9Z$mjdQ>Fp0uQjne!L()DyT}jSC>2e^ z_rD=2x1E;2X}_%Gv7HffQJVd{opX)1QT2?brbSgDFap&XB2 z%~40X(p+zy^xu&1usdu8@3i-QBP z_W0#;Mqh#_vlRXD*cx)4(KF8aonvGnsGgV8v%C)12A#E}xbuN6sp! z7+PcG>;@t9P<;KUefw68-5 zFKIKHTBZTxED~Woh3?vZQ|IZ(nRc$fJN=|~L=TxW^@_}lpA3NUKf7Zte})G8<*nzu zy;ln3gqXnQ1}ut^(UgW2`$ZW~6{*Knd9F*{FFv?jrmFE@d;BJIHPhE+QJ*LkqZ3?i zOijNUkc`I2A)R3+_*9?Ma+BBF0>O{CjOElK#{9m!%tk-E{)goj{aQK$?u+=P)V8IF z=f%r6i;EqO5zi5pX4~6OE-njw)2B!D4M&v3Jlj&GeXYF?Jya8OFBkf}v?acQ%|5~! zIU<3ve%GHK+51`+^|CX`0!gl=2Eps|MT9=@0qY{~;8tgG!8|i2y6a=953TbE=aU)T zmr$OE<1>}hLE*QnYpScYpo5K@%$&W6Zw;Y3)Ejl6<6#ADaO`-y`-mBjnHx0^n_gAY zp4}?pSIkHnf6@0)NuyVdMy$BFGsfPT`nI2-O-V9`znQ+?(dxvJfj*M_tvn0sMG5KO z*eKQCS>OjI>rr|W)hOU``@Fu+nRC57EV^!{xV_gCYS(wiBCFa?&F0Y7GRsHg-ip2h zjBLGQ^A&g5f~!#o!+NY1WYM9G`?xw0YTv(rl`WUd&IteLx*S%G;BrpWv%nFNy{%5g zaPy*wU4C&P-q1GikUKLAbG?#Qh7GP*|kg6J7cW!_3QDrlpA#yPb0>!l+&`` zR~YT2VXfaesfd&N&@e-I(C1TN_J}t(V%mm#s~4CF7&xUOjh))(@TqH zGVJZtZVqFxm{BMz*VoV!UyMA&jKJfTPalAFOkJQSRbwPKD)zG`PovqQ^yv`6X6=wE zY0g02BcE$>Z9mmFh9PR-d(?Nb$NNH;lMtIfTR`n8g*`N9eiVpQ$e8mDd*_;o>wPO4 zK^n?j#Qio!=8rebC9T&E>)Q{{Yi`&4`K}J9S}a+$d>GfCAQ5AN#G@%BP`f7?)V_S7 zFBAcVF22>lcz+Hc`48d+&hqAJDh5Gt73h4YDHy~xt1CJiw&`=C4yIb#ukL_-xNF~7 zXnNU-H2ish|HUOI?r^R_4cWSsyhv2&+Yxe*+uxd2y+kF?C;x>g@>P-c6N-utX=E#T z4ny;mfwbX1ds(|=N68psl16D+<|&d#phy)2&DWZhaOzDiBQl95+(Z(iXdcMxW1KbdY3Y29xm93pmbEzVmWazL=&Yh4NXYl$ z;F_=pZCMKixVhDJbwxom54nqf#-`h?pD{*{2VG?(!SYkJ7n|69+4H`59PKE}-%*RP2c^*K+Ml z*UHj}rQWa!n~4hU7`j{OeMREn>e4ue*J_IiCs9blZ&AsRKE7=_-dFDa0QhN)h|pEX zf;>8)$o7fq;$YtDfK*%scV47Pdb+~cz{Q)GhjVwCzP{AjJTT4G!)-lc5!a~2E*6o!6~%+SwG12YzMMy* zS>?VE*J&1^awqw~H+*uFeL%T*rdAOx7;HGpVL| z@QvrTbGs<6q0l0udKSrmGjs>o%F&Qwi^Y;)I+Mv53mz<{e6oN|7ar8Eh3M6>q=cJB zjcvw$0l=T1FU)?B)Rl#*cptG#^Kf{`QxX9$7{^@Who=jw@ftS|LLQ9NHu+0Q4^qXfKpw8lmrg`eUh9Ge)AaLMW%3(I`sGT)cl4YdZu0Blr2bmHzqM zyuJZByZ$s(L`8K`Qh$Bv+rA=RUB8yoc(QO)5V@3O`c_HAXL70<6SllRU3cmrJwc~v zu2m0GbpeWO<>i#O)^PkjRLUVUpc+(byHAN&hOv!sxku-GF!w;N55MyPi2!3cg!JZ?zjlX0r69o zTM_i~Y+R;529SGYx|Fgcq(TCX33I25EIAI%aK;~&A^;Lu=7hy@l|LBOmp=ln@RK=h zojr8d_-f2&ZXaioj$lfHqOWDL{{@H#Bw`+<**fgjS&&NAd1hqVjc#bYHE4!CJ0n-A zEpi5(VJH=Ipcpz70^&RUZ{{Y_gjRiF^!}7^6NKUeI;hbl=%;%ZZJDnxst#Enw>`U^0UF}c$ zU0OKrFnOPXS(;7o7JZ{EzPvT$_dE@6&2};biDPWHk=l#G%Ui6y_pVE7bTaPh(dS00 z>yP&$PX6n=7Q#(a(Iaagi?9}iLRaqS^DIqY?2I|MZgU-Qo7}Z2;Qh0(u?vw7yc^-b z$H2{B2Q_@ylCYutr%#s`f!pqyuDze%rmrMkBp3UyX971HA4#>6Pg=K^0o;cCy#j_Kjr)?NN$R$lDNsbt-*l&hVQz8K_B(MkKa(fGQ9Lv&aZ?5?DS*EYIq-)xX7XfA5g z57n?GRI}5zBH3Pac-I4op#Y0&LRG_2$5~$Zc#T972;*BfV#qzV@^YwHrTJZgl64O8 zh#D#s|EmS$NuJ@OmW16g2XEimWcX(!N>xJt-zr}a2AB|AU&>VHVdBJC?t`@tJmj8MDV8rR8-*; zIho9AS#l!C%Io!b@z-Ggp8~HY*u!EkEEjU?&<-k{OqvEQm%7I-Xnu{x*3zLRF1Xg| z;}dc@Gc{AQZ_K4ABWiKV@T$%`?co0oB0MqI9OYX#r;)I!BL;`yRsC&iLAKyG{c+Qk z-rB74vkN>s8T{O=8EItooO_&|(`+dHYlQw!P~&|PTBK=wT9=YlSLfB2+vB?y*)3qB z%acV2bVwyHX9hcUV<~>op)QwS#OSyvV&L};9Q+-S^gRiRJnnr%3>xr?nhbJcenBza z=dw=ML7*1Z1}9b`iqvP0FX>=BXsiljRhgywHQ4b5N^+p5yBR_>Z@^$2sW!K+Lp5;*iO(M0U-}?b%!3b|6@vZ^3ze zduzv{Q&8W0PA051#=cdyJn+*z=^H}(8?Z@L)3TIr+^@!tm8?MAJf9zXUN-(1+G=m+X>Mb6&; zXG6^_c)2ZRmWLL9UHHJP+)W|a=8_YKS7eLKA&kI3MC)lw|7<80zYW+$A5Dv1 zQ?SVq>o-G{A;%uz358ucVMEu&Twc2Ms6)*JJxIM61Z|kxdFxK8jW9HaN zw;>>DyPalWFr&LlCE?;`r!o6gbHdXlzWSt5nHLWOtJ^bMriQSLIayweMa#HhurReSK^$|?io5A@ee3fNqO#uXQ& zLtn1vBM;fa;~7UkOMolA&mIyF;?d{il8V>cK39lq*D%I=i@8);uMNMlwMoNxs~UWv z^Y-`sKzCr|5~~qQX-sB#hwNPtlL!YKQ!a4^T5&co^gZ{(`BKW*ZGkku2v%g&&8CR0 zB5NojOyWjQn2bvVd060d#WC4hr=ZNVknvDWVI9qZ|6xZ09#@oog zIVZnmWImN*fql*V@Eg9g;>IVn=4gnU`oSOU0BmPWIn8mIA!FwA>NjH{TluLlA6#v| zhc$m;B+mT;X{%Ofc(7pz7+y(M>mctJLCxHV1ocfdRTyiCet+mkiyJHp1NZbZ`!C7A z^(`$j2%ntR<}>qJpP*%c74rc+ASM4bD5vzSB%Xg+gdRHg!9C7(&kvT(>X7&joJ5R9 zVC}n2L*dSjPpVIw$)KnRP#+LHG&;!pRH8v0M^>i-lztF+4vEuCa)0@JODo#;^SsinO)Mmjj3LCI$zcU2LycVv8~ zCdfoigW>lLHEKh-uin*<8-vI95tP18sFFCD-lvD~=Ht5R`XV?jL|p4X8D^UMF-osL zVOwd!L8r>@rn!O{`RXEUugQBrxg`lQ)jR6yS2@)9#l zjKLxrf#)rAt`&(28u29!|0g<-bdjHmbNbpsS7l6ey+98#IGL-2ilk`z88XAsc5 zn{%pJglJWK!u#_QBY?eL`x&%yV!mVxW$!GKnJ0%-rASkMJkVC@-j;s3`ML@lXl&YX z13ei5^@_q}Mkx8-V#bmu_&D6VZ)oZM_g&T{`IU0Y2~nT1msi|dNWVDBp;VIjmtkUD z^2B_x54rv^M|3@h6=cO|GS~dE`=3m zXzOccdu1La!!Ee#$BTT`7Tu-~ZOtAmt&G+f#4%##ez=}Zg$}|(Ip(5djU&k$eq@N- zimoc{jA3DT0*k(FBho)|05P`ZHo|9k$J=shx`?rR2tU*{^uy;?@@syXJJl7^`j79J zvPT;tsxA5~N2LDIl$sQMb>9w657sjB=R|&_aKHj73w;g`V6G}(cxC5P^^N&TE8%Tpx#`y z%AGV}EgTG8sZ23V1uBb&uFr1#v!L16@JrJ}*S>|HDCJi`6pu%QVU3qGDu1ubbVsoc zw+OaOHna;IF@$|$JtS&VCqg)6DnFN5)n#7o-FRO#mssw5F$=${HC{7==Q`B|Gh6u; zw_9IPvHu~t0D5*#iP7b(pGT6cB2%iS3G!?Har9V z52!Y3!x1%2F&$H&m>W=JCUB-#iYsM=U&z9vMMAw5_SJ7fABN|XLtf(wxo zBL98iTKBFkT(Gy~)y86M%+`;UU-!vG_zYxrvG_kEqo*QMwCzSOUr`dTtQ=AhxJe79 zQ8^nI(9dqL0PTCSp+jmFLH}Ek-V<|4{>5(+CmbAvn|ubtbE$nO0Z(8v>xWz#-#`=` ztaZ*i-JImNASZx?^$~b-T)LlN9}fMVz>7F7H;>98N_SIKopdj=O(G|LFt^VMixrRB zv0m1Ms)Htyly}?Y`nGM>`7ju45gn}U`N0Q#7>NhX-7PcMIMS#Peq`u_UzkS}SHU&V)jD0@D+eLq?dN{N zg}_jQAzO)?LGua4PSwieQ$0nEVYC^IcwP!&eY+-=U7QC4bP!G9gx@0;Guqm}3#t2!$p)jz2yC)aueH1yH_zKw&$6oxxfhizc3 z@_O<7q{Of+8nztI9tt1PfhyGRs79#eVX z#Ri9VlhR`0Mn3}YEoLxmUNmeF+J*K^@U5Oa#~}+=qOVWh?$H43;rm86`A4ZRp{TT8 zHH3H1Ew;=LpDL{`HbEs%GkXm}(mf;#FIipbUP+QZ{YS#Ui`0;4nU~Zsto_g_iv5m9%7+6`NAUIL+{vta<%i=Otf%b ztGd$lT(5&*+|;~HVA`#n zFU5sWT9VXV_GCf)DNF6mQCBM}O5rf@j=X2|V`C2gNYLSO1HEbj<$nFb)tBhq&^C3# z@sM=k($IN)n~Q}$?Vg*@^uURN9Hk>jIL(F1Fe07n_I}gi?Ttkzj>_tYTte;%c&G zVL#Sjec=LYKU)Pa$?Ouk-p?Ak1g~k%M$X`YjI0djZHnwA<%K<3`N>WiN05d#$6*um??Vvu zM2$@%juD{5+aP$Li009t+VY35Su~gTt)qD}XS~wdw5(^W8P&z`t9LJFWgm;CRP^Wty1iljxA6lj$++-%;8v4-N_}l;CF15m! z4@%v$wk1kkNTZRMeDxZ-j&p$Rq33~)?pduBs}P!*Y88lgK04_F0&+Q<36)HbKpzm` z8;6kXPR7dV>hY|(W{z2R~QW^1eu0wz7VXB`yNd;hk z&S9xpbz>!{kl=XXH(6)~J>A)G{CyXrvK%#5l5Ed?NkS69{hOi- z)bl#OW4xfRz&hs5+x+grGbIzoh)BbH_Ds}pLWcPW?j@2MIopAx^??W zD;!0nVQ_CS{UXRB6`*O*!5Zb^M;!WQjJgXNW^nVt4v=kjGGTv_nz^$On@DRJ=5f6-B4jJ zu-%t0H#0GRT?2Z_uWJ^B%vnX0)H8A*1FoTN6tcerG+x2{XU(BNS)7A4S%g@REn@=V`4v)^sYzmh zu9jnISGcl=-r8YX@t6k;M<~vtlEF>L!S3tpOuc zAmTt@npK6rnu_1hk-Reo|LMPBRv4P`ssT`_3gCI8aRLuE9kxeJu59Jm4}-Jb8dk%j z3-ookC_z=R_Hw}g)8~EGK$f-f#|b&KjdmztEZ~_!Nb1UesIKpf@p5zDESUrdqm}51 zBkEM{d^6ehQAI9CjNB~?fxSNeFCks~L zlp$A%3>#y{MKoC8z}M^JZhcU`G55ZN8$Lw$j%R>0_f5^C~gVLx71UDj`jQAgYs z2gNOpkNuo}!tLAwW@;AyA=BqWMjz(-@G0rzPYGHB-jz8d=*_{NVZ_4Rgk_C80CVEo z(%98%V9fh-u}dFgPkjEpEFAEP*!4W_eVCe4)koJ|`tO1%Hz_wLeV!RtM-1gWRqW7Z zE%6HdiODWa9PP{S9(Oaz7$EVXq5%kXM&xI-tVyxafXAcXVDvUYn$=7-i+GKvru{$z zvY7Q)je3R4%!zZ1|8Cr%nZ&Z*wl<`cbA4j~%BoaGfZe-@iK5e|_8si6stvtMraZUY zn6BfORqRryZ7*NAYsU2QiTX z<2UZ~0S1y^*-6w{?jrO-1a!HaIPmvnC_9RMKJ^smnj`}^*UK* z&l+s>gFF`KC!B|vr}~U%ggQAW5{&AxZrn?|4lcG6B1ZRZzUSx-F+=%UD_B%mn;%!e z4&7OvXTI+;Zv`M30=YWZeCgU`tw$YVVYp1xwY2?!7-cl5dAw;0WK{S}Vk{2Mc3#|z z6(dt`A@YkID;3^PSuXd|hGJ{DMQ6G=TS1qpH&EB8Gx$ogTYgzVD=j{y+bJd`_R(67 z^1gFq!~6by3} zct~Bza^f&AZ~TcW)vFH1NNE(qY?Bkku0aJA^@9{HjcNNu!0HwhY=p{Ja9R@)D9CUq z+GPYjM=p1!a>^eFnx5cQ+G0m@snU0WcJHerYkOR|%Co7LK`smrSG1l^jH<04)CN1; z4NJdIEWF!;;2yX%eRmPpKOK`k?C2bK@Y^-!-}NNH=_W#bJ+SE{Me4 zM(?L(9i}s)e4G(9_nIKL zB$VUvu#XU+B4lk!jE z24LelAkFYaYHOkj^UXMlsAg~ZA~i@?&7NWrp?{_V2N`k=a_9moih8iBK}^T?4Mn~M zJ7bXCM#JctuE#i_wMpRdQN_dB;B$-F3f~U~ci(Wx$ zpz=A8cN?)7BN>y~Pj$Bt-7Bt6%mUR$ZLkzdz;D)Ybah1Cs)tO~iP~=WK6k~POs9-@ zRD@XUVU}j0@QoIlDnTgU#gHx7$>qS8TyrRrEK|db+jF8JN!8IuQ5ODe4UQkrv?pra zwW@-`^(aM6oG0|2D?n!t5QWg~y>Da?=-Gq}WLu`}-UW0sv0$GC?hWMtA4Bg`K=kM=F5s++;naKtBHr=kHZO6MFzh=_?cAL9JEWMuQbXgeoZ571|M_F3Q zPubWb1W(nW3TfuM%{_jN6@)wc!r{!?T-@sC`I?5(k0J8fFn{lNd;lLuGh@=7oTYrb z45p-Rld(frHcw0W#+#YzDKYXq_D$8v&iHgGqZc_~zk714OB{-yPhW-m71RRsLs-Ph z$=wpybzQ)fdxE^N4==(_SE^`9TTfKEVit)MVmqdg6d3uk1;ro4^o5UH-x6ty-6`C( z0`~P@3ne)-rQRWf@St3zK#;?=-0@x2?KtU{wWhiZymG{`ksISo`gjsZFi<(%?Us0c zJah2Zj1TD4hs%Hx!AV{vx@JplI1muB=ZQ-E4a;8*(Kn zQ!nUeT0%tqX-(nynJe9oF{?A0s#UI6mjS#Q!t%QR?BRKkJ+EN_C>O~1TE9Lq`dsR? zS7EcIWx-J^Y{5@GrI+Xi5F|)brz@6e>$?^prDD?k7QmFxrO22hafcui;A|ZbPl%W$ zBEZ@xRsE&X13R?8R>KhhW8W!VZF=m`o2H&|7!1{I{}Q1kO z?1%@U^M(TLe=^n=gcF#x!gl8Uei41|RUsW`G=Pb~%{D5OFWM!MNTt!%->nJ0-v0$% zRsR8;UwR)aaq(w#?T|>5nqZ~*u=HULzO~bgzf?-PT-27!O?duYKt?jBFzFAak*6BdPIMnlD{`%TZJL7I3<9 zM9zSAkwmrDJy2_|RSI2+H$_~?>eCIMsLbvzQsP$o$-21in8NJ@XO#-?MAxVDX%-y- zPpbGgeI3fgB4%zzppfBbJ&WEJ19R7cJ>$iO;hrHFiGC7J(T zGnW4KDp1ZvID$icfdVW!81TDQstupnPV&C|acaLLav8CJv&r?vm&1Tn5pnlu?X3u! zN*FIxr7kTBR_9rUJh*pgS(>ahE&HOS1h-?{d5fC>Q__x=gTWS2j-`y!C!LZ1y_o)5 z{Hx(4j=v>ve}j=Pr(bux_l90=WHVgP9(yC@9Mn_G3YF$!*~xz~ms#zuIuswG^&X+{>8nND?g^s)%NVMq>9htKp=R_&Thr6sqVq8wUCNvoCsmsnM; zRuK4PLmLN$BIXgKz@mUI&5P!91y;oE0uGQdscTd*PZ336ikz276)f^Wl$<7iRST}p zF<_O9Z zLV#tl%8SIR;vfu14*3D{hq3}~5}<1lrXe0}7+PAbH)TlV>&}TGsUGcBaa4!Jp|G+$ zs9!>>8Yy+~ zdUwdQ@=wi)0wX_)=;KCKVlhK)oZ`z=C}0uj_MYDW_8-shxN7a{8rLaxFwQpKrbU|ZL&b42>=kZ9XfKJeun*pjMANi*+p3J`_yCo z(}tC<3q0G*A$TW*QRisgDqH>X-#GSvj+V^=xC@gUAT`8XYuOIQ_@pAVIo5Tv1C_aS z{u7b0cC1)5QXE&Te*Wa}*WOzN(2y-~j4>>6bm1gFv{>S5>t*={6oRP3s}t}dT?8jq zVfg0uznwVfTkEN4?`jQM%*0dzN2+_P3Ls8J>EM2#&~go^J7E%4?EBa1;<)ye`h}c% z2D-~4cF0cZ*|)U=vJ!JK#6*KkqgVd2f&a3sL{iIo9a#w%zG=F3E^*!j)ekhYDz7Yy zQ2{m`-*}A?6(Y-$UL#H2V*LAwim%NY(fceAZ*-)QzE*CkabLr2E?o@PCH1M5v*TLT z7j`J#v%0yE{|BahS}&T$>>rYyhA(Eqd|IBo3GTnpY?~6B;GSgR|EzeperwG&02knLa=hWshtg5hru{K|Zlm6!FIV#PBXU`;r*A z>P9vbJ63eU+6I?fT(9>$#+e>-OLwLkj$(mvaT z%n;6QPDgZO1GK;~YH)gHyL!=Vm+INfx2om_{0ZmCTb$i_qnJM+^09kP>>nuMwrD{~ z_XJbgnfpN@(-}AFK0a=Ib{%b|bZD&y{G}!9F&Bf@;nH=%}&l4g5% z#)wC~Mz3*U*3Jk0HF5k(HSP7?tSx!Uio|0FEGGg-NZ5g0>P4pdj8{DW>5n&xfSHeZ zDbl9XA@X=2OTYv(o%B9`dRxGq!T^Z z3hDLZ@_d|MtLMR-uJqlNI#yU_YT*CIwSyL?O;HTOjch?~74*c_j%rvi!rn?sBn26I zQiO@uN=+7H)Mo;E_@uGh1ne4*B}|7xcC8=3%AKA3piRq7WEzx8 zeDRKGoVbfY=#t5>Uqaegkp;CJ;6O>YVH6lQhk`jnA_yYq1?v)UwCNX*wew4Br?38H z|1%7Q;bU$*;dz`41sq*B^9$80-(7i=^b8K3f>0u2pMs;=;@SGu2xlrNJ9AjryD8CWRg0p=l-tsAn0D70$m=;j zLkSo2|F{ed4OW+yfG7;i9;h0)13aV$78ujIdqoa0FD z1S?BwW;G=UfWr9BuLlp-db^H*&r~`e`LpG&_W&#U%Xzu4YzOHzocAyHu-4N7s^I^@s{d^Q zO5(f={}&GDKABa=xEwNg6TE;4CzrTDak5j}aK*Yjpedxy={nLuQ7~U>7z*XKbyu`* zy)LGA!%%vzW>2k72_y42}AJ^(Yr3d`OsPuwP91;8O?HtThkFofMV%yW+?48j}IwVLNHzu zNleTLhSUGiz9AyQU|LgezAE0Ij)lVO%(9MkD41pZ-gm%5TrPk#F@#D>I~{yT^&2ryWB)!j;K9=p4kN*;;R zAHChpJr$QbUO?G1QwhH%XO_O=-a9jTLT3oR_I={NL4m8HVPgfewWd7(hnYZu`ATRuj}#!_bI;b0t+@HLs8|A9 zCOOKLd#d?N_F~>+cdLmSiHeT_27nWmFjyR2$pV`)XX<9^C9=o>OT%hLs1GO^mZ4gX z03oRjO?AUH!cR_Bnxl}l#a{8&jTO5g2#Ks*QXAC{sDT7Lan~8T0b2?=h{Ull_58H% z`86U8J9Kn-qIUT$ANJ-UP*sZGur25BKC;zHG|BcwlP_`c6^~Oj7}BpE1`aAb>6@K% zVdLUeku##Lw)vA4n+W|$>eQhXN%&;c>){GnIM8d!K5jLf-&u^L8WdPva3|&ealoGp zkzrL-znq@BlvH42nk0Av^y|wIC)_KTRfF2E#fISI;Oj4K5BWUlJSP;nsclb2k&#sg zu>9=CAPKp()`x@NXpGqD-A~6;d?43T&vNAe8e;j|ktaRM8eU$b10?HI8L2L$NwwCds4C}j1Q@WNa59@;Q$u49`_sX`zbFvF zg9AsM?fPDvx{z#}pWod9FFPu9b#e1_L-Nik26z^}lZP0x62#0)7=q`R)*WoLPocCfP`O*R-A#4G(fWcsnasoxmQW z2M^bHj!z3(i5h(p$*$M+eCvW7Aa^HMtuB}* z8e11T<~yNJnT5`i-TV?^y7#6hm}F-%wD@(6t9{!plO(KHP(aD6OS_*ow`(jc>uCITA)SV#A-p5dZT9d%Tk`~5BVo||>MeO%r3 z;f5Oo7G2GC_4G?n4S(>C(C{vkhiW*^^73SLmuh>Q!CuK$qeLgCl0M`GB{Y|gDE&o? zc=$xt^VmFqX;QL?ctPHOf)ljYdlM>2k(&UKpL%KCa*R#XIQ>VGYBE`iaFkJ`o~)6y zU4)5cJ#crJSW>A9N0cIRd`;FgH+3MX-Y}@r}g=P!6d} zFmdS*^oS*D8Tj}yVr-Ne(e(Pa#_`iE!(C|^VL3Q{f6*}F59BYuPR%XgZI_F#mVX@a zRwx?c_CuI?S;h=a;~6=ewz7LXS90b3g>Hg<+#+1H8vT1Y5?r3MGlu`n=-#2fV-Hr( zI`ivg1*yA~C2gL09e(k|{4RmV8PNKL6_mcI$SG;S`0i!V4BG^+OREi^PG�+4rJi z=0P(z>W;-mdkF(jZIEajkQ#VDtq)^Q&8CcJBF^<<9&T#;utPcd7}#>-<;3kRydamw zj;BXM4QM@>_ziP^WQpD9P8ulx8hAoUyjm1=ZW-V_?9BpthQlmAoas^NfymTjs#_kk z7&n(`a2I_z?^%_v+M=Pq(4Q!G3A!ou;U$S*H4FsK4r>I)`+cymLED^ydmT?*Xm`x( zy66S$VjzIO&eNU4U!NG}6;NzP3vP7#<4{4u&(uw51>C~bj{%*$g*^wc zDj{tdo$iUEEpV%-^Hx7gIJr(}S*W7e_dT*3PM5k@IOjbaBYd!0hHAf1hf-}G(sy-1 zwHFfi4fl%D>|D$DCl0TXr|Mks=uY~rH3kwZ#f^5zg%#wAT zT(1-45KGx*SwrA3ImOC{A~MLRB&jfFwc2hyh3)fH)H0Fi2?Xl3X8SO5Y@&j5Htz}) zDGc(IiC#gSh0kEDUzW7*l|V?Zq{i!K8O&(STcE_8UM!}&n&6ObLOHRaJd3?DV!8&^ z5M>Lt)~1z=kKA*Vo?4gb$rP1?K~8m;JmNCyL@n~@qvHaE>x8DUH@DRUHQ7?Z-p$2I zr~t)(fY%b!aCz-$<(D6v_*Ur0mI^-@+j)!vnn@|M6A%D&p8jGxG=CrYWO4 zJtSbt2ee+1pW0EqUYyMFRe{JC?(J{~bVWYlRHfOoO_`@%!;XX?+r}S5%IUyb)TUQXkSIHU!I?zIJtenZizpohhqsB8Vk zO$d6@zDBKC!cW8q?k*9E^P`+w9`K5Jt`k5JAcmTK6FPqBLn*tM>B zqi!zvY?eAb$jl?RP4+bjxWsj781ROd}U2#lqM@t1Rc<>0D{~jB@<~oHjTUwFc(!M`13XE+_fvJlX+;gJQ<~yI&fao?!j-auqRFntqqyc{ z>BpDfcL&>C_x8XZHRl_jWTob|AHa<-oOlZ0iu#Y9<~gl%tSI)%_d!5HsYm3`qC&q9{92lA z=!;WllWp}68s_{_+PYh!tHo5efcG9*IvK{=_-k0ivZ3i3qtP*&Mp#HoZm+E;Bt+pi zUwU)F>Q4TR77cN>Ym1?f^}*8Cw$a+woYA`VT6ZrsW7O%mB6L)%pv!Ejy;bpg=09F0_7GN5|s#ox`%ERyIT z^i{o@tR)lFiT>EE)BY}~=zcuMq&dZ=Am2JYky36fo3?^dQ3inj)cg?v*OU!<;#6|f zYSD}j>m=9$ONm3-q$tGYbzTzKddYHu@Cwal9SsJ*?=p0B zr8CCwsXB?9E)3}=9#yik70-W4qMaEu4Ap1mdH7G&On_c&u+Yq3aMr54NS|ol7YM4! zNI->&DW*=b-w8gPEFfFuWh(;lHn0HLHc;$Es0)c}*5}0o+Dy1HCyS&nz6h?mJajry zLs!U7^H4mqz()-dJgH?Gk!mO8=I>PN0X}G0m6M86?uzFEXAB5vtL0}ixGs0v^Q-5c zoAMW(3NtLX6fzd}E;vmDS@KuVAGw5&0bM~nF(E{AKIVBRXgTXm6T8i*`*HO%(;l>8 z+lOv9IwG+iyU^P&rlwIv4MW6_fM;NB6WvgkH5li-7>DD~^%NGu-aI5Uu~K)2`G&c+ zy!XXPRsXN{zC51l{Qq}m`qoUE&_qoX+EJDgg)FH?o1H9KMnaZ@-p9R<$M1gcx4(6Ke9q_de!X7L z?G-S4UwryLt9<0Ls$#T}e=f63aJErz@m3-Gmo!a4il5x6oVUL=t z6>}vm)?Y1$ttltPQj;obHhY9K3XJn#fMKNkzU#snDhv01SWaE-$zAisoj@+0KmJ(^ zC(3wbg5B-(kF5_5#$wZt#H{k1eI^hkUjLKU3XOGp_~n0WopEvyBXmiWC)rIna9zDhPCB6X3_3&XEsre5wE?`C^VV&Lg2d47xW62$7m`sjVAP*t`K!X@_RH_94cB1 zv&3Wb$u$MOcwfu`?oPd{7uWuRReE4Y8u4qo`(>8h8HX)`Tl8b(owQdt;D%x>cSkA3 zand@B9HVcoYKcor4aDt_|KUX#zEgvnTldDsv5XX=xAL~OPTNxpa*>YY)uOd3!P+a@ z-7}t5M3Ym+TC+qM@eN z#&oxnXptI6O>8SR0CTV`=e!2BZR(`9y`f?TL57$1T~pZ*_8$CQoWlw7IJs6ckWeRkT?DZJGwzc1&I-67^*r+T8T-DqNE^ zYi_A0)W+V|$Thvdk^3%fvXy(HStr+iGUZ@>_`{2S4M$c+MrZz`V?n~n7T%&(Bg4@G z_x3XfqFzY)dpJ%B=kl!$&+e$G=~Y{lETDK%M!4)qok>zm{?INXAxRs|p2f<$_k%O9 zSS_}_bMV0jri&)M#zio8QjGMgTZL zi~O0v<)1-Q@O}MtX9&#mf8&o6KRwf9`*j69jQX&I1`Z)J;M2l~F~tt;E4n=~u6$Me z302MjBNDp`?bD)-YJ<;V?8q@P;eBVFa0pij#jA_GwspYJ<*7~TS zS@u*6vcQbdbM3ofnmX=U7Z>$O*$s&IUxONvX@Epl1H8GeQmTK&47Jr0WR$;N%S@jS zHSAF7B}Y>WDSrADqxz}56*GkZW!eJ~m#jV)b>fpy{j;c=BoooQf5J>$pA4AAt76Q= zGgO7S83~sfR$FscSSV8qY{_GZybPdM5qR=#E&!-9r=K~uJvOqoX!_y4oINqyB!64v zzPRLc8b^*_GwkC_#1HymUy`jW!Bf%cqU<8W;|lK|4VZlM+zBjZ+Zz=NSVYws{$t3AyZ)L53jj=%im zCR+K**OmDu{mA$wa%P8K^HJK?m2Y#8e$u}qszTmO%<-pd*rRLJ&o8-b7j_s}-L)t3 ziq1NJyh?a9)|@X<=2(RcPC2|dWx0hEmPdR@roAJ=Tyn+KA;?|F_yd4N1G52k_F zfM6%MQ30G3iQYM*H$rp+nfv6MHN>_@0r~JvuMIknG`N$r34FNe)E?>L1Y!AaR^C6g zS0#t%d5AB-oJO-*mX%5Z$L$X)$~#wp2D#hR{yuQFKX*HDXBF((&Q2ZdupU4zxMlA= zse_3+K5H-)EhNlrj9L&k{m|jD2lHdEEmdl8<-Gw(TYP^Z4akqU>(u3`o!%hB8zcYJ zA2*zt>_)M9+}O${GjED4*6$YKi%}DuF%|Fg;6w|%=5-a?!B@bPR?2lK_5b*c8$x3# zbiFKBYmpzj8oftOFc8@f#7Cx0wyVy{3dEJqll+u}G_+!h!)`Pj6=JS;=U)7nm#5Nu zw$D$QxMn9odImo@pnG_9K_fha)+Alg+2@V9Seo^n&hgZOs=?~ZH3e5^`l-@YgI9V; z4zZTu5)r$pN4N~Zh0H5v0wsdA3!i@?OQ_PYbxq zGZGh)F&n!NuHAr5lBo?4X|EccJYB6iT=BsJ_qsZW{ruhh7*Y+8jqW=1AJ}eCxHY0K zG~@hP+C7nTX^$FAq&hMR;sVoYlF}d@h^c7yTC?Gr!Tn+#1Z}yL8D2{7I3~AVe$N5Z zoK3eR*BV8iY`SZ>j?m-lL67VHfrM;e_p&*NEsnh#*F~snaZ16pDusyTxk~q6uuDp8 zHEtz@bQlzVjg23VX;hDl3oP}X;PZTuw()(mpnh(#oz&x>+ivNpghkV8i2O&8nF)Lu}o+9Agupz_z>0g4d)Gnq)s ze0y>Sfi(_H#DGWFBuM?mlDp0UYcF6Sncy+{SbAvV^%x6(yb1H+RD3XfO%+$VCx=X1 z^L%hYpP)uZZr6i7RsLlXQ_L#WrI%yp%Jc4`f|lDj>%vw}4TTsGu#WjH&J`D^RfEXm z*Z6i-_&vf;gnR{VPbO?i%ppj30a{5Mm18j~6hry2Ui=%?YWTlsp z6#p8=IombIqT=Fw(wS#_z)GjVN#7J~rWurENf_K0VOzgfE5kJ0KG9RZG)rHK!+Q)D zkYu-Jw1DKK+pm>DR;nxq??(u^VI-lX|4fv}EOuzHVkIl{dHork?37IrvpH2PL2D%~ z^L}PJ+1Ev~f7;(8;7;S-#zo$m)O)QXl_z~N41!p=SPu8m6~jXH;xLh;J5!_U&e&Ox z3fNk-du?#^E7uAq=$pnA-$qbj-ASHfO&g4!WmAjIX_?yf+H3=+F(U*f8B_Up`+a&? zWu%U^PG|q?)c;F@TGz*@Wu)$y;amp)>4pd35fQZ`j!c_J$({@)lD^WD<4&h+ySYK? z+x~=fuEW`c8)AaCaXtj*Tqc-C$Ov4ri*x9=#7}TO)NC-bZ2L-Yr>S%Qbj`jf34W|8 zEk&EvrcjhU*b!yq5uNm9dUXMJ#SATn?r|Z?Vb;j5iQP)irD=+|t9$!bK2pLN6K*+` z@u&-W&E|hMpLe~XZtkwOZM>bxsNWazH>ZQ1U9DCf@fr48QT%H5hAf8O>($58zcIb; zbz=i~tDTU2=cN?^M~^&@(cz|ZFLEW{!;FqEI#$S69H3OG+;y6%{OxTbJQ}crkyf&7nnU{@n3Y3T?7bJ>TwESMg^@M5G#c^FAiB7^DA>T9-XL zmRfRXm?2u=Gq^<`Wv}A=TNfXB;r+yg+0+Lo;*A170(TT-8coyE)(ek49+=)*X~Rus zM%~H|FWv(&z93xqf3yc*?u5{8->({ecGTEQndsz%Sa)cXkwb#$h4fL;Dt*j-@d{YS zS3n8|Tb=1k9yrSe#zevA%`s>UFF4`Iw=*z1#|ycy*aIKsM7$ z6Fr66CCzoX4edTtw>u;cf4%nN*L=RUaQa73s|ZQ{+lqPHw|oi+`P)N^Wg7d{1T^Ps zaJ*QG%3NX$yk2gSfXg zPb3#t(ZdV_y|MY zCz_*ru#5=1le;=-p>-k-NnxBFUh_cw7D01<0xmh5=-05iE;ZVwH*zVg1qE$Rc*JV)<9u{bakQStVmJwog$^)5p|bYqyw`&nB!DsbnAUCT&|rAX=1iTHbpP|(&U6Dq5WC;_MiwP|NT_9J~Pwh+Kq+| z8V-P^^F37CD|@FC0&yZfhZ=urb;so*`gRS~eAZ^D#E^V}>5`vbU)g9Fn;bF4g<|4ME7 z>~TZ`U|{p12LYqX{8&lX5^lvc4Od2Yv9*FFR`7ZpBb}+^>+9dFv zD+D!UX=OtA8&a#nz;utonn(&ioxNVT1rMX|DK+Cc?8)ErVm|+y{YH)t1$AVDzHRLK zRpF3=y%PLLhvUyo)?VF?c^hXJ6ViG!2w!@k0z(uZ6?l)PU>Jz+m_5RJbmtLctIQvd zwa&=z>VR9(t(GvEh4qLl#veEBA4sQ8r^ra|z{Z~<+wiI#+G|{&{*_S-HVK-8woy!h zp7^E+QM1=sr|rbrEtc^X`y1U_($8#{O!7b6oft4~3fm5_;8*fXXNpr;$8T;3?h2ia zvWW_q4O~B8+F(jZGXJ1yD1!UAHGaR3zW9!+1z)?;tOC7~5xnG@sg9137m{>vF-iEtGWKbj1Cc%-JfeFi?`XT?6)&o1o;NI^Q!r7X`CSnv2Y(O2`TU#yZvoOQw?HiOj}`Bi)8TT$pMcR-sIqdu zLe*P+Wl(!PENM&S^Q`DsB033IW*5BYyV$H;5n4ILjvttOY?SFv&U_0xhnv!_moS&P z-qYeB*v1L77yLY17W{%}h4#$_Bdw3RiZTqYX1lm-iWc?!1+VZnWk6xz=7hq+)Fsba ziQU&|NY{RQ^MrM07|{!$jLYGAH$Fv2{cStI0d#|`dzJyxt6@geC5Tl?{^Dlu*@%Vi zHP0a~M=26Vi#h11+l>D>;79f0&JN5nvR(5(X_P?m)eed+eL5NZ13&vU#VaVbfo^Y` zxrmBmDw53(OaiBkkfE2o1cBugkEW!+^lhg+#wlAPuRxPnglcd*(i>Gb5|N*v8kBmV zX5x=-zQ1pKk7f@wzY@7-Q6WB8=5G6uDY1L=`u3$v9&kDIwk9V?nHKEmVM1EYH{c_- zpV6gAS(G2x+i~j$DYeuI)nxJ3Nq9f zl}gQ|^o(bbB_HdX7?#;c6|EP%?U!G?1r4$jitva*{%-_u4Y)@b=zPolxo4TT7`RGu z?vuH@48U{5Uy$I;&*~U@ZK?1sGuIhmR1UkhY;4yH#eO`(4dM3mOwQ_F#Fi0kbPGof zvohRT=b7{#i7aq$9>bR-L6Ymz`rj}n}=NYfS5{VBGB*CfBhn#wJBG+dgOItYb(Nuh-m%5QkHoljBhpnO)@M2$Xmgu)n6#D*@l|{R=e$<_~k@TTB zfzmX|Wt3glEcG)DMXA#Ufdo^Zw~+8*nxrYh+h0t;8k&R-dYEMsVne|Ezge4{AWx@L zhjt6 z+4ng_KOm{;Hp@sHm@+cu^c3pA)A9u4`b2?(M*#uB@Q`+EGBTx|Jl~OJNc=Z6nu-iF zVx3RFNIs?7^fSdNt?QcR=AJ`6XG!X34D&gx4!O(yew{fCMzR3K6RR|ZOgPa_?`aTk zH8@Vd`A>${VLm*tcjqdA{EX2+mgb28je9z_nM>}$q=_%{w%~P-6L#DLj>nmZ^n!&0 z?I7(%>ea{Ulty(a9qp>NM#ZL=tu$D>LrjYelH?d(aqzT)-aD99p2J92r%X=1cspwz z5ctvdP})pHxF%tIyU|muV}Nlf80dsUhIC2lFy9^%yp9wd0@sz&l+TKd7Ffmupeo;( zL&<0jf+Wc=PT})24DMo+=CG;wLπ^nz2*l5=@i6Ye^x-NhPuIQB=-<8G&KN~&k+ zM=Ya5EP)M5qzYZzDDASgTS}KuZkX2^p5Exa0bq_8pxD9eNjfHI2EQA|YQE9wVWde3 znYBEI8+sT`ulSPnF>Y&m*UwRkK|Xy?Tx%LNq>Mue0FFpBo_t1V{h3vHBd(ZmMTdOg zs(yH`=wVLUS9&qU;W6gkE~QyrcV=IIm*AVlJbmVdO2XvaiRQ0s=qcOey0qPSrtp5_ zLhOUF2|j@1GQj@4CMa3^PCNT90}vx@+{Y9?<3?jSHKRxX6-x98?n4`Ir0rVHcL}&f z($pZda7ea#&KRh>R*4y|LhxpXu`h+ zivD2C``e-7-{Rwv{UQ0S^h3FYhh^(9BXkBkF3|2od0;uswu*-YHQczOL%%Htd;X7( zi_hM<@JdfJ|Uja+b|lLEcpt?lTZc4&3x zOw!n0ojME@`$dKo2jE@9z|3i#fJDbVZ1s*;?G?t&JqqjV{3kDoPbB)W9fYAPvFMm< z$a_Maw-^H!l1fT`gHQC^nnZFyHm%rw;O5qK`U`( z8iFByt96{|2n&MrflW5V>=2 ziz!s7kv?B-=9BqH#SYj)Mj4s64Ev!&;*V#l{p}6cg~Y6Tbyxh1NE6sNQcsB)jLohI zQ59bJB}3w$4F0Pw(x(RBh4TiKCiEAbyk{Zq*-gTiGSIY2S28NQOyC`GPYFI3L^TwH zoy2pPC}V;YU0`@{wxl0#gG8B~(UkovB<-)Y+ZBPfavqJb3MMOMo0QPqZ_;y!_;40) zOi7==&A9^aOQmz;%B~^R&{JWsmrv8DF6O!^E>5J{trG!?t3BW1oL}nD)iRoUYt>x+ zD`oDte(#Et%WSVE`bgf#Czz|j9FyfVY`DxAEuglQviLxueEUj>yL{|cXmr z+B;RJf5O$cH}O*x+P!!R?KmTo179;lwT-m3t)tCy4DJmzaTNq^mgg8mvl2qfvlvc` z9;R~qqRbhS(k#qg(}F|hf=f&99%A zy~T4p>T2Aj*!$Aim{G2P{0hC2jF)GKQ?>M?41j^^(X%KtSBH?JrOe$WX+w5XRfw!G z?2Uo_YCrzTFc9eqiVpfL?^9F&ZOPF=UwuI11{v$f>@IdXx;5P(2K)E}$k2pb_DpY= zEU(huE;8G|k7fx^$Vk0LqphhV%an;_WirhvBhO~Tj!Na82`mNL_ZssIWgaV8#G@_M z5nh;{NBOn}baT>lWg)B@PV*r#MfB8bHmmnK70@(T&IfQ^FBGuAf$I35_pOeMkD{|HvBC3 zWz=rrb@3u3JjS#^zy?o=(Ia&VWA_#>!%}}Rum|bOF0x^^?XFAFgq6DfGkq9>?uN$~ zo(#`DvaW=I^su?;uMn_g4?*bjbPw0%*;f_FM?{?4blq>f z;Og4NUvTI0&j*1FA_`JtB=|*#yA>k?IaDbowkfv=+;Ytpu1LE8Pb+A{OEh5*Orp@m zrrgp4=Iu&g2fS0E72W~9`ySRZZvxHly@T4~or6%7+fhO=vNY`INz=P(82AyIW(?qv zo`4FQfFdH8d@``~*P+zdnmP%Ff)W(Xi}G&~n`;8t?m23wPF<&J1#`3161;m4cdyZ( z@^0k7Z~~E93)Zfv+aE8MvV4k+M^)Q){$~6C+orJLpag#eQ;2frNb*D3D)r5ZZ);@f zvSs}-2D=vDN;GmTK_5P#Dk;IT2tib{iO3#)QTqJ#;_@XNLAg7)IS$d#%`^fGPEAv-YnxjSU(YxD!``&$*grH^r zVpuTS$w=F^yhkJkqT`qKlm8wQM%GwoPBucKrissMczc?E@ITOlGRkWMRA0S1M7x$} zzPM$ozK;FkV{^^qRD9y5Xth6Zaox5XlUz|ti%yxm1VpjnwUibw-H2M&v1yxpu?P{*H#aW+=t`Q1V^npx z3=olf899sX){+of_`?;Rr(p2pooNc#TLzIOLpMl>lX~A6Yhj&0kF#>938}QQ=E&)+ zEqZTkE+p+_g=9$M3XJCZI?;p*%dF)+xK;_Xd%#^?D@36lhzx{q&~FIa=#hh>0dudV z4-M@_Lq+llILdl;1}0H@$qk)^?)w2r=a*TDcC9?G zPjX-o4I>vEbchnfqy zz0Dgx!iB_il3LOxOG&{Gtbufm;DxoerMo^~J*GJSuxYQM=^@+6)cdof<4xWa_H<_Beu?#I#2H91%3utuLf*Jq zRwk7f{7@9ei)ELg=W|vEq-~ei(UKsBX<0w{?~%vQmilW(fC!$UXv+4ks)KJjLor6$ z!VzR*Bw&a3_*s7tTUr+Z4)XLGcU0~I|evVrvdVwsA(+fSGZ z*82=94Z;julES%-=txJ~5+L9ADup4`f(I6bvH&T+#A@GVk&TWGko;paFG9zp{1A3R z@H%I_lB%8D2xwGdZ%0`Kd&g%XrKQ5r9u4;%2*mpvS9Mwt7F4G9O`fT;g$7L3*WeW}Uc^c9Ha zSOq&X=0DWTSU>s&@O0!DFM6b4714mUx5ADt&g5xaFt#KVqTl+;SlDui1(+Yb)q%EZaf)&^2mBi!mA3qCLxq5+k@7qWIi(b(`k&4cwCQ!`ldd)8 zyO(ow(l#b+)IQgDIcAH-)? zFgyNHzfDsyr=xIjD`UZoC^o7PbyaeQkz*JPH%siILPC3%hCV&L=wKr76<+Zi`w!%f4s*3RS?YJMZdCZl^|aIxLPzA#hJnT8K~xrtcj6+GJNQ`4~Mw<+5Gr6OeOXqiU;hyaDUI z;5cs_^28as4J{Lyi7e3NmxQA1T=o%iu?ogG2H;QWtdwdu1er07#=1F)J)WBW)7Ih| zSkrLj@xpxT_XCi(0ZS#)Y4w*or)2_oI#96$QMi?hu*pHK$+@hoZ+58%Hx zQaKorN=D4|S+T_aqEHrN>f*FQmfteKFwU#{`%=cDJZZFl;tL5Qh}ts($I*71m(w^6m`eb|Fg&8W2iC6H0QE{IZ?{8JZ6nK|EAH z#Iph__ss&7ivbdxxc6_1$iG?2=O+ z99a^wmwzbF5J!N(fXDkfu8JL61tLBKVL&!$E0BOY>{)66E#aL6&o`IYC~pMdk^6i2 zb1O%A;$i7j-V@jqa1A3098VcJ6sc#$+0!QFs*t1109V8}9YMrwnz5BnY$d1G2~j3= zqO$y)F`0Eprc(Bh`2w^O07FyVI3VDn0w^*Rt=3X}xzv1kL34}m1Fete{Y`d3h>Ht| zU}EC~(=||_5^oHa3e{kUEldwy4v8SiIR?Bbl8M+YqR1y0o3^+pgWTe1>@HQ&nOMjX zvR0nx++H(#GVoA9-(dt#RF`w8GP!n&yW|9HAw`k;^vn69`v?X5B{uUx+PSkQjM_|O3a4rCN zhIpPM@@8e1cSJ$V~lg6!_&1E`z2 zeb78!IY=>9&zr6#N|C zc_E{~bSr!q$hM+ITOK=8fGpVCWEF-z`ma0K;f~?leDyKdz61+4uE;C0o>P=eyq3W` zE>jWre^_Am|Ix>75pKW9dtFxgw^B+%x{zp@YHOYZ+-X*c2$v$I1d0J@x&=T|@dr^uFVXOFK^|26S&DZm)Q z#Q$tkdj9z37cYO!QACZ`T{M#{1inAOzK~8aO*O+tq}V)+j%LN z(f+Tp7|$DB_+n}4rn^19KTFMs|N z&ipAY88($l{3-GBKR4@0B1Dt^Z{IA3*DD{cI$?5<^2H9~X8NPS5uN?4oj!K-56R8J zYTk|wct*TtOvg1}nHSME6YX}1Ci7rV44E$9UcR~2-5>SVCzrltJ4l2TatjpwJsj6g zY-#^-{_IYj-=CWQJflmm*neK=|Gdz@owabl{pa-i{S^6A81R2e#iw}f|I2mt<{tj0 bg@F6aOF6>x_DNsBzmvzckL76mc;$Zpg=BtY diff --git a/src/15-tcprtt/README.md b/src/15-tcprtt/README.md index dae903d..e87310b 100644 --- a/src/15-tcprtt/README.md +++ b/src/15-tcprtt/README.md @@ -39,10 +39,10 @@ const volatile bool targ_ms = false; /// @sample {"interval": 1000, "type" : "log2_hist"} struct { - __uint(type, BPF_MAP_TYPE_HASH); - __uint(max_entries, MAX_ENTRIES); - __type(key, u64); - __type(value, struct hist); + __uint(type, BPF_MAP_TYPE_HASH); + __uint(max_entries, MAX_ENTRIES); + __type(key, u64); + __type(value, struct hist); } hists SEC(".maps"); static struct hist zero; @@ -50,43 +50,43 @@ static struct hist zero; SEC("fentry/tcp_rcv_established") int BPF_PROG(tcp_rcv, struct sock *sk) { - const struct inet_sock *inet = (struct inet_sock *)(sk); - struct tcp_sock *ts; - struct hist *histp; - u64 key, slot; - u32 srtt; + const struct inet_sock *inet = (struct inet_sock *)(sk); + struct tcp_sock *ts; + struct hist *histp; + u64 key, slot; + u32 srtt; - if (targ_sport && targ_sport != inet->inet_sport) - return 0; - if (targ_dport && targ_dport != sk->__sk_common.skc_dport) - return 0; - if (targ_saddr && targ_saddr != inet->inet_saddr) - return 0; - if (targ_daddr && targ_daddr != sk->__sk_common.skc_daddr) - return 0; + if (targ_sport && targ_sport != inet->inet_sport) + return 0; + if (targ_dport && targ_dport != sk->__sk_common.skc_dport) + return 0; + if (targ_saddr && targ_saddr != inet->inet_saddr) + return 0; + if (targ_daddr && targ_daddr != sk->__sk_common.skc_daddr) + return 0; - if (targ_laddr_hist) - key = inet->inet_saddr; - else if (targ_raddr_hist) - key = inet->sk.__sk_common.skc_daddr; - else - key = 0; - histp = bpf_map_lookup_or_try_init(&hists, &key, &zero); - if (!histp) - return 0; - ts = (struct tcp_sock *)(sk); - srtt = BPF_CORE_READ(ts, srtt_us) >> 3; - if (targ_ms) - srtt /= 1000U; - slot = log2l(srtt); - if (slot >= MAX_SLOTS) - slot = MAX_SLOTS - 1; - __sync_fetch_and_add(&histp->slots[slot], 1); - if (targ_show_ext) { - __sync_fetch_and_add(&histp->latency, srtt); - __sync_fetch_and_add(&histp->cnt, 1); - } - return 0; + if (targ_laddr_hist) + key = inet->inet_saddr; + else if (targ_raddr_hist) + key = inet->sk.__sk_common.skc_daddr; + else + key = 0; + histp = bpf_map_lookup_or_try_init(&hists, &key, &zero); + if (!histp) + return 0; + ts = (struct tcp_sock *)(sk); + srtt = BPF_CORE_READ(ts, srtt_us) >> 3; + if (targ_ms) + srtt /= 1000U; + slot = log2l(srtt); + if (slot >= MAX_SLOTS) + slot = MAX_SLOTS - 1; + __sync_fetch_and_add(&histp->slots[slot], 1); + if (targ_show_ext) { + __sync_fetch_and_add(&histp->latency, srtt); + __sync_fetch_and_add(&histp->cnt, 1); + } + return 0; } ``` @@ -95,20 +95,15 @@ int BPF_PROG(tcp_rcv, struct sock *sk) 这段代码是基于eBPF的网络延迟分析工具，它通过hooking TCP协议栈中的tcp_rcv_established函数来统计TCP连接的RTT分布。下面是这段代码的主要工作原理： 1. 首先定义了一个名为"hists"的eBPF哈希表，用于保存RTT直方图数据。 - 2. 当tcp_rcv_established函数被调用时，它首先从传入的socket结构体中获取TCP相关信息，包括本地/远程IP地址、本地/远程端口号以及TCP状态信息等。 - 3. 接下来，代码会检查用户指定的条件是否匹配当前TCP连接。如果匹配失败，则直接返回。 - 4. 如果匹配成功，则从"hists"哈希表中查找与本地/远程IP地址匹配的直方图数据。如果该IP地址的直方图不存在，则创建一个新的直方图并插入哈希表中。 - 5. 接下来，代码会从socket结构体中获取当前TCP连接的RTT(srtt)，并根据用户设置的选项来将srtt值进行处理。如果用户设置了"-ms"选项，则将srtt值除以1000。 - 6. 接着，代码会将srtt值转换为直方图的槽位(slot)，并将该槽位的计数器+1。 - 7. 如果用户设置了"-show-ext"选项，则还会累加直方图的总延迟(latency)和计数(cnt)。 ## 编译运行 + eunomia-bpf 是一个结合 Wasm 的开源 eBPF 动态加载运行时和开发工具链，它的目的是简化 eBPF 程序的开发、构建、分发、运行。可以参考 下载和安装 ecc 编译工具链和 ecli 运行时。我们使用 eunomia-bpf 编译运行这个例子。 Compile: