This commit is contained in:
Officeyutong
2024-02-22 13:14:00 +00:00
parent 403aff5b66
commit 55d5e641bf
47 changed files with 1483 additions and 1918 deletions

View File

@@ -1,141 +0,0 @@
# SPDX-License-Identifier: (LGPL-2.1 OR BSD-2-Clause)
OUTPUT := .output
CLANG ?= clang
LIBBPF_SRC := $(abspath ../third_party/libbpf/src)
BPFTOOL_SRC := $(abspath ../third_party/bpftool/src)
LIBBPF_OBJ := $(abspath $(OUTPUT)/libbpf.a)
BPFTOOL_OUTPUT ?= $(abspath $(OUTPUT)/bpftool)
BPFTOOL ?= $(BPFTOOL_OUTPUT)/goroutine/bpftool
LIBBLAZESYM_SRC := $(abspath ../third_party/blazesym/)
LIBBLAZESYM_OBJ := $(abspath $(OUTPUT)/libblazesym.a)
LIBBLAZESYM_HEADER := $(abspath $(OUTPUT)/blazesym.h)
ARCH ?= $(shell uname -m | sed 's/x86_64/x86/' \
| sed 's/arm.*/arm/' \
| sed 's/aarch64/arm64/' \
| sed 's/ppc64le/powerpc/' \
| sed 's/mips.*/mips/' \
| sed 's/riscv64/riscv/' \
| sed 's/loongarch64/loongarch/')
VMLINUX := ../third_party/vmlinux/$(ARCH)/vmlinux.h
# Use our own libbpf API headers and Linux UAPI headers distributed with
# libbpf to avoid dependency on system-wide headers, which could be missing or
# outdated
INCLUDES := -I$(OUTPUT) -I../third_party/libbpf/include/uapi -I$(dir $(VMLINUX))
CFLAGS := -g -Wall
ALL_LDFLAGS := $(LDFLAGS) $(EXTRA_LDFLAGS)
APPS = goroutine # minimal minimal_legacy uprobe kprobe fentry usdt sockfilter tc ksyscall
CARGO ?= $(shell which cargo)
ifeq ($(strip $(CARGO)),)
BZS_APPS :=
else
BZS_APPS := # profile
APPS += $(BZS_APPS)
# Required by libblazesym
ALL_LDFLAGS += -lrt -ldl -lpthread -lm
endif
# Get Clang's default includes on this system. We'll explicitly add these dirs
# to the includes list when compiling with `-target bpf` because otherwise some
# architecture-specific dirs will be "missing" on some architectures/distros -
# headers such as asm/types.h, asm/byteorder.h, asm/socket.h, asm/sockios.h,
# sys/cdefs.h etc. might be missing.
#
# Use '-idirafter': Don't interfere with include mechanics except where the
# build would have failed anyways.
CLANG_BPF_SYS_INCLUDES ?= $(shell $(CLANG) -v -E - </dev/null 2>&1 \
| sed -n '/<...> search starts here:/,/End of search list./{ s| \(/.*\)|-idirafter \1|p }')
ifeq ($(V),1)
Q =
msg =
else
Q = @
msg = @printf ' %-8s %s%s\n' \
"$(1)" \
"$(patsubst $(abspath $(OUTPUT))/%,%,$(2))" \
"$(if $(3), $(3))";
MAKEFLAGS += --no-print-directory
endif
define allow-override
$(if $(or $(findstring environment,$(origin $(1))),\
$(findstring command line,$(origin $(1)))),,\
$(eval $(1) = $(2)))
endef
$(call allow-override,CC,$(CROSS_COMPILE)cc)
$(call allow-override,LD,$(CROSS_COMPILE)ld)
.PHONY: all
all: $(APPS)
.PHONY: clean
clean:
$(call msg,CLEAN)
$(Q)rm -rf $(OUTPUT) $(APPS)
$(OUTPUT) $(OUTPUT)/libbpf $(BPFTOOL_OUTPUT):
$(call msg,MKDIR,$@)
$(Q)mkdir -p $@
# Build libbpf
$(LIBBPF_OBJ): $(wildcard $(LIBBPF_SRC)/*.[ch] $(LIBBPF_SRC)/Makefile) | $(OUTPUT)/libbpf
$(call msg,LIB,$@)
$(Q)$(MAKE) -C $(LIBBPF_SRC) BUILD_STATIC_ONLY=1 \
OBJDIR=$(dir $@)/libbpf DESTDIR=$(dir $@) \
INCLUDEDIR= LIBDIR= UAPIDIR= \
install
# Build bpftool
$(BPFTOOL): | $(BPFTOOL_OUTPUT)
$(call msg,BPFTOOL,$@)
$(Q)$(MAKE) ARCH= CROSS_COMPILE= OUTPUT=$(BPFTOOL_OUTPUT)/ -C $(BPFTOOL_SRC) goroutine
$(LIBBLAZESYM_SRC)/target/release/libblazesym.a::
$(Q)cd $(LIBBLAZESYM_SRC) && $(CARGO) build --features=cheader,dont-generate-test-files --release
$(LIBBLAZESYM_OBJ): $(LIBBLAZESYM_SRC)/target/release/libblazesym.a | $(OUTPUT)
$(call msg,LIB, $@)
$(Q)cp $(LIBBLAZESYM_SRC)/target/release/libblazesym.a $@
$(LIBBLAZESYM_HEADER): $(LIBBLAZESYM_SRC)/target/release/libblazesym.a | $(OUTPUT)
$(call msg,LIB,$@)
$(Q)cp $(LIBBLAZESYM_SRC)/target/release/blazesym.h $@
# Build BPF code
$(OUTPUT)/%.bpf.o: %.bpf.c $(LIBBPF_OBJ) $(wildcard %.h) $(VMLINUX) | $(OUTPUT) $(BPFTOOL)
$(call msg,BPF,$@)
$(Q)$(CLANG) -g -O2 -target bpf -D__TARGET_ARCH_$(ARCH) \
$(INCLUDES) $(CLANG_BPF_SYS_INCLUDES) \
-c $(filter %.c,$^) -o $(patsubst %.bpf.o,%.tmp.bpf.o,$@)
$(Q)$(BPFTOOL) gen object $@ $(patsubst %.bpf.o,%.tmp.bpf.o,$@)
# Generate BPF skeletons
$(OUTPUT)/%.skel.h: $(OUTPUT)/%.bpf.o | $(OUTPUT) $(BPFTOOL)
$(call msg,GEN-SKEL,$@)
$(Q)$(BPFTOOL) gen skeleton $< > $@
# Build user-space code
$(patsubst %,$(OUTPUT)/%.o,$(APPS)): %.o: %.skel.h
$(OUTPUT)/%.o: %.c $(wildcard %.h) | $(OUTPUT)
$(call msg,CC,$@)
$(Q)$(CC) $(CFLAGS) $(INCLUDES) -c $(filter %.c,$^) -o $@
$(patsubst %,$(OUTPUT)/%.o,$(BZS_APPS)): $(LIBBLAZESYM_HEADER)
$(BZS_APPS): $(LIBBLAZESYM_OBJ)
# Build application binary
$(APPS): %: $(OUTPUT)/%.o $(LIBBPF_OBJ) | $(OUTPUT)
$(call msg,BINARY,$@)
$(Q)$(CC) $(CFLAGS) $^ $(ALL_LDFLAGS) -lelf -lz -o $@
# delete failed targets
.DELETE_ON_ERROR:
# keep intermediate (.skel.h, .bpf.o, etc) targets
.SECONDARY:

BIN
31-goroutine/go-server-http/main Executable file

Binary file not shown.

View File

@@ -0,0 +1,34 @@
package main
import (
"crypto/rand"
"fmt"
"log"
"net/http"
"os"
"strconv"
)
var http_body []byte
func handler(w http.ResponseWriter, r *http.Request) {
w.Write(http_body)
}
func main() {
args := os.Args
if len(args) > 1 {
body_len, _ := strconv.ParseInt(args[1], 10, 64)
http_body = make([]byte, body_len)
rand.Read(http_body)
fmt.Println("Body set to", body_len, "bytes of random stuff")
} else {
http_body = []byte("Hello,World!")
}
http.HandleFunc("/", handler)
fmt.Println("Server started!")
err := http.ListenAndServe(":447", nil)
if err != nil {
log.Fatalf("Failed to start server: %v", err)
}
}

View File

@@ -1,7 +1,7 @@
/*
* This code runs using bpf in the Linux kernel.
* Copyright 2022- The Yunshan Networks Authors.
*
*
* Modify from https://github.com/deepflowio/deepflow
* By Yusheng Zheng <1067852565@qq.com>
*
@@ -17,359 +17,44 @@
*
* You should have received a copy of the GNU General Public License
* along with this program; if not, write to the Free Software
* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301,
* USA.
*
* SPDX-License-Identifier: GPL-2.0
*/
#include "vmlinux.h"
#include <vmlinux.h>
#include "goroutine.h"
#include <bpf/bpf_core_read.h>
#include <bpf/bpf_helpers.h>
#include <bpf/bpf_tracing.h>
#include <bpf/bpf_core_read.h>
#define NAME(N) __##N
#define HASH_ENTRIES_MAX 40960
#define MAX_SYSTEM_THREADS 40960
struct sched_comm_fork_ctx {
__u64 __pad_0;
char parent_comm[16];
__u32 parent_pid;
char child_comm[16];
__u32 child_pid;
};
#define GOID_OFFSET 0x98
struct sched_comm_exit_ctx {
__u64 __pad_0; /* 0 8 */
char comm[16]; /* offset:8; size:16 */
pid_t pid; /* offset:24; size:4 */
int prio; /* offset:28; size:4 */
};
struct {
__uint(type, BPF_MAP_TYPE_RINGBUF);
__uint(max_entries, 256 * 1024);
} rb SEC(".maps");
// struct ebpf_proc_info -> offsets[] arrays index.
enum offsets_index {
OFFSET_IDX_GOID_RUNTIME_G,
OFFSET_IDX_CONN_TLS_CONN,
OFFSET_IDX_SYSFD_POLL_FD,
OFFSET_IDX_CONN_HTTP2_SERVER_CONN,
OFFSET_IDX_TCONN_HTTP2_CLIENT_CONN,
OFFSET_IDX_CC_HTTP2_CLIENT_CONN_READ_LOOP,
OFFSET_IDX_CONN_GRPC_HTTP2_CLIENT,
OFFSET_IDX_CONN_GRPC_HTTP2_SERVER,
OFFSET_IDX_FRAMER_GRPC_TRANSPORT_LOOPY_WRITER,
OFFSET_IDX_WRITER_GRPC_TRANSPORT_FRAMER,
OFFSET_IDX_CONN_GRPC_TRANSPORT_BUFWRITER,
OFFSET_IDX_SIDE_GRPC_TRANSPORT_LOOPY_WRITER,
OFFSET_IDX_FIELDS_HTTP2_META_HEADERS_FRAME,
OFFSET_IDX_STREAM_HTTP2_CLIENT_CONN,
OFFSET_IDX_STREAM_ID_HTTP2_FRAME_HEADER,
OFFSET_IDX_MAX,
};
// Store the ebpf_proc_info to eBPF Map.
struct ebpf_proc_info {
__u32 version;
__u16 offsets[OFFSET_IDX_MAX];
// In golang, itab represents type, and in interface, struct is represented
// by the address of itab. We use itab to judge the structure type, and
// find the fd representing the connection after multiple jumps. These
// types are not available in Go ELF files without a symbol table.
// Go 用 itab 表示类型, 在 interface 中通过 itab 确定具体的 struct, 并根据
// struct 找到表示连接的 fd.
__u64 net_TCPConn_itab;
__u64 crypto_tls_Conn_itab; // TLS_HTTP1,TLS_HTTP2
__u64 credentials_syscallConn_itab; // gRPC
};
#define GO_VERSION(a, b, c) (((a) << 16) + ((b) << 8) + ((c) > 255 ? 255 : (c)))
// Go implements a new way of passing function arguments and results using
// registers instead of the stack. We need the go version and the computer
// architecture to determine the parameter locations
static __inline bool is_register_based_call(struct ebpf_proc_info *info)
{
#if defined(__x86_64__)
// https://go.dev/doc/go1.17
return info->version >= GO_VERSION(1, 17, 0);
#elif defined(__aarch64__)
// https://groups.google.com/g/golang-checkins/c/SO9OmZYkOXU
return info->version >= GO_VERSION(1, 18, 0);
#else
_Pragma("error \"Must specify a BPF target arch\"");
#endif
SEC("uprobe/./go-server-http/main:runtime.casgstatus")
int uprobe_runtime_casgstatus(struct pt_regs *ctx) {
int newval = ctx->cx;
void *gp = ctx->ax;
struct goroutine_execute_data *data;
u64 goid;
if (bpf_probe_read_user(&goid, sizeof(goid), gp + GOID_OFFSET) == 0) {
data = bpf_ringbuf_reserve(&rb, sizeof(*data), 0);
if (data) {
u64 pid_tgid = bpf_get_current_pid_tgid();
data->pid = pid_tgid;
data->tgid = pid_tgid >> 32;
data->goid = goid;
data->state = newval;
bpf_ringbuf_submit(data, 0);
}
}
return 0;
}
struct bpf_map_def {
unsigned int type;
unsigned int key_size;
unsigned int value_size;
unsigned int max_entries;
};
// Process ID and coroutine ID, marking the coroutine in the system
struct go_key {
__u32 tgid;
__u64 goid;
} __attribute__((packed));
// The mapping of coroutines to ancestors, the map is updated when a new
// coroutine is created
// key : current gorouting (struct go_key)
// value : ancerstor goid
struct bpf_map_def SEC("maps") go_ancerstor_map = {
.type = BPF_MAP_TYPE_LRU_HASH,
.key_size = sizeof(struct go_key),
.value_size = sizeof(__u64),
.max_entries = HASH_ENTRIES_MAX,
};
// Used to determine the timeout, as a termination condition for finding
// ancestors.
// key : current gorouting (struct go_key)
// value: timestamp when the data was inserted into the map
struct bpf_map_def SEC("maps") go_rw_ts_map = {
.type = BPF_MAP_TYPE_LRU_HASH,
.key_size = sizeof(struct go_key),
.value_size = sizeof(__u64),
.max_entries = HASH_ENTRIES_MAX,
};
/*
* The binary executable file offset of the GO process
* key: pid
* value: struct ebpf_proc_info
*/
struct bpf_map_def SEC("maps") proc_info_map = {
.type = BPF_MAP_TYPE_HASH,
.key_size = sizeof(int),
.value_size = sizeof(struct ebpf_proc_info),
.max_entries = HASH_ENTRIES_MAX,
};
// Pass data between coroutine entry and exit functions
struct go_newproc_caller {
__u64 goid;
void *sp; // stack pointer
} __attribute__((packed));
struct bpf_map_def SEC("maps") pid_tgid_callerid_map = {
.type = BPF_MAP_TYPE_HASH,
.key_size = sizeof(__u64),
.value_size = sizeof(struct go_newproc_caller),
.max_entries = HASH_ENTRIES_MAX,
};
/*
* Goroutines Map
* key: {tgid, pid}
* value: goroutine ID
*/
struct bpf_map_def SEC("maps") goroutines_map = {
.type = BPF_MAP_TYPE_HASH,
.key_size = sizeof(__u64),
.value_size = sizeof(__u64),
.max_entries = MAX_SYSTEM_THREADS,
};
SEC("uprobe/runtime.execute")
int runtime_execute(struct pt_regs *ctx)
{
__u64 pid_tgid = bpf_get_current_pid_tgid();
__u32 tgid = pid_tgid >> 32;
struct ebpf_proc_info *info = bpf_map_lookup_elem(&proc_info_map, &tgid);
if (!info) {
return 0;
}
int offset_g_goid = info->offsets[OFFSET_IDX_GOID_RUNTIME_G];
if (offset_g_goid < 0) {
return 0;
}
void *g_ptr;
if (is_register_based_call(info)) {
g_ptr = (void *)PT_GO_REGS_PARM1(ctx);
} else {
bpf_probe_read(&g_ptr, sizeof(g_ptr), (void *)(PT_REGS_SP(ctx) + 8));
}
__s64 goid = 0;
bpf_probe_read(&goid, sizeof(goid), g_ptr + offset_g_goid);
bpf_map_update_elem(&goroutines_map, &pid_tgid, &goid, BPF_ANY);
return 0;
}
// This function creates a new go coroutine, and the parent and child
// coroutine numbers are in the parameters and return values respectively.
// Pass the function parameters through pid_tgid_callerid_map
//
// go 1.15 ~ 1.17: func newproc1(fn *funcval, argp unsafe.Pointer, narg int32, callergp *g, callerpc uintptr) *g
// go1.18+ :func newproc1(fn *funcval, callergp *g, callerpc uintptr) *g
SEC("uprobe/enter_runtime.newproc1")
int enter_runtime_newproc1(struct pt_regs *ctx)
{
__u64 pid_tgid = bpf_get_current_pid_tgid();
__u32 tgid = pid_tgid >> 32;
struct ebpf_proc_info *info =
bpf_map_lookup_elem(&proc_info_map, &tgid);
if (!info) {
return 0;
}
// go less than 1.15 cannot get parent-child coroutine relationship
// ~ go1.14: func newproc1(fn *funcval, argp unsafe.Pointer, narg int32, callergp *g, callerpc uintptr)
if (info->version < GO_VERSION(1, 15, 0)) {
return 0;
}
int offset_g_goid = info->offsets[OFFSET_IDX_GOID_RUNTIME_G];
if (offset_g_goid < 0) {
return 0;
}
void *g_ptr;
if (is_register_based_call(info)) {
// https://github.com/golang/go/commit/8e5304f7298a0eef48e4796017c51b4d9aeb52b5
if (info->version >= GO_VERSION(1, 18, 0)) {
g_ptr = (void *)PT_GO_REGS_PARM2(ctx);
} else {
g_ptr = (void *)PT_GO_REGS_PARM4(ctx);
}
} else {
if (info->version >= GO_VERSION(1, 18, 0)) {
bpf_probe_read(&g_ptr, sizeof(g_ptr),
(void *)(PT_REGS_SP(ctx) + 16));
} else {
bpf_probe_read(&g_ptr, sizeof(g_ptr),
(void *)(PT_REGS_SP(ctx) + 32));
}
}
__s64 goid = 0;
bpf_probe_read(&goid, sizeof(goid), g_ptr + offset_g_goid);
if (!goid) {
return 0;
}
struct go_newproc_caller caller = {
.goid = goid,
.sp = (void *)PT_REGS_SP(ctx),
};
bpf_map_update_elem(&pid_tgid_callerid_map, &pid_tgid, &caller,
BPF_ANY);
return 0;
}
// The mapping relationship between parent and child coroutines is stored in go_ancerstor_map
//
// go 1.15 ~ 1.17: func newproc1(fn *funcval, argp unsafe.Pointer, narg int32, callergp *g, callerpc uintptr) *g
// go1.18+ :func newproc1(fn *funcval, callergp *g, callerpc uintptr) *g
SEC("uprobe/exit_runtime.newproc1")
int exit_runtime_newproc1(struct pt_regs *ctx)
{
__u64 pid_tgid = bpf_get_current_pid_tgid();
__u32 tgid = pid_tgid >> 32;
struct ebpf_proc_info *info =
bpf_map_lookup_elem(&proc_info_map, &tgid);
if (!info) {
return 0;
}
if(info->version < GO_VERSION(1, 15, 0)){
return 0;
}
int offset_g_goid = info->offsets[OFFSET_IDX_GOID_RUNTIME_G];
if (offset_g_goid < 0) {
return 0;
}
struct go_newproc_caller *caller =
bpf_map_lookup_elem(&pid_tgid_callerid_map, &pid_tgid);
if (!caller) {
return 0;
}
void *g_ptr;
if (is_register_based_call(info)) {
g_ptr = (void *)PT_GO_REGS_PARM1(ctx);
} else {
if (info->version >= GO_VERSION(1, 18, 0)) {
bpf_probe_read(&g_ptr, sizeof(g_ptr), caller->sp + 32);
} else {
bpf_probe_read(&g_ptr, sizeof(g_ptr), caller->sp + 48);
}
}
__s64 goid = 0;
bpf_probe_read(&goid, sizeof(goid), g_ptr + offset_g_goid);
if (!goid) {
bpf_map_delete_elem(&pid_tgid_callerid_map, &pid_tgid);
return 0;
}
struct go_key key = { .tgid = tgid, .goid = goid };
goid = caller->goid;
bpf_map_update_elem(&go_ancerstor_map, &key, &goid, BPF_ANY);
bpf_map_delete_elem(&pid_tgid_callerid_map, &pid_tgid);
return 0;
}
// /sys/kernel/debug/tracing/events/sched/sched_process_exit/format
SEC("tracepoint/sched/sched_process_exit")
int bpf_func_sched_process_exit(struct sched_comm_exit_ctx *ctx)
{
pid_t pid, tid;
__u64 id;
id = bpf_get_current_pid_tgid();
pid = id >> 32;
tid = (__u32)id;
// If is a process, clear proc_info_map element and submit event.
if (pid == tid) {
bpf_map_delete_elem(&proc_info_map, &pid);
struct process_event_t data;
data.pid = pid;
data.meta.event_type = EVENT_TYPE_PROC_EXIT;
bpf_get_current_comm(data.name, sizeof(data.name));
// int ret = bpf_perf_event_output(ctx, &NAME(socket_data),
// BPF_F_CURRENT_CPU, &data,
// sizeof(data));
// if (ret) {
// bpf_debug
// ("bpf_func_sched_process_exit event output failed: %d\n",
// ret);
// }
}
bpf_map_delete_elem(&goroutines_map, &id);
return 0;
}
// /sys/kernel/debug/tracing/events/sched/sched_process_fork/format
SEC("tracepoint/sched/sched_process_fork")
int bpf_func_sched_process_fork(struct sched_comm_fork_ctx *ctx)
{
struct process_event_t data;
data.meta.event_type = EVENT_TYPE_PROC_EXEC;
data.pid = ctx->child_pid;
bpf_get_current_comm(data.name, sizeof(data.name));
// int ret = bpf_perf_event_output(ctx, &NAME(socket_data),
// BPF_F_CURRENT_CPU, &data, sizeof(data));
// if (ret) {
// bpf_debug(
// "bpf_func_sys_exit_execve event output() failed: %d\n",
// ret);
// }
return 0;
}
char LICENSE[] SEC("license") = "GPL";

View File

@@ -1,33 +1,24 @@
#ifndef EBPF_EXAMPLE_GOROUTINE_H
#define EBPF_EXAMPLE_GOROUTINE_H
enum {
/*
* 0 ~ 16 for L7 socket event (struct socket_data_buffer),
* indicates the number of socket data in socket_data_buffer.
*/
/*
* For event registrion
*/
EVENT_TYPE_MIN = 1 << 5,
EVENT_TYPE_PROC_EXEC = 1 << 5,
EVENT_TYPE_PROC_EXIT = 1 << 6
// Add new event type here.
enum goroutine_state {
IDLE,
RUNNABLE,
RUNNING,
SYSCALL,
WAITING,
MORIBUND_UNUSED,
DEAD,
ENQUEUE_UNUSED,
COPYSTACK,
PREEMPTED,
};
// Description Provides basic information about an event
struct event_meta {
__u32 event_type;
struct goroutine_execute_data {
enum goroutine_state state;
unsigned long goid;
int pid;
int tgid;
};
// Process execution or exit event data
struct process_event_t {
struct event_meta meta;
__u32 pid; // process ID
__u8 name[TASK_COMM_LEN]; // process name
};
#endif