From 9e37387803a05c729c64072eec8f1d20c0f013d5 Mon Sep 17 00:00:00 2001 From: LinuxDev9002 Date: Wed, 22 Jul 2026 01:35:47 -0700 Subject: [PATCH] feat: deepen advanced eBPF tutorial tools --- .github/workflows/test-libbpf.yml | 16 + src/55-dns-egress/.config | 10 + src/55-dns-egress/dns_egress.bpf.c | 200 +++--- src/55-dns-egress/dns_egress.c | 348 ++++++---- src/55-dns-egress/tests/test_dns_egress.py | 2 +- src/56-slow-syscall-index/.gitignore | 2 - src/56-slow-syscall-index/README.md | 604 ----------------- src/56-slow-syscall-index/README.zh.md | 604 ----------------- .../slow_syscall_index.bpf.c | 88 --- .../slow_syscall_index.c | 389 ----------- .../slow_syscall_index.h | 17 - .../tests/test_slow_syscall_index.py | 25 - src/56-tc-flow-index/.config | 10 + src/56-tc-flow-index/.gitignore | 2 + .../Makefile | 4 +- src/56-tc-flow-index/bpf_experimental.h | 40 ++ src/56-tc-flow-index/tc_flow_index.bpf.c | 286 ++++++++ src/56-tc-flow-index/tc_flow_index.c | 316 +++++++++ src/56-tc-flow-index/tc_flow_index.h | 32 + .../tests/test_tc_flow_index.py | 44 ++ src/57-oom-watch/.config | 10 + src/57-oom-watch/oom_watch.bpf.c | 143 +++- src/57-oom-watch/oom_watch.c | 627 ++++++++++++++---- src/57-oom-watch/oom_watch.h | 35 +- src/57-oom-watch/tests/test_oom_watch.py | 8 +- src/60-afxdp-dump/.config | 10 + src/60-afxdp-dump/afxdp_dump.c | 82 ++- 27 files changed, 1802 insertions(+), 2152 deletions(-) create mode 100644 src/55-dns-egress/.config delete mode 100644 src/56-slow-syscall-index/.gitignore delete mode 100644 src/56-slow-syscall-index/README.md delete mode 100644 src/56-slow-syscall-index/README.zh.md delete mode 100644 src/56-slow-syscall-index/slow_syscall_index.bpf.c delete mode 100644 src/56-slow-syscall-index/slow_syscall_index.c delete mode 100644 src/56-slow-syscall-index/slow_syscall_index.h delete mode 100644 src/56-slow-syscall-index/tests/test_slow_syscall_index.py create mode 100644 src/56-tc-flow-index/.config create mode 100644 src/56-tc-flow-index/.gitignore rename src/{56-slow-syscall-index => 56-tc-flow-index}/Makefile (97%) create mode 100644 src/56-tc-flow-index/bpf_experimental.h create mode 100644 src/56-tc-flow-index/tc_flow_index.bpf.c create mode 100644 src/56-tc-flow-index/tc_flow_index.c create mode 100644 src/56-tc-flow-index/tc_flow_index.h create mode 100644 src/56-tc-flow-index/tests/test_tc_flow_index.py create mode 100644 src/57-oom-watch/.config create mode 100644 src/60-afxdp-dump/.config diff --git a/.github/workflows/test-libbpf.yml b/.github/workflows/test-libbpf.yml index 2c77e5c..b1eaae2 100644 --- a/.github/workflows/test-libbpf.yml +++ b/.github/workflows/test-libbpf.yml @@ -197,6 +197,22 @@ jobs: run: | make -C src/54-exec-image-inspector + - name: test 55 DNS-derived allowlist + run: | + make -C src/55-dns-egress + + - name: test 56 TC flow index + run: | + make -C src/56-tc-flow-index + + - name: test 57 OOM reclaim profiler + run: | + make -C src/57-oom-watch + + - name: test 60 AF_XDP dump + run: | + make -C src/60-afxdp-dump + - name: test features bpf_token run: | make -C src/features/bpf_token diff --git a/src/55-dns-egress/.config b/src/55-dns-egress/.config new file mode 100644 index 0000000..159521c --- /dev/null +++ b/src/55-dns-egress/.config @@ -0,0 +1,10 @@ +level=Depth +type=Security +kernel_min=5.8 +kernel_min_basis=required-feature +architectures=x86_64 +btf=required +kernel_config=CONFIG_BPF=y,CONFIG_BPF_SYSCALL=y,CONFIG_BPF_JIT=y,CONFIG_CGROUP_BPF=y,CONFIG_DEBUG_INFO_BTF=y,CONFIG_INET=y +hardware=none +root=required +test_status=ci-build diff --git a/src/55-dns-egress/dns_egress.bpf.c b/src/55-dns-egress/dns_egress.bpf.c index b10bca5..c2baeed 100644 --- a/src/55-dns-egress/dns_egress.bpf.c +++ b/src/55-dns-egress/dns_egress.bpf.c @@ -90,6 +90,53 @@ static __noinline bool matches_qname(struct __sk_buff *skb, __u32 offset) return true; } +static __always_inline bool parse_query_transport( + struct __sk_buff *skb, __u32 *dns_offset, struct dns_query_key *key) +{ + struct udphdr udp; + struct iphdr ip; + __u32 ip_header_len; + + if (bpf_skb_load_bytes(skb, 0, &ip, sizeof(ip))) + return false; + if (ip.version != 4 || ip.protocol != IPPROTO_UDP || ip.ihl < 5 || + (bpf_ntohs(ip.frag_off) & (IP_MF | IP_OFFSET)) || + ip.daddr != dns_server_ip) + return false; + ip_header_len = ip.ihl * 4; + if (bpf_skb_load_bytes(skb, ip_header_len, &udp, sizeof(udp)) || + bpf_ntohs(udp.dest) != dns_server_port) + return false; + *dns_offset = ip_header_len + sizeof(udp); + key->server_ip = ip.daddr; + key->client_ip = ip.saddr; + key->client_port = udp.source; + return true; +} + +static __always_inline bool parse_dns_query(struct __sk_buff *skb, + __u32 dns_offset, + struct dns_query_key *key) +{ + struct dns_question question; + struct dns_header header; + __u16 flags; + + if (bpf_skb_load_bytes(skb, dns_offset, &header, sizeof(header))) + return false; + flags = bpf_ntohs(header.flags); + if ((flags & 0xf800) || bpf_ntohs(header.questions) != 1) + return false; + if (!matches_qname(skb, dns_offset + sizeof(header))) + return false; + dns_offset += sizeof(header) + configured_qname_length; + if (bpf_skb_load_bytes(skb, dns_offset, &question, sizeof(question)) || + bpf_ntohs(question.type) != 1 || bpf_ntohs(question.class) != 1) + return false; + key->transaction_id = header.id; + return true; +} + SEC("cgroup_skb/egress") int record_dns_query(struct __sk_buff *skb) { @@ -97,107 +144,110 @@ int record_dns_query(struct __sk_buff *skb) .expires_ns = bpf_ktime_get_ns() + DNS_QUERY_LIFETIME_NS, }; struct dns_query_key key = {}; - struct dns_question question; - struct dns_header header; + __u32 dns_offset; + + if (!parse_query_transport(skb, &dns_offset, &key) || + !parse_dns_query(skb, dns_offset, &key)) + return 1; + bpf_map_update_elem(&pending_queries, &key, &state, BPF_ANY); + return 1; +} + +static __always_inline bool parse_response_transport( + struct __sk_buff *skb, __u32 *dns_offset, struct dns_query_key *query_key) +{ struct udphdr udp; struct iphdr ip; __u32 ip_header_len; - __u32 dns_offset; - __u16 flags; if (bpf_skb_load_bytes(skb, 0, &ip, sizeof(ip))) - return 1; + return false; if (ip.version != 4 || ip.protocol != IPPROTO_UDP || ip.ihl < 5 || (bpf_ntohs(ip.frag_off) & (IP_MF | IP_OFFSET)) || - ip.daddr != dns_server_ip) - return 1; + ip.saddr != dns_server_ip) + return false; ip_header_len = ip.ihl * 4; if (bpf_skb_load_bytes(skb, ip_header_len, &udp, sizeof(udp)) || - bpf_ntohs(udp.dest) != dns_server_port) - return 1; - dns_offset = ip_header_len + sizeof(udp); + bpf_ntohs(udp.source) != dns_server_port) + return false; + *dns_offset = ip_header_len + sizeof(udp); + query_key->server_ip = ip.saddr; + query_key->client_ip = ip.daddr; + query_key->client_port = udp.dest; + return true; +} + +static __always_inline bool pending_query_is_live( + struct dns_query_key *query_key) +{ + struct dns_query_state *query; + + query = bpf_map_lookup_elem(&pending_queries, query_key); + if (!query) + return false; + if (bpf_ktime_get_ns() >= query->expires_ns) { + bpf_map_delete_elem(&pending_queries, query_key); + return false; + } + return true; +} + +static __always_inline bool parse_response_question( + struct __sk_buff *skb, __u32 dns_offset, + struct dns_query_key *query_key, __u32 *answer_offset) +{ + struct dns_question question; + struct dns_header header; + if (bpf_skb_load_bytes(skb, dns_offset, &header, sizeof(header))) - return 1; - flags = bpf_ntohs(header.flags); - if ((flags & 0xf800) || bpf_ntohs(header.questions) != 1) - return 1; + return false; + query_key->transaction_id = header.id; + if (!pending_query_is_live(query_key)) + return false; + if ((bpf_ntohs(header.flags) & 0xf80f) != 0x8000 || + bpf_ntohs(header.questions) != 1 || !bpf_ntohs(header.answers)) + return false; if (!matches_qname(skb, dns_offset + sizeof(header))) - return 1; + return false; dns_offset += sizeof(header) + configured_qname_length; if (bpf_skb_load_bytes(skb, dns_offset, &question, sizeof(question)) || bpf_ntohs(question.type) != 1 || bpf_ntohs(question.class) != 1) - return 1; + return false; + *answer_offset = dns_offset + sizeof(question); + return true; +} - key.server_ip = ip.daddr; - key.client_ip = ip.saddr; - key.client_port = udp.source; - key.transaction_id = header.id; - bpf_map_update_elem(&pending_queries, &key, &state, BPF_ANY); - return 1; +static __always_inline bool parse_direct_a_answer(struct __sk_buff *skb, + __u32 answer_offset, + __u32 *key, __u32 *ttl) +{ + struct dns_a_answer answer; + + if (bpf_skb_load_bytes(skb, answer_offset, &answer, sizeof(answer)) || + bpf_ntohs(answer.name) != 0xc00c || bpf_ntohs(answer.type) != 1 || + bpf_ntohs(answer.class) != 1 || + bpf_ntohs(answer.address_length) != 4) + return false; + *key = answer.address; + *ttl = bpf_ntohl(answer.ttl); + return *ttl && *ttl <= 86400; } SEC("cgroup_skb/ingress") int learn_dns_answer(struct __sk_buff *skb) { - struct dns_a_answer answer; - struct dns_question question; - struct dns_header header; struct dns_query_key query_key = {}; - struct dns_query_state *query; struct dns_state state = {}; - struct udphdr udp; - struct iphdr ip; __u64 ttl_ns, expires; - __u32 ip_header_len; - __u32 dns_offset; - __u32 key; - __u32 ttl; + __u32 dns_offset, answer_offset; + __u32 key, ttl; - if (bpf_skb_load_bytes(skb, 0, &ip, sizeof(ip))) - return 1; - if (ip.version != 4 || ip.protocol != IPPROTO_UDP || ip.ihl < 5 || - (bpf_ntohs(ip.frag_off) & (IP_MF | IP_OFFSET)) || - ip.saddr != dns_server_ip) - return 1; - ip_header_len = ip.ihl * 4; - if (bpf_skb_load_bytes(skb, ip_header_len, &udp, sizeof(udp)) || - bpf_ntohs(udp.source) != dns_server_port) - return 1; - dns_offset = ip_header_len + sizeof(udp); - if (bpf_skb_load_bytes(skb, dns_offset, &header, sizeof(header))) - return 1; - query_key.server_ip = ip.saddr; - query_key.client_ip = ip.daddr; - query_key.client_port = udp.dest; - query_key.transaction_id = header.id; - query = bpf_map_lookup_elem(&pending_queries, &query_key); - if (!query) - return 1; - if (bpf_ktime_get_ns() >= query->expires_ns) { - bpf_map_delete_elem(&pending_queries, &query_key); - return 1; - } - if ((bpf_ntohs(header.flags) & 0xf80f) != 0x8000 || - bpf_ntohs(header.questions) != 1 || !bpf_ntohs(header.answers)) - return 1; - if (!matches_qname(skb, dns_offset + sizeof(header))) - return 1; - dns_offset += sizeof(header) + configured_qname_length; - if (bpf_skb_load_bytes(skb, dns_offset, &question, sizeof(question)) || - bpf_ntohs(question.type) != 1 || bpf_ntohs(question.class) != 1) - return 1; - dns_offset += sizeof(question); - if (bpf_skb_load_bytes(skb, dns_offset, &answer, sizeof(answer)) || - bpf_ntohs(answer.name) != 0xc00c || bpf_ntohs(answer.type) != 1 || - bpf_ntohs(answer.class) != 1 || - bpf_ntohs(answer.address_length) != 4) + if (!parse_response_transport(skb, &dns_offset, &query_key) || + !parse_response_question(skb, dns_offset, &query_key, + &answer_offset) || + !parse_direct_a_answer(skb, answer_offset, &key, &ttl)) return 1; bpf_map_delete_elem(&pending_queries, &query_key); - - key = answer.address; - ttl = bpf_ntohl(answer.ttl); - if (!ttl || ttl > 86400) - return 1; ttl_ns = (__u64)ttl * 1000000000ULL; expires = bpf_ktime_get_ns() + ttl_ns; state.expires_ns = expires; diff --git a/src/55-dns-egress/dns_egress.c b/src/55-dns-egress/dns_egress.c index 8bd07f2..bfe0988 100644 --- a/src/55-dns-egress/dns_egress.c +++ b/src/55-dns-egress/dns_egress.c @@ -31,6 +31,15 @@ struct options { bool demo; }; +struct dns_runtime { + struct dns_egress_bpf *skel; + struct bpf_link *query_link; + struct bpf_link *ingress_link; + struct bpf_link *connect_link; + struct ring_buffer *ring; + int cgroup_fd; +}; + static int event_counts[5]; static volatile sig_atomic_t stop; @@ -87,6 +96,42 @@ static void usage(const char *program) " %s --demo\n", program, program); } +static int apply_option(int option, const char *program, + struct options *options) +{ + switch (option) { + case 'c': options->cgroup_path = optarg; return 0; + case 'n': options->domain = optarg; return 0; + case 'r': options->dns_server = optarg; return 0; + case 'p': + return parse_uint(optarg, 65535, &options->port) || !options->port ? + -1 : 0; + case 's': + return parse_uint(optarg, 65535, &options->dns_port) || + !options->dns_port ? -1 : 0; + case 'd': + return parse_uint(optarg, 86400, &options->duration_seconds); + case 'D': options->demo = true; return 0; + case 'h': usage(program); exit(0); + default: return -1; + } +} + +static int finish_options(struct options *options) +{ + if (!options->demo) + return options->cgroup_path && options->domain && + options->dns_server ? 0 : -1; + if (options->cgroup_path || options->domain || options->dns_server) + return -1; + options->cgroup_path = "/sys/fs/cgroup"; + options->domain = "lab.test"; + options->dns_server = "127.0.0.1"; + options->port = DEMO_TCP_PORT; + options->dns_port = DEMO_DNS_PORT; + return 0; +} + static int parse_options(int argc, char **argv, struct options *options) { static const struct option long_options[] = { @@ -103,45 +148,10 @@ static int parse_options(int argc, char **argv, struct options *options) int option; while ((option = getopt_long(argc, argv, "c:n:r:p:s:d:Dh", long_options, - NULL)) != -1) { - switch (option) { - case 'c': options->cgroup_path = optarg; break; - case 'n': options->domain = optarg; break; - case 'r': options->dns_server = optarg; break; - case 'p': - if (parse_uint(optarg, 65535, &options->port) || - !options->port) - return -1; - break; - case 's': - if (parse_uint(optarg, 65535, &options->dns_port) || - !options->dns_port) - return -1; - break; - case 'd': - if (parse_uint(optarg, 86400, - &options->duration_seconds)) - return -1; - break; - case 'D': options->demo = true; break; - case 'h': usage(argv[0]); exit(0); - default: return -1; - } - } - if (optind != argc) - return -1; - if (options->demo) { - if (options->cgroup_path || options->domain || options->dns_server) + NULL)) != -1) + if (apply_option(option, argv[0], options)) return -1; - options->cgroup_path = "/sys/fs/cgroup"; - options->domain = "lab.test"; - options->dns_server = "127.0.0.1"; - options->port = DEMO_TCP_PORT; - options->dns_port = DEMO_DNS_PORT; - return 0; - } - return options->cgroup_path && options->domain && options->dns_server ? - 0 : -1; + return optind == argc ? finish_options(options) : -1; } static int encode_qname(const char *domain, unsigned char output[DNS_QNAME_MAX], @@ -341,18 +351,64 @@ static int complete_tcp(int listener, int client) { int accepted = accept4(listener, NULL, NULL, SOCK_CLOEXEC); char byte = 'x'; + int result = 0; - if (accepted < 0) - return -1; - if (write(accepted, &byte, 1) != 1 || read(client, &byte, 1) != 1) { - close(accepted); + if (accepted < 0) { + close(client); return -1; } + if (write(accepted, &byte, 1) != 1 || read(client, &byte, 1) != 1) + result = -1; close(accepted); close(client); + return result; +} + +static int poll_demo_events(struct ring_buffer *ring) +{ + int result = ring_buffer__poll(ring, 100); + + return result < 0 && result != -EINTR ? -1 : 0; +} + +static int expect_blocked_connect(struct ring_buffer *ring, + unsigned int port, const char *step) +{ + int client; + + errno = 0; + client = connect_tcp(port); + if (client >= 0) { + close(client); + return -1; + } + if (errno != EPERM || poll_demo_events(ring)) + return -1; + printf("demo step=%s result=blocked\n", step); return 0; } +static int expect_allowed_connect(struct ring_buffer *ring, int listener, + unsigned int port) +{ + int client = connect_tcp(port); + + if (client < 0 || complete_tcp(listener, client)) + return -1; + if (poll_demo_events(ring)) + return -1; + printf("demo step=live-answer result=allowed\n"); + return 0; +} + +static int expected_demo_events(void) +{ + return event_counts[DNS_LEARNED] == 1 && + event_counts[DNS_ALLOWED] == 1 && + event_counts[DNS_DENIED] == 4 && + event_counts[DNS_EXPIRED] == 1 ? 0 : -1; +} + static int run_demo(struct ring_buffer *ring, const struct options *options, const unsigned char *qname, unsigned int qname_length) { @@ -369,7 +425,7 @@ static int run_demo(struct ring_buffer *ring, const struct options *options, struct sockaddr_in response_client_address; unsigned char dns_message[512]; size_t query_length; - int dns_server = -1, dns_client = -1, listener = -1, client = -1; + int dns_server = -1, dns_client = -1, listener = -1; int err = -1; dns_server = bind_udp(&server_address); @@ -378,22 +434,14 @@ static int run_demo(struct ring_buffer *ring, const struct options *options, if (dns_server < 0 || dns_client < 0 || listener < 0) goto cleanup; - errno = 0; - client = connect_tcp(options->port); - if (client >= 0 || errno != EPERM) + if (expect_blocked_connect(ring, options->port, "before-dns")) goto cleanup; - ring_buffer__poll(ring, 100); - printf("demo step=before-dns result=blocked\n"); if (send_unsolicited_dns(dns_server, dns_client, qname, qname_length)) goto cleanup; - ring_buffer__poll(ring, 100); - errno = 0; - client = connect_tcp(options->port); - if (client >= 0 || errno != EPERM) + if (poll_demo_events(ring) || + expect_blocked_connect(ring, options->port, "unsolicited-response")) goto cleanup; - ring_buffer__poll(ring, 100); - printf("demo step=unsolicited-response result=blocked\n"); if (begin_dns_exchange(dns_server, dns_client, &server_address, qname, qname_length, dns_message, &query_length, @@ -401,59 +449,128 @@ static int run_demo(struct ring_buffer *ring, const struct options *options, send_dns_answer(dns_server, dns_client, &response_client_address, dns_message, query_length, DNS_ID + 1, 30)) goto cleanup; - ring_buffer__poll(ring, 100); - errno = 0; - client = connect_tcp(options->port); - if (client >= 0 || errno != EPERM) + if (poll_demo_events(ring) || + expect_blocked_connect(ring, options->port, "wrong-transaction-id")) goto cleanup; - ring_buffer__poll(ring, 100); - printf("demo step=wrong-transaction-id result=blocked\n"); if (send_dns_answer(dns_server, dns_client, &response_client_address, dns_message, query_length, DNS_ID, 1)) goto cleanup; - ring_buffer__poll(ring, 100); - client = connect_tcp(options->port); - if (client < 0 || complete_tcp(listener, client)) + if (poll_demo_events(ring) || + expect_allowed_connect(ring, listener, options->port)) goto cleanup; - client = -1; - ring_buffer__poll(ring, 100); - printf("demo step=live-answer result=allowed\n"); nanosleep(&wait_time, NULL); - ring_buffer__poll(ring, 100); - errno = 0; - client = connect_tcp(options->port); - if (client >= 0 || errno != EPERM) + if (poll_demo_events(ring) || + expect_blocked_connect(ring, options->port, "expired-answer")) goto cleanup; - ring_buffer__poll(ring, 100); - printf("demo step=expired-answer result=blocked\n"); - if (event_counts[DNS_LEARNED] != 1 || event_counts[DNS_ALLOWED] != 1 || - event_counts[DNS_DENIED] != 4 || event_counts[DNS_EXPIRED] != 1) + if (expected_demo_events()) goto cleanup; err = 0; cleanup: - if (client >= 0) close(client); if (listener >= 0) close(listener); if (dns_client >= 0) close(dns_client); if (dns_server >= 0) close(dns_server); return err; } +static bool link_failed(struct bpf_link **link) +{ + if (!libbpf_get_error(*link)) + return false; + *link = NULL; + return true; +} + +static int prepare_runtime(struct dns_runtime *runtime, + const struct options *options, + const struct in_addr *dns_server, + const unsigned char *qname, + unsigned int qname_length) +{ + bool failed; + + runtime->cgroup_fd = open(options->cgroup_path, + O_RDONLY | O_DIRECTORY | O_CLOEXEC); + if (runtime->cgroup_fd < 0) { + fprintf(stderr, "failed to open cgroup %s: %s\n", + options->cgroup_path, strerror(errno)); + return -1; + } + runtime->skel = dns_egress_bpf__open(); + if (!runtime->skel) + return -1; + runtime->skel->rodata->target_tgid = options->demo ? getpid() : 0; + runtime->skel->rodata->dns_server_ip = dns_server->s_addr; + runtime->skel->rodata->dns_server_port = options->dns_port; + runtime->skel->rodata->protected_tcp_port = options->port; + runtime->skel->rodata->configured_qname_length = qname_length; + memcpy((void *)runtime->skel->rodata->configured_qname, qname, + qname_length); + if (dns_egress_bpf__load(runtime->skel)) { + fprintf(stderr, "failed to load DNS egress BPF programs\n"); + return -1; + } + runtime->query_link = bpf_program__attach_cgroup( + runtime->skel->progs.record_dns_query, runtime->cgroup_fd); + runtime->ingress_link = bpf_program__attach_cgroup( + runtime->skel->progs.learn_dns_answer, runtime->cgroup_fd); + runtime->connect_link = bpf_program__attach_cgroup( + runtime->skel->progs.enforce_dns_policy, runtime->cgroup_fd); + failed = link_failed(&runtime->query_link); + failed |= link_failed(&runtime->ingress_link); + failed |= link_failed(&runtime->connect_link); + if (failed) { + fprintf(stderr, "failed to attach programs to cgroup %s\n", + options->cgroup_path); + return -1; + } + runtime->ring = ring_buffer__new( + bpf_map__fd(runtime->skel->maps.events), handle_event, NULL, NULL); + return runtime->ring ? 0 : -1; +} + +static int poll_policy_events(struct ring_buffer *ring, + unsigned int duration_seconds) +{ + unsigned long long deadline = 0; + + signal(SIGINT, handle_signal); + signal(SIGTERM, handle_signal); + if (duration_seconds) + deadline = monotonic_ns() + + (unsigned long long)duration_seconds * 1000000000ULL; + while (!stop && (!deadline || monotonic_ns() < deadline)) { + int result = ring_buffer__poll(ring, 100); + + if (result < 0 && result != -EINTR) { + fprintf(stderr, "ring buffer poll failed: %d\n", result); + return -1; + } + } + return 0; +} + +static void destroy_runtime(struct dns_runtime *runtime) +{ + ring_buffer__free(runtime->ring); + bpf_link__destroy(runtime->connect_link); + bpf_link__destroy(runtime->ingress_link); + bpf_link__destroy(runtime->query_link); + if (runtime->cgroup_fd >= 0) + close(runtime->cgroup_fd); + dns_egress_bpf__destroy(runtime->skel); +} + int main(int argc, char **argv) { struct options options = { .port = 443, .dns_port = 53 }; - struct dns_egress_bpf *skel = NULL; - struct bpf_link *query_link = NULL, *ingress_link = NULL; - struct bpf_link *connect_link = NULL; - struct ring_buffer *ring = NULL; + struct dns_runtime runtime = { .cgroup_fd = -1 }; struct in_addr dns_server = {}; unsigned char qname[DNS_QNAME_MAX] = {}; - unsigned long long deadline = 0; unsigned int qname_length = 0; - int cgroup_fd = -1; int err = 1; setvbuf(stdout, NULL, _IONBF, 0); @@ -463,78 +580,21 @@ int main(int argc, char **argv) usage(argv[0]); return 2; } - cgroup_fd = open(options.cgroup_path, - O_RDONLY | O_DIRECTORY | O_CLOEXEC); - if (cgroup_fd < 0) { - fprintf(stderr, "failed to open cgroup %s: %s\n", - options.cgroup_path, strerror(errno)); - goto cleanup; - } - - skel = dns_egress_bpf__open(); - if (!skel) - goto cleanup; - skel->rodata->target_tgid = options.demo ? getpid() : 0; - skel->rodata->dns_server_ip = dns_server.s_addr; - skel->rodata->dns_server_port = options.dns_port; - skel->rodata->protected_tcp_port = options.port; - skel->rodata->configured_qname_length = qname_length; - memcpy((void *)skel->rodata->configured_qname, qname, qname_length); - if (dns_egress_bpf__load(skel)) { - fprintf(stderr, "failed to load DNS egress BPF programs\n"); - goto cleanup; - } - query_link = bpf_program__attach_cgroup(skel->progs.record_dns_query, - cgroup_fd); - ingress_link = bpf_program__attach_cgroup(skel->progs.learn_dns_answer, - cgroup_fd); - connect_link = bpf_program__attach_cgroup(skel->progs.enforce_dns_policy, - cgroup_fd); - if (libbpf_get_error(query_link) || libbpf_get_error(ingress_link) || - libbpf_get_error(connect_link)) { - fprintf(stderr, "failed to attach programs to cgroup %s\n", - options.cgroup_path); - query_link = libbpf_get_error(query_link) ? NULL : query_link; - ingress_link = libbpf_get_error(ingress_link) ? NULL : ingress_link; - connect_link = libbpf_get_error(connect_link) ? NULL : connect_link; - goto cleanup; - } - ring = ring_buffer__new(bpf_map__fd(skel->maps.events), handle_event, - NULL, NULL); - if (!ring) + if (prepare_runtime(&runtime, &options, &dns_server, qname, + qname_length)) goto cleanup; printf("dns-egress attached cgroup=%s domain=%s resolver=%s tcp_port=%u dns_port=%u\n", options.cgroup_path, options.domain, options.dns_server, options.port, options.dns_port); if (options.demo) { - if (run_demo(ring, &options, qname, qname_length)) + if (run_demo(runtime.ring, &options, qname, qname_length)) goto cleanup; - } else { - signal(SIGINT, handle_signal); - signal(SIGTERM, handle_signal); - if (options.duration_seconds) - deadline = monotonic_ns() + - (unsigned long long)options.duration_seconds * - 1000000000ULL; - while (!stop && (!deadline || monotonic_ns() < deadline)) { - int poll_result = ring_buffer__poll(ring, 100); - - if (poll_result < 0 && poll_result != -EINTR) { - fprintf(stderr, "ring buffer poll failed: %d\n", - poll_result); - goto cleanup; - } - } - } + } else if (poll_policy_events(runtime.ring, options.duration_seconds)) + goto cleanup; err = 0; cleanup: - ring_buffer__free(ring); - bpf_link__destroy(connect_link); - bpf_link__destroy(ingress_link); - bpf_link__destroy(query_link); - if (cgroup_fd >= 0) close(cgroup_fd); - dns_egress_bpf__destroy(skel); + destroy_runtime(&runtime); return err; } diff --git a/src/55-dns-egress/tests/test_dns_egress.py b/src/55-dns-egress/tests/test_dns_egress.py index 02874aa..34065ae 100644 --- a/src/55-dns-egress/tests/test_dns_egress.py +++ b/src/55-dns-egress/tests/test_dns_egress.py @@ -22,7 +22,7 @@ def main(): assert "event=expired" in output, output assert "demo step=expired-answer result=blocked" in output, output assert output.count("event=denied") == 4, output - print("DNS-aware egress integration test: PASS") + print("DNS-derived allowlist integration test: PASS") if __name__ == "__main__": diff --git a/src/56-slow-syscall-index/.gitignore b/src/56-slow-syscall-index/.gitignore deleted file mode 100644 index 442bb93..0000000 --- a/src/56-slow-syscall-index/.gitignore +++ /dev/null @@ -1,2 +0,0 @@ -.output/ -slow_syscall_index diff --git a/src/56-slow-syscall-index/README.md b/src/56-slow-syscall-index/README.md deleted file mode 100644 index 3dfb6b0..0000000 --- a/src/56-slow-syscall-index/README.md +++ /dev/null @@ -1,604 +0,0 @@ -# eBPF Tutorial by Example: Slow Syscall Latency Index - -Which syscalls are taking the longest in your application? This tutorial builds a tool that traces syscall latency per process using BPF task local storage, aggregates the results by TGID and syscall number, and ranks them by total latency. The result shows which recorded slow syscalls account for most of the observed syscall latency. - -> Complete source code: - -## eBPF and Syscall Tracing - -eBPF lets verified programs run at Linux kernel hooks and send selected state to user space. This tutorial uses the `raw_syscalls/sys_enter` and `raw_syscalls/sys_exit` tracepoints, which fire on every syscall entry and exit system-wide. To correlate entry with exit, we use BPF task local storage, a map type introduced in Linux 5.11 that attaches storage directly to each task struct. The kernel removes that storage when the task exits, which fits per-thread state with task lifetime. The measurement is syscall enter-to-exit wall-clock latency rather than scheduler-only blocked time. This implementation requires Linux 5.11 or later. - -## How the Implementation Works - -The tool attaches to the raw syscall entry and exit tracepoints. On entry, it records the start timestamp and syscall ID in task local storage. On exit, it calculates the duration and reports events exceeding the configured threshold. User space aggregates events by TGID and syscall, tracking count, total latency, maximum latency, and error count. - -## Header File - -```c -/* SPDX-License-Identifier: (LGPL-2.1 OR BSD-2-Clause) */ -#ifndef __SLOW_SYSCALL_INDEX_H -#define __SLOW_SYSCALL_INDEX_H - -#define SLOW_SYSCALL_COMM_LEN 16 - -struct slow_syscall_event { - unsigned long long timestamp_ns; - unsigned long long duration_ns; - long long return_value; - unsigned int tgid; - unsigned int tid; - unsigned int syscall_id; - char comm[SLOW_SYSCALL_COMM_LEN]; -}; - -#endif /* __SLOW_SYSCALL_INDEX_H */ -``` - -The header defines the event structure sent to user space via ring buffer. Each event includes the timestamp, duration, return value, TGID, TID, syscall number, and process name. - -## BPF Program - -```c -// SPDX-License-Identifier: GPL-2.0 -#include "vmlinux.h" -#include -#include "slow_syscall_index.h" - -char LICENSE[] SEC("license") = "GPL"; - -const volatile __u64 minimum_ns = 10 * 1000 * 1000ULL; -const volatile __u32 target_tgid; -const volatile __u32 ignored_tgid; - -struct syscall_state { - __u64 started_ns; - __u32 syscall_id; - __u32 pad; -}; - -struct { - __uint(type, BPF_MAP_TYPE_TASK_STORAGE); - __uint(map_flags, BPF_F_NO_PREALLOC); - __type(key, int); - __type(value, struct syscall_state); -} active_syscalls SEC(".maps"); - -struct { - __uint(type, BPF_MAP_TYPE_RINGBUF); - __uint(max_entries, 256 * 1024); -} events SEC(".maps"); - -__u64 completed_syscalls; -__u64 slow_syscalls; -__u64 dropped_events; - -SEC("tp/raw_syscalls/sys_enter") -int record_syscall_entry(struct trace_event_raw_sys_enter *ctx) -{ - struct task_struct *task; - struct syscall_state *state; - __u64 pid_tgid = bpf_get_current_pid_tgid(); - __u32 tgid = pid_tgid >> 32; - - if (tgid == ignored_tgid || (target_tgid && tgid != target_tgid)) - return 0; - task = bpf_get_current_task_btf(); - state = bpf_task_storage_get(&active_syscalls, task, 0, - BPF_LOCAL_STORAGE_GET_F_CREATE); - if (!state) - return 0; - state->started_ns = bpf_ktime_get_ns(); - state->syscall_id = ctx->id; - return 0; -} - -SEC("tp/raw_syscalls/sys_exit") -int report_slow_syscall(struct trace_event_raw_sys_exit *ctx) -{ - struct task_struct *task = bpf_get_current_task_btf(); - struct slow_syscall_event *event; - struct syscall_state *state; - __u64 pid_tgid = bpf_get_current_pid_tgid(); - __u64 duration_ns; - - state = bpf_task_storage_get(&active_syscalls, task, 0, 0); - if (!state) - return 0; - duration_ns = bpf_ktime_get_ns() - state->started_ns; - __sync_fetch_and_add(&completed_syscalls, 1); - if (duration_ns < minimum_ns) - goto out; - - __sync_fetch_and_add(&slow_syscalls, 1); - event = bpf_ringbuf_reserve(&events, sizeof(*event), 0); - if (!event) { - __sync_fetch_and_add(&dropped_events, 1); - goto out; - } - event->timestamp_ns = bpf_ktime_get_ns(); - event->duration_ns = duration_ns; - event->return_value = ctx->ret; - event->tgid = pid_tgid >> 32; - event->tid = (__u32)pid_tgid; - event->syscall_id = state->syscall_id; - bpf_get_current_comm(event->comm, sizeof(event->comm)); - bpf_ringbuf_submit(event, 0); -out: - bpf_task_storage_delete(&active_syscalls, task); - return 0; -} -``` - -The BPF program uses `BPF_MAP_TYPE_TASK_STORAGE` to store syscall state per task. This map type was introduced in Linux 5.11 and provides storage that is automatically cleaned up when the task exits. - -The `record_syscall_entry` function runs on syscall entry, storing the start timestamp and syscall ID. It filters by target TGID and ignores the tracer process itself. - -The `report_slow_syscall` function runs on syscall exit. It calculates the duration by subtracting the start timestamp from the current time. If the duration exceeds the threshold, it reserves space in the ring buffer, fills in the event, and submits it. The task storage is deleted after each syscall to keep memory usage bounded. - -The program tracks three global counters: `completed_syscalls` counts all traced syscalls, `slow_syscalls` counts those exceeding the threshold, and `dropped_events` counts ring buffer allocation failures. - -## User Space Program - -```c -// SPDX-License-Identifier: (LGPL-2.1 OR BSD-2-Clause) -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include "slow_syscall_index.h" -#include "slow_syscall_index.skel.h" - -#define MAX_STATS 512 - -struct options { - unsigned int pid; - unsigned int minimum_ms; - unsigned int duration_seconds; - unsigned int top; - bool demo; -}; - -struct syscall_stats { - unsigned int tgid; - unsigned int syscall_id; - unsigned long long count; - unsigned long long total_ns; - unsigned long long maximum_ns; - unsigned long long errors; - char comm[SLOW_SYSCALL_COMM_LEN]; -}; - -static struct syscall_stats stats[MAX_STATS]; -static size_t stats_count; -static volatile sig_atomic_t stop; -static unsigned int received_events; -static unsigned int unaggregated_events; - -static void handle_signal(int signal_number) -{ - (void)signal_number; - stop = 1; -} - -static unsigned long long monotonic_ns(void) -{ - struct timespec now; - - clock_gettime(CLOCK_MONOTONIC, &now); - return (unsigned long long)now.tv_sec * 1000000000ULL + now.tv_nsec; -} - -static const char *syscall_name(unsigned int id) -{ - switch (id) { -#ifdef __NR_read - case __NR_read: return "read"; -#endif -#ifdef __NR_write - case __NR_write: return "write"; -#endif -#ifdef __NR_openat - case __NR_openat: return "openat"; -#endif -#ifdef __NR_close - case __NR_close: return "close"; -#endif -#ifdef __NR_fsync - case __NR_fsync: return "fsync"; -#endif -#ifdef __NR_fdatasync - case __NR_fdatasync: return "fdatasync"; -#endif -#ifdef __NR_poll - case __NR_poll: return "poll"; -#endif -#ifdef __NR_ppoll - case __NR_ppoll: return "ppoll"; -#endif -#ifdef __NR_epoll_wait - case __NR_epoll_wait: return "epoll_wait"; -#endif -#ifdef __NR_epoll_pwait - case __NR_epoll_pwait: return "epoll_pwait"; -#endif -#ifdef __NR_futex - case __NR_futex: return "futex"; -#endif -#ifdef __NR_nanosleep - case __NR_nanosleep: return "nanosleep"; -#endif -#ifdef __NR_clock_nanosleep - case __NR_clock_nanosleep: return "clock_nanosleep"; -#endif -#ifdef __NR_connect - case __NR_connect: return "connect"; -#endif -#ifdef __NR_accept - case __NR_accept: return "accept"; -#endif -#ifdef __NR_accept4 - case __NR_accept4: return "accept4"; -#endif -#ifdef __NR_recvfrom - case __NR_recvfrom: return "recvfrom"; -#endif -#ifdef __NR_recvmsg - case __NR_recvmsg: return "recvmsg"; -#endif -#ifdef __NR_sendto - case __NR_sendto: return "sendto"; -#endif -#ifdef __NR_sendmsg - case __NR_sendmsg: return "sendmsg"; -#endif - default: return "unknown"; - } -} - -static struct syscall_stats *get_stats(const struct slow_syscall_event *event) -{ - struct syscall_stats *entry; - - for (size_t i = 0; i < stats_count; i++) - if (stats[i].tgid == event->tgid && - stats[i].syscall_id == event->syscall_id) - return &stats[i]; - if (stats_count == MAX_STATS) - return NULL; - entry = &stats[stats_count++]; - entry->tgid = event->tgid; - entry->syscall_id = event->syscall_id; - memcpy(entry->comm, event->comm, sizeof(entry->comm)); - entry->comm[sizeof(entry->comm) - 1] = '\0'; - return entry; -} - -static int handle_event(void *ctx, void *data, size_t size) -{ - const struct slow_syscall_event *event = data; - struct syscall_stats *entry; - - (void)ctx; - if (size != sizeof(*event)) - return 0; - entry = get_stats(event); - if (entry) { - entry->count++; - entry->total_ns += event->duration_ns; - if (event->duration_ns > entry->maximum_ns) - entry->maximum_ns = event->duration_ns; - if (event->return_value < 0) - entry->errors++; - } else { - unaggregated_events++; - } - received_events++; - printf("pid=%u tid=%u comm=%s syscall=%s(%u) latency_ms=%.3f return=%lld\n", - event->tgid, event->tid, event->comm, - syscall_name(event->syscall_id), event->syscall_id, - event->duration_ns / 1000000.0, event->return_value); - return 0; -} - -static int compare_stats(const void *left, const void *right) -{ - const struct syscall_stats *a = left; - const struct syscall_stats *b = right; - - return a->total_ns < b->total_ns ? 1 : a->total_ns > b->total_ns ? -1 : 0; -} - -static void print_summary(unsigned int top) -{ - size_t limit; - - qsort(stats, stats_count, sizeof(stats[0]), compare_stats); - limit = stats_count < top ? stats_count : top; - printf("\nSlow syscall index, ranked by total latency:\n"); - printf("%-7s %-16s %-18s %8s %12s %12s %8s\n", - "PID", "COMM", "SYSCALL", "COUNT", "TOTAL_MS", "MAX_MS", "ERRORS"); - for (size_t i = 0; i < limit; i++) { - const struct syscall_stats *entry = &stats[i]; - - printf("%-7u %-16s %-12s(%3u) %8llu %12.3f %12.3f %8llu\n", - entry->tgid, entry->comm, syscall_name(entry->syscall_id), - entry->syscall_id, entry->count, - entry->total_ns / 1000000.0, - entry->maximum_ns / 1000000.0, entry->errors); - } -} - -static int parse_uint(const char *text, unsigned int maximum, - unsigned int *value) -{ - char *end = NULL; - unsigned long parsed; - - errno = 0; - parsed = strtoul(text, &end, 10); - if (errno || !*text || *end || parsed > maximum) - return -1; - *value = parsed; - return 0; -} - -static void usage(const char *program) -{ - printf("Usage: %s [--pid PID] [--min-ms MS] [--duration SEC] [--top N] [--demo]\n", - program); -} - -static int parse_options(int argc, char **argv, struct options *options) -{ - static const struct option long_options[] = { - { "pid", required_argument, NULL, 'p' }, - { "min-ms", required_argument, NULL, 'm' }, - { "duration", required_argument, NULL, 'd' }, - { "top", required_argument, NULL, 't' }, - { "demo", no_argument, NULL, 'D' }, - { "help", no_argument, NULL, 'h' }, - {}, - }; - int option; - - while ((option = getopt_long(argc, argv, "p:m:d:t:Dh", long_options, - NULL)) != -1) { - switch (option) { - case 'p': - if (parse_uint(optarg, 0xffffffffU, &options->pid)) - return -1; - break; - case 'm': - if (parse_uint(optarg, 60000, &options->minimum_ms)) - return -1; - break; - case 'd': - if (parse_uint(optarg, 86400, &options->duration_seconds)) - return -1; - break; - case 't': - if (parse_uint(optarg, MAX_STATS, &options->top)) - return -1; - break; - case 'D': options->demo = true; break; - case 'h': usage(argv[0]); exit(0); - default: return -1; - } - } - return optind == argc && options->minimum_ms && options->top ? 0 : -1; -} - -static pid_t start_demo(int gate[2], int data_pipe[2]) -{ - pid_t child = fork(); - char byte; - - if (child) - return child; - close(gate[1]); - close(data_pipe[1]); - if (read(gate[0], &byte, 1) != 1) - _exit(2); - if (read(data_pipe[0], &byte, 1) != 1) - _exit(3); - _exit(0); -} - -int main(int argc, char **argv) -{ - struct options options = { .minimum_ms = 10, .top = 10 }; - struct slow_syscall_index_bpf *skel = NULL; - struct ring_buffer *ring = NULL; - unsigned long long deadline = 0; - int gate[2] = { -1, -1 }, data_pipe[2] = { -1, -1 }; - pid_t demo_child = -1; - int demo_status = 0; - int err = 1; - - setvbuf(stdout, NULL, _IONBF, 0); - if (parse_options(argc, argv, &options)) { - usage(argv[0]); - return 2; - } - if (options.demo) { - if (options.pid) { - fprintf(stderr, "--demo and --pid cannot be combined\n"); - return 2; - } - if (pipe(gate) || pipe(data_pipe)) { - perror("pipe"); - goto cleanup; - } - demo_child = start_demo(gate, data_pipe); - if (demo_child < 0) { - perror("fork"); - goto cleanup; - } - options.pid = demo_child; - } - - skel = slow_syscall_index_bpf__open(); - if (!skel) - goto cleanup; - skel->rodata->minimum_ns = (unsigned long long)options.minimum_ms * 1000000ULL; - skel->rodata->target_tgid = options.pid; - skel->rodata->ignored_tgid = getpid(); - if (slow_syscall_index_bpf__load(skel) || - slow_syscall_index_bpf__attach(skel)) { - fprintf(stderr, "failed to load and attach slow syscall tracer\n"); - goto cleanup; - } - ring = ring_buffer__new(bpf_map__fd(skel->maps.events), handle_event, - NULL, NULL); - if (!ring) - goto cleanup; - - signal(SIGINT, handle_signal); - signal(SIGTERM, handle_signal); - if (options.pid) - printf("Tracing PID %u syscalls slower than %u ms. Press Ctrl-C to stop.\n", - options.pid, options.minimum_ms); - else - printf("Tracing all syscalls slower than %u ms. Press Ctrl-C to stop.\n", - options.minimum_ms); - if (options.duration_seconds) - deadline = monotonic_ns() + - (unsigned long long)options.duration_seconds * 1000000000ULL; - - if (options.demo) { - char byte = 'x'; - struct timespec delay = { - .tv_sec = options.minimum_ms / 1000, - .tv_nsec = (long)(options.minimum_ms % 1000 + 20) * 1000000L, - }; - - if (delay.tv_nsec >= 1000000000L) { - delay.tv_sec++; - delay.tv_nsec -= 1000000000L; - } - close(gate[0]); gate[0] = -1; - close(data_pipe[0]); data_pipe[0] = -1; - write(gate[1], &byte, 1); - nanosleep(&delay, NULL); - write(data_pipe[1], &byte, 1); - waitpid(demo_child, &demo_status, 0); - demo_child = -1; - for (int i = 0; i < 20 && !received_events; i++) - ring_buffer__poll(ring, 50); - } else { - while (!stop && (!deadline || monotonic_ns() < deadline)) { - int poll_result = ring_buffer__poll(ring, 100); - - if (poll_result < 0 && poll_result != -EINTR) { - fprintf(stderr, "ring buffer poll failed: %d\n", poll_result); - goto cleanup; - } - } - } - - print_summary(options.top); - printf("completed=%llu slow=%llu dropped=%llu unaggregated=%u\n", - (unsigned long long)skel->bss->completed_syscalls, - (unsigned long long)skel->bss->slow_syscalls, - (unsigned long long)skel->bss->dropped_events, - unaggregated_events); - if (options.demo && (!WIFEXITED(demo_status) || WEXITSTATUS(demo_status) || - !received_events)) - goto cleanup; - err = 0; - -cleanup: - if (demo_child > 0) { - kill(demo_child, SIGKILL); - waitpid(demo_child, NULL, 0); - } - for (size_t i = 0; i < 2; i++) { - if (gate[i] >= 0) close(gate[i]); - if (data_pipe[i] >= 0) close(data_pipe[i]); - } - ring_buffer__free(ring); - slow_syscall_index_bpf__destroy(skel); - return err; -} -``` - -The user space program aggregates events by TGID and syscall number. Each event updates the count, total latency, maximum latency, and error count for that combination. On exit, it sorts the results by total latency and prints the top N entries. - -The demo mode forks a child process that blocks on a pipe read. The parent delays long enough to exceed the threshold, then writes to unblock the child. This creates a guaranteed slow syscall for testing. - -## Compilation and Execution - -Build the tool: - -```bash -cd src/56-slow-syscall-index -make -``` - -Run with options: - -```bash -sudo ./slow_syscall_index --pid 1234 --min-ms 10 --duration 60 --top 20 -``` - -Or run the built-in demo: - -```bash -sudo ./slow_syscall_index --demo --min-ms 10 --top 10 -``` - -Example output: - -```text -Tracing PID 1255 syscalls slower than 10 ms. Press Ctrl-C to stop. -pid=1255 tid=1255 comm=slow_syscall_in syscall=read(0) latency_ms=30.080 return=1 - -Slow syscall index, ranked by total latency: -PID COMM SYSCALL COUNT TOTAL_MS MAX_MS ERRORS -1255 slow_syscall_in read ( 0) 1 30.080 30.080 0 -completed=1 slow=1 dropped=0 unaggregated=0 -``` - -## Requirements - -| Requirement | Details | -|-------------|---------| -| Kernel | Linux 5.11+ (BPF task local storage) | -| Config | `CONFIG_BPF_SYSCALL`, `CONFIG_DEBUG_INFO_BTF` | -| Privileges | Root | - -## Understanding the Output - -The index shows: - -- **PID**: Thread group ID (the process ID carried in the event) -- **COMM**: Process name -- **SYSCALL**: Syscall name and number -- **COUNT**: Number of slow invocations -- **TOTAL_MS**: Sum of all slow syscall durations -- **MAX_MS**: Longest single invocation -- **ERRORS**: Count of invocations returning negative values - -The `completed` counter shows all traced syscalls, `slow` shows those exceeding the threshold, and `dropped` shows ring buffer allocation failures. User space keeps 512 `(TGID, syscall)` groups; events beyond that table capacity are still received and counted as `unaggregated`. - -## Summary - -Task local storage joins each syscall exit to its matching entry without a user-managed PID map. Kernel-side thresholding keeps the event stream focused, and user-space aggregation turns the remaining completions into a ranked latency index. The result measures wall-clock syscall latency; scheduler correlation can be added when off-CPU attribution is needed. - -> If you'd like to dive deeper into eBPF, check out our tutorial repository at or visit our website at . - -## References - -- [BPF task local storage documentation](https://docs.ebpf.io/linux/map-type/BPF_MAP_TYPE_TASK_STORAGE/) -- [BPF ring buffer documentation](https://docs.kernel.org/6.6/bpf/ringbuf.html) -- [raw_syscalls tracepoints](https://github.com/torvalds/linux/blob/master/include/trace/events/syscalls.h) -- [Task local storage commit](https://github.com/torvalds/linux/commit/4cf1bc1f1045) diff --git a/src/56-slow-syscall-index/README.zh.md b/src/56-slow-syscall-index/README.zh.md deleted file mode 100644 index b022ef7..0000000 --- a/src/56-slow-syscall-index/README.zh.md +++ /dev/null @@ -1,604 +0,0 @@ -# eBPF 实战教程：慢系统调用延迟索引 - -应用程序中哪些系统调用耗时最长？本教程构建一个工具，使用 BPF 任务本地存储跟踪每个进程的系统调用延迟，按 TGID 和系统调用号聚合结果，并按总延迟排名，最终找出哪些慢系统调用占据了最多的观测延迟。 - -> 完整源代码： - -## eBPF 与系统调用跟踪 - -eBPF 让经过验证器检查的程序运行在 Linux 内核 hook 上，并把选出的状态发送到用户态。本教程使用 `raw_syscalls/sys_enter` 和 `raw_syscalls/sys_exit` 跟踪点，它们在系统范围内每次系统调用进入和退出时触发。为了关联进入和退出，我们使用 Linux 5.11 引入的 BPF 任务本地存储，把状态直接附加到 task 结构上，并由内核在任务退出时清理。这里测量的是系统调用从进入到退出的挂钟延迟，而不是只统计调度器阻塞时间，因此本例需要 Linux 5.11 或更高版本。 - -## 实现原理 - -该工具附加到原始系统调用进入和退出跟踪点。在进入时，它在任务本地存储中记录开始时间戳和系统调用 ID。在退出时，它计算持续时间并报告超过配置阈值的事件。用户空间按 TGID 和系统调用聚合事件，跟踪计数、总延迟、最大延迟和错误计数。 - -## 头文件 - -```c -/* SPDX-License-Identifier: (LGPL-2.1 OR BSD-2-Clause) */ -#ifndef __SLOW_SYSCALL_INDEX_H -#define __SLOW_SYSCALL_INDEX_H - -#define SLOW_SYSCALL_COMM_LEN 16 - -struct slow_syscall_event { - unsigned long long timestamp_ns; - unsigned long long duration_ns; - long long return_value; - unsigned int tgid; - unsigned int tid; - unsigned int syscall_id; - char comm[SLOW_SYSCALL_COMM_LEN]; -}; - -#endif /* __SLOW_SYSCALL_INDEX_H */ -``` - -头文件定义了通过 ring buffer 发送到用户空间的事件结构。每个事件包括时间戳、持续时间、返回值、TGID、TID、系统调用号和进程名。 - -## BPF 程序 - -```c -// SPDX-License-Identifier: GPL-2.0 -#include "vmlinux.h" -#include -#include "slow_syscall_index.h" - -char LICENSE[] SEC("license") = "GPL"; - -const volatile __u64 minimum_ns = 10 * 1000 * 1000ULL; -const volatile __u32 target_tgid; -const volatile __u32 ignored_tgid; - -struct syscall_state { - __u64 started_ns; - __u32 syscall_id; - __u32 pad; -}; - -struct { - __uint(type, BPF_MAP_TYPE_TASK_STORAGE); - __uint(map_flags, BPF_F_NO_PREALLOC); - __type(key, int); - __type(value, struct syscall_state); -} active_syscalls SEC(".maps"); - -struct { - __uint(type, BPF_MAP_TYPE_RINGBUF); - __uint(max_entries, 256 * 1024); -} events SEC(".maps"); - -__u64 completed_syscalls; -__u64 slow_syscalls; -__u64 dropped_events; - -SEC("tp/raw_syscalls/sys_enter") -int record_syscall_entry(struct trace_event_raw_sys_enter *ctx) -{ - struct task_struct *task; - struct syscall_state *state; - __u64 pid_tgid = bpf_get_current_pid_tgid(); - __u32 tgid = pid_tgid >> 32; - - if (tgid == ignored_tgid || (target_tgid && tgid != target_tgid)) - return 0; - task = bpf_get_current_task_btf(); - state = bpf_task_storage_get(&active_syscalls, task, 0, - BPF_LOCAL_STORAGE_GET_F_CREATE); - if (!state) - return 0; - state->started_ns = bpf_ktime_get_ns(); - state->syscall_id = ctx->id; - return 0; -} - -SEC("tp/raw_syscalls/sys_exit") -int report_slow_syscall(struct trace_event_raw_sys_exit *ctx) -{ - struct task_struct *task = bpf_get_current_task_btf(); - struct slow_syscall_event *event; - struct syscall_state *state; - __u64 pid_tgid = bpf_get_current_pid_tgid(); - __u64 duration_ns; - - state = bpf_task_storage_get(&active_syscalls, task, 0, 0); - if (!state) - return 0; - duration_ns = bpf_ktime_get_ns() - state->started_ns; - __sync_fetch_and_add(&completed_syscalls, 1); - if (duration_ns < minimum_ns) - goto out; - - __sync_fetch_and_add(&slow_syscalls, 1); - event = bpf_ringbuf_reserve(&events, sizeof(*event), 0); - if (!event) { - __sync_fetch_and_add(&dropped_events, 1); - goto out; - } - event->timestamp_ns = bpf_ktime_get_ns(); - event->duration_ns = duration_ns; - event->return_value = ctx->ret; - event->tgid = pid_tgid >> 32; - event->tid = (__u32)pid_tgid; - event->syscall_id = state->syscall_id; - bpf_get_current_comm(event->comm, sizeof(event->comm)); - bpf_ringbuf_submit(event, 0); -out: - bpf_task_storage_delete(&active_syscalls, task); - return 0; -} -``` - -BPF 程序使用 `BPF_MAP_TYPE_TASK_STORAGE` 存储每个任务的系统调用状态。这种映射类型在 Linux 5.11 中引入，提供在任务退出时自动清理的存储。 - -`record_syscall_entry` 函数在系统调用进入时运行，存储开始时间戳和系统调用 ID。它按目标 TGID 过滤并忽略跟踪器进程本身。 - -`report_slow_syscall` 函数在系统调用退出时运行。它通过从当前时间减去开始时间戳来计算持续时间。如果持续时间超过阈值，它在 ring buffer 中预留空间，填写事件并提交。每次系统调用后删除任务存储以保持内存使用有界。 - -程序跟踪三个全局计数器：`completed_syscalls` 计数所有跟踪的系统调用，`slow_syscalls` 计数超过阈值的调用，`dropped_events` 计数 ring buffer 分配失败。 - -## 用户空间程序 - -```c -// SPDX-License-Identifier: (LGPL-2.1 OR BSD-2-Clause) -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include "slow_syscall_index.h" -#include "slow_syscall_index.skel.h" - -#define MAX_STATS 512 - -struct options { - unsigned int pid; - unsigned int minimum_ms; - unsigned int duration_seconds; - unsigned int top; - bool demo; -}; - -struct syscall_stats { - unsigned int tgid; - unsigned int syscall_id; - unsigned long long count; - unsigned long long total_ns; - unsigned long long maximum_ns; - unsigned long long errors; - char comm[SLOW_SYSCALL_COMM_LEN]; -}; - -static struct syscall_stats stats[MAX_STATS]; -static size_t stats_count; -static volatile sig_atomic_t stop; -static unsigned int received_events; -static unsigned int unaggregated_events; - -static void handle_signal(int signal_number) -{ - (void)signal_number; - stop = 1; -} - -static unsigned long long monotonic_ns(void) -{ - struct timespec now; - - clock_gettime(CLOCK_MONOTONIC, &now); - return (unsigned long long)now.tv_sec * 1000000000ULL + now.tv_nsec; -} - -static const char *syscall_name(unsigned int id) -{ - switch (id) { -#ifdef __NR_read - case __NR_read: return "read"; -#endif -#ifdef __NR_write - case __NR_write: return "write"; -#endif -#ifdef __NR_openat - case __NR_openat: return "openat"; -#endif -#ifdef __NR_close - case __NR_close: return "close"; -#endif -#ifdef __NR_fsync - case __NR_fsync: return "fsync"; -#endif -#ifdef __NR_fdatasync - case __NR_fdatasync: return "fdatasync"; -#endif -#ifdef __NR_poll - case __NR_poll: return "poll"; -#endif -#ifdef __NR_ppoll - case __NR_ppoll: return "ppoll"; -#endif -#ifdef __NR_epoll_wait - case __NR_epoll_wait: return "epoll_wait"; -#endif -#ifdef __NR_epoll_pwait - case __NR_epoll_pwait: return "epoll_pwait"; -#endif -#ifdef __NR_futex - case __NR_futex: return "futex"; -#endif -#ifdef __NR_nanosleep - case __NR_nanosleep: return "nanosleep"; -#endif -#ifdef __NR_clock_nanosleep - case __NR_clock_nanosleep: return "clock_nanosleep"; -#endif -#ifdef __NR_connect - case __NR_connect: return "connect"; -#endif -#ifdef __NR_accept - case __NR_accept: return "accept"; -#endif -#ifdef __NR_accept4 - case __NR_accept4: return "accept4"; -#endif -#ifdef __NR_recvfrom - case __NR_recvfrom: return "recvfrom"; -#endif -#ifdef __NR_recvmsg - case __NR_recvmsg: return "recvmsg"; -#endif -#ifdef __NR_sendto - case __NR_sendto: return "sendto"; -#endif -#ifdef __NR_sendmsg - case __NR_sendmsg: return "sendmsg"; -#endif - default: return "unknown"; - } -} - -static struct syscall_stats *get_stats(const struct slow_syscall_event *event) -{ - struct syscall_stats *entry; - - for (size_t i = 0; i < stats_count; i++) - if (stats[i].tgid == event->tgid && - stats[i].syscall_id == event->syscall_id) - return &stats[i]; - if (stats_count == MAX_STATS) - return NULL; - entry = &stats[stats_count++]; - entry->tgid = event->tgid; - entry->syscall_id = event->syscall_id; - memcpy(entry->comm, event->comm, sizeof(entry->comm)); - entry->comm[sizeof(entry->comm) - 1] = '\0'; - return entry; -} - -static int handle_event(void *ctx, void *data, size_t size) -{ - const struct slow_syscall_event *event = data; - struct syscall_stats *entry; - - (void)ctx; - if (size != sizeof(*event)) - return 0; - entry = get_stats(event); - if (entry) { - entry->count++; - entry->total_ns += event->duration_ns; - if (event->duration_ns > entry->maximum_ns) - entry->maximum_ns = event->duration_ns; - if (event->return_value < 0) - entry->errors++; - } else { - unaggregated_events++; - } - received_events++; - printf("pid=%u tid=%u comm=%s syscall=%s(%u) latency_ms=%.3f return=%lld\n", - event->tgid, event->tid, event->comm, - syscall_name(event->syscall_id), event->syscall_id, - event->duration_ns / 1000000.0, event->return_value); - return 0; -} - -static int compare_stats(const void *left, const void *right) -{ - const struct syscall_stats *a = left; - const struct syscall_stats *b = right; - - return a->total_ns < b->total_ns ? 1 : a->total_ns > b->total_ns ? -1 : 0; -} - -static void print_summary(unsigned int top) -{ - size_t limit; - - qsort(stats, stats_count, sizeof(stats[0]), compare_stats); - limit = stats_count < top ? stats_count : top; - printf("\nSlow syscall index, ranked by total latency:\n"); - printf("%-7s %-16s %-18s %8s %12s %12s %8s\n", - "PID", "COMM", "SYSCALL", "COUNT", "TOTAL_MS", "MAX_MS", "ERRORS"); - for (size_t i = 0; i < limit; i++) { - const struct syscall_stats *entry = &stats[i]; - - printf("%-7u %-16s %-12s(%3u) %8llu %12.3f %12.3f %8llu\n", - entry->tgid, entry->comm, syscall_name(entry->syscall_id), - entry->syscall_id, entry->count, - entry->total_ns / 1000000.0, - entry->maximum_ns / 1000000.0, entry->errors); - } -} - -static int parse_uint(const char *text, unsigned int maximum, - unsigned int *value) -{ - char *end = NULL; - unsigned long parsed; - - errno = 0; - parsed = strtoul(text, &end, 10); - if (errno || !*text || *end || parsed > maximum) - return -1; - *value = parsed; - return 0; -} - -static void usage(const char *program) -{ - printf("Usage: %s [--pid PID] [--min-ms MS] [--duration SEC] [--top N] [--demo]\n", - program); -} - -static int parse_options(int argc, char **argv, struct options *options) -{ - static const struct option long_options[] = { - { "pid", required_argument, NULL, 'p' }, - { "min-ms", required_argument, NULL, 'm' }, - { "duration", required_argument, NULL, 'd' }, - { "top", required_argument, NULL, 't' }, - { "demo", no_argument, NULL, 'D' }, - { "help", no_argument, NULL, 'h' }, - {}, - }; - int option; - - while ((option = getopt_long(argc, argv, "p:m:d:t:Dh", long_options, - NULL)) != -1) { - switch (option) { - case 'p': - if (parse_uint(optarg, 0xffffffffU, &options->pid)) - return -1; - break; - case 'm': - if (parse_uint(optarg, 60000, &options->minimum_ms)) - return -1; - break; - case 'd': - if (parse_uint(optarg, 86400, &options->duration_seconds)) - return -1; - break; - case 't': - if (parse_uint(optarg, MAX_STATS, &options->top)) - return -1; - break; - case 'D': options->demo = true; break; - case 'h': usage(argv[0]); exit(0); - default: return -1; - } - } - return optind == argc && options->minimum_ms && options->top ? 0 : -1; -} - -static pid_t start_demo(int gate[2], int data_pipe[2]) -{ - pid_t child = fork(); - char byte; - - if (child) - return child; - close(gate[1]); - close(data_pipe[1]); - if (read(gate[0], &byte, 1) != 1) - _exit(2); - if (read(data_pipe[0], &byte, 1) != 1) - _exit(3); - _exit(0); -} - -int main(int argc, char **argv) -{ - struct options options = { .minimum_ms = 10, .top = 10 }; - struct slow_syscall_index_bpf *skel = NULL; - struct ring_buffer *ring = NULL; - unsigned long long deadline = 0; - int gate[2] = { -1, -1 }, data_pipe[2] = { -1, -1 }; - pid_t demo_child = -1; - int demo_status = 0; - int err = 1; - - setvbuf(stdout, NULL, _IONBF, 0); - if (parse_options(argc, argv, &options)) { - usage(argv[0]); - return 2; - } - if (options.demo) { - if (options.pid) { - fprintf(stderr, "--demo and --pid cannot be combined\n"); - return 2; - } - if (pipe(gate) || pipe(data_pipe)) { - perror("pipe"); - goto cleanup; - } - demo_child = start_demo(gate, data_pipe); - if (demo_child < 0) { - perror("fork"); - goto cleanup; - } - options.pid = demo_child; - } - - skel = slow_syscall_index_bpf__open(); - if (!skel) - goto cleanup; - skel->rodata->minimum_ns = (unsigned long long)options.minimum_ms * 1000000ULL; - skel->rodata->target_tgid = options.pid; - skel->rodata->ignored_tgid = getpid(); - if (slow_syscall_index_bpf__load(skel) || - slow_syscall_index_bpf__attach(skel)) { - fprintf(stderr, "failed to load and attach slow syscall tracer\n"); - goto cleanup; - } - ring = ring_buffer__new(bpf_map__fd(skel->maps.events), handle_event, - NULL, NULL); - if (!ring) - goto cleanup; - - signal(SIGINT, handle_signal); - signal(SIGTERM, handle_signal); - if (options.pid) - printf("Tracing PID %u syscalls slower than %u ms. Press Ctrl-C to stop.\n", - options.pid, options.minimum_ms); - else - printf("Tracing all syscalls slower than %u ms. Press Ctrl-C to stop.\n", - options.minimum_ms); - if (options.duration_seconds) - deadline = monotonic_ns() + - (unsigned long long)options.duration_seconds * 1000000000ULL; - - if (options.demo) { - char byte = 'x'; - struct timespec delay = { - .tv_sec = options.minimum_ms / 1000, - .tv_nsec = (long)(options.minimum_ms % 1000 + 20) * 1000000L, - }; - - if (delay.tv_nsec >= 1000000000L) { - delay.tv_sec++; - delay.tv_nsec -= 1000000000L; - } - close(gate[0]); gate[0] = -1; - close(data_pipe[0]); data_pipe[0] = -1; - write(gate[1], &byte, 1); - nanosleep(&delay, NULL); - write(data_pipe[1], &byte, 1); - waitpid(demo_child, &demo_status, 0); - demo_child = -1; - for (int i = 0; i < 20 && !received_events; i++) - ring_buffer__poll(ring, 50); - } else { - while (!stop && (!deadline || monotonic_ns() < deadline)) { - int poll_result = ring_buffer__poll(ring, 100); - - if (poll_result < 0 && poll_result != -EINTR) { - fprintf(stderr, "ring buffer poll failed: %d\n", poll_result); - goto cleanup; - } - } - } - - print_summary(options.top); - printf("completed=%llu slow=%llu dropped=%llu unaggregated=%u\n", - (unsigned long long)skel->bss->completed_syscalls, - (unsigned long long)skel->bss->slow_syscalls, - (unsigned long long)skel->bss->dropped_events, - unaggregated_events); - if (options.demo && (!WIFEXITED(demo_status) || WEXITSTATUS(demo_status) || - !received_events)) - goto cleanup; - err = 0; - -cleanup: - if (demo_child > 0) { - kill(demo_child, SIGKILL); - waitpid(demo_child, NULL, 0); - } - for (size_t i = 0; i < 2; i++) { - if (gate[i] >= 0) close(gate[i]); - if (data_pipe[i] >= 0) close(data_pipe[i]); - } - ring_buffer__free(ring); - slow_syscall_index_bpf__destroy(skel); - return err; -} -``` - -用户空间程序按 TGID 和系统调用号聚合事件。每个事件更新该组合的计数、总延迟、最大延迟和错误计数。在退出时，它按总延迟排序结果并打印前 N 个条目。 - -演示模式 fork 一个子进程，该进程阻塞在管道读取上。父进程延迟足够长的时间以超过阈值，然后写入以解除子进程阻塞。这会创建一个有保证的慢系统调用用于测试。 - -## 编译和执行 - -构建工具： - -```bash -cd src/56-slow-syscall-index -make -``` - -使用选项运行： - -```bash -sudo ./slow_syscall_index --pid 1234 --min-ms 10 --duration 60 --top 20 -``` - -或运行内置演示： - -```bash -sudo ./slow_syscall_index --demo --min-ms 10 --top 10 -``` - -示例输出： - -```text -Tracing PID 1255 syscalls slower than 10 ms. Press Ctrl-C to stop. -pid=1255 tid=1255 comm=slow_syscall_in syscall=read(0) latency_ms=30.080 return=1 - -Slow syscall index, ranked by total latency: -PID COMM SYSCALL COUNT TOTAL_MS MAX_MS ERRORS -1255 slow_syscall_in read ( 0) 1 30.080 30.080 0 -completed=1 slow=1 dropped=0 unaggregated=0 -``` - -## 环境要求 - -| 要求 | 详情 | -|------|------| -| 内核 | Linux 5.11+（BPF 任务本地存储） | -| 配置 | `CONFIG_BPF_SYSCALL`、`CONFIG_DEBUG_INFO_BTF` | -| 权限 | Root | - -## 理解输出 - -索引显示： - -- **PID**：线程组 ID，也就是事件中携带的进程 ID -- **COMM**：进程名 -- **SYSCALL**：系统调用名和号 -- **COUNT**：慢调用次数 -- **TOTAL_MS**：所有慢系统调用持续时间之和 -- **MAX_MS**：单次最长调用 -- **ERRORS**：返回负值的调用次数 - -`completed` 计数器显示所有跟踪的系统调用，`slow` 显示超过阈值的调用，`dropped` 显示 ring buffer 分配失败。用户态最多保存 512 个 `(TGID, syscall)` 分组，超过表容量的事件仍会被接收，并计入 `unaggregated`。 - -## 总结 - -任务本地存储把每次系统调用退出与对应的入口关联起来，无需在用户态维护 PID 映射。内核中的阈值过滤让事件流保持集中，用户态聚合再把慢调用整理成延迟排名。这里得到的是系统调用的挂钟延迟，如果还需要区分 off-CPU 时间，可以继续关联调度器事件。 - -> 如果你想深入了解 eBPF，请查看我们的教程代码仓库 或访问我们的网站 。 - -## 参考资料 - -- [BPF 任务本地存储文档](https://docs.ebpf.io/linux/map-type/BPF_MAP_TYPE_TASK_STORAGE/) -- [BPF ring buffer 文档](https://docs.kernel.org/6.6/bpf/ringbuf.html) -- [raw_syscalls 跟踪点](https://github.com/torvalds/linux/blob/master/include/trace/events/syscalls.h) -- [任务本地存储 commit](https://github.com/torvalds/linux/commit/4cf1bc1f1045) diff --git a/src/56-slow-syscall-index/slow_syscall_index.bpf.c b/src/56-slow-syscall-index/slow_syscall_index.bpf.c deleted file mode 100644 index 5e06a8e..0000000 --- a/src/56-slow-syscall-index/slow_syscall_index.bpf.c +++ /dev/null @@ -1,88 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0 -#include "vmlinux.h" -#include -#include "slow_syscall_index.h" - -char LICENSE[] SEC("license") = "GPL"; - -const volatile __u64 minimum_ns = 10 * 1000 * 1000ULL; -const volatile __u32 target_tgid; -const volatile __u32 ignored_tgid; - -struct syscall_state { - __u64 started_ns; - __u32 syscall_id; - __u32 pad; -}; - -struct { - __uint(type, BPF_MAP_TYPE_TASK_STORAGE); - __uint(map_flags, BPF_F_NO_PREALLOC); - __type(key, int); - __type(value, struct syscall_state); -} active_syscalls SEC(".maps"); - -struct { - __uint(type, BPF_MAP_TYPE_RINGBUF); - __uint(max_entries, 256 * 1024); -} events SEC(".maps"); - -__u64 completed_syscalls; -__u64 slow_syscalls; -__u64 dropped_events; - -SEC("tp/raw_syscalls/sys_enter") -int record_syscall_entry(struct trace_event_raw_sys_enter *ctx) -{ - struct task_struct *task; - struct syscall_state *state; - __u64 pid_tgid = bpf_get_current_pid_tgid(); - __u32 tgid = pid_tgid >> 32; - - if (tgid == ignored_tgid || (target_tgid && tgid != target_tgid)) - return 0; - task = bpf_get_current_task_btf(); - state = bpf_task_storage_get(&active_syscalls, task, 0, - BPF_LOCAL_STORAGE_GET_F_CREATE); - if (!state) - return 0; - state->started_ns = bpf_ktime_get_ns(); - state->syscall_id = ctx->id; - return 0; -} - -SEC("tp/raw_syscalls/sys_exit") -int report_slow_syscall(struct trace_event_raw_sys_exit *ctx) -{ - struct task_struct *task = bpf_get_current_task_btf(); - struct slow_syscall_event *event; - struct syscall_state *state; - __u64 pid_tgid = bpf_get_current_pid_tgid(); - __u64 duration_ns; - - state = bpf_task_storage_get(&active_syscalls, task, 0, 0); - if (!state) - return 0; - duration_ns = bpf_ktime_get_ns() - state->started_ns; - __sync_fetch_and_add(&completed_syscalls, 1); - if (duration_ns < minimum_ns) - goto out; - - __sync_fetch_and_add(&slow_syscalls, 1); - event = bpf_ringbuf_reserve(&events, sizeof(*event), 0); - if (!event) { - __sync_fetch_and_add(&dropped_events, 1); - goto out; - } - event->timestamp_ns = bpf_ktime_get_ns(); - event->duration_ns = duration_ns; - event->return_value = ctx->ret; - event->tgid = pid_tgid >> 32; - event->tid = (__u32)pid_tgid; - event->syscall_id = state->syscall_id; - bpf_get_current_comm(event->comm, sizeof(event->comm)); - bpf_ringbuf_submit(event, 0); -out: - bpf_task_storage_delete(&active_syscalls, task); - return 0; -} diff --git a/src/56-slow-syscall-index/slow_syscall_index.c b/src/56-slow-syscall-index/slow_syscall_index.c deleted file mode 100644 index e8b5cbc..0000000 --- a/src/56-slow-syscall-index/slow_syscall_index.c +++ /dev/null @@ -1,389 +0,0 @@ -// SPDX-License-Identifier: (LGPL-2.1 OR BSD-2-Clause) -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include "slow_syscall_index.h" -#include "slow_syscall_index.skel.h" - -#define MAX_STATS 512 - -struct options { - unsigned int pid; - unsigned int minimum_ms; - unsigned int duration_seconds; - unsigned int top; - bool demo; -}; - -struct syscall_stats { - unsigned int tgid; - unsigned int syscall_id; - unsigned long long count; - unsigned long long total_ns; - unsigned long long maximum_ns; - unsigned long long errors; - char comm[SLOW_SYSCALL_COMM_LEN]; -}; - -static struct syscall_stats stats[MAX_STATS]; -static size_t stats_count; -static volatile sig_atomic_t stop; -static unsigned int received_events; -static unsigned int unaggregated_events; - -static void handle_signal(int signal_number) -{ - (void)signal_number; - stop = 1; -} - -static unsigned long long monotonic_ns(void) -{ - struct timespec now; - - clock_gettime(CLOCK_MONOTONIC, &now); - return (unsigned long long)now.tv_sec * 1000000000ULL + now.tv_nsec; -} - -static const char *syscall_name(unsigned int id) -{ - switch (id) { -#ifdef __NR_read - case __NR_read: return "read"; -#endif -#ifdef __NR_write - case __NR_write: return "write"; -#endif -#ifdef __NR_openat - case __NR_openat: return "openat"; -#endif -#ifdef __NR_close - case __NR_close: return "close"; -#endif -#ifdef __NR_fsync - case __NR_fsync: return "fsync"; -#endif -#ifdef __NR_fdatasync - case __NR_fdatasync: return "fdatasync"; -#endif -#ifdef __NR_poll - case __NR_poll: return "poll"; -#endif -#ifdef __NR_ppoll - case __NR_ppoll: return "ppoll"; -#endif -#ifdef __NR_epoll_wait - case __NR_epoll_wait: return "epoll_wait"; -#endif -#ifdef __NR_epoll_pwait - case __NR_epoll_pwait: return "epoll_pwait"; -#endif -#ifdef __NR_futex - case __NR_futex: return "futex"; -#endif -#ifdef __NR_nanosleep - case __NR_nanosleep: return "nanosleep"; -#endif -#ifdef __NR_clock_nanosleep - case __NR_clock_nanosleep: return "clock_nanosleep"; -#endif -#ifdef __NR_connect - case __NR_connect: return "connect"; -#endif -#ifdef __NR_accept - case __NR_accept: return "accept"; -#endif -#ifdef __NR_accept4 - case __NR_accept4: return "accept4"; -#endif -#ifdef __NR_recvfrom - case __NR_recvfrom: return "recvfrom"; -#endif -#ifdef __NR_recvmsg - case __NR_recvmsg: return "recvmsg"; -#endif -#ifdef __NR_sendto - case __NR_sendto: return "sendto"; -#endif -#ifdef __NR_sendmsg - case __NR_sendmsg: return "sendmsg"; -#endif - default: return "unknown"; - } -} - -static struct syscall_stats *get_stats(const struct slow_syscall_event *event) -{ - struct syscall_stats *entry; - - for (size_t i = 0; i < stats_count; i++) - if (stats[i].tgid == event->tgid && - stats[i].syscall_id == event->syscall_id) - return &stats[i]; - if (stats_count == MAX_STATS) - return NULL; - entry = &stats[stats_count++]; - entry->tgid = event->tgid; - entry->syscall_id = event->syscall_id; - memcpy(entry->comm, event->comm, sizeof(entry->comm)); - entry->comm[sizeof(entry->comm) - 1] = '\0'; - return entry; -} - -static int handle_event(void *ctx, void *data, size_t size) -{ - const struct slow_syscall_event *event = data; - struct syscall_stats *entry; - - (void)ctx; - if (size != sizeof(*event)) - return 0; - entry = get_stats(event); - if (entry) { - entry->count++; - entry->total_ns += event->duration_ns; - if (event->duration_ns > entry->maximum_ns) - entry->maximum_ns = event->duration_ns; - if (event->return_value < 0) - entry->errors++; - } else { - unaggregated_events++; - } - received_events++; - printf("pid=%u tid=%u comm=%s syscall=%s(%u) latency_ms=%.3f return=%lld\n", - event->tgid, event->tid, event->comm, - syscall_name(event->syscall_id), event->syscall_id, - event->duration_ns / 1000000.0, event->return_value); - return 0; -} - -static int compare_stats(const void *left, const void *right) -{ - const struct syscall_stats *a = left; - const struct syscall_stats *b = right; - - return a->total_ns < b->total_ns ? 1 : a->total_ns > b->total_ns ? -1 : 0; -} - -static void print_summary(unsigned int top) -{ - size_t limit; - - qsort(stats, stats_count, sizeof(stats[0]), compare_stats); - limit = stats_count < top ? stats_count : top; - printf("\nSlow syscall index, ranked by total latency:\n"); - printf("%-7s %-16s %-18s %8s %12s %12s %8s\n", - "PID", "COMM", "SYSCALL", "COUNT", "TOTAL_MS", "MAX_MS", "ERRORS"); - for (size_t i = 0; i < limit; i++) { - const struct syscall_stats *entry = &stats[i]; - - printf("%-7u %-16s %-12s(%3u) %8llu %12.3f %12.3f %8llu\n", - entry->tgid, entry->comm, syscall_name(entry->syscall_id), - entry->syscall_id, entry->count, - entry->total_ns / 1000000.0, - entry->maximum_ns / 1000000.0, entry->errors); - } -} - -static int parse_uint(const char *text, unsigned int maximum, - unsigned int *value) -{ - char *end = NULL; - unsigned long parsed; - - errno = 0; - parsed = strtoul(text, &end, 10); - if (errno || !*text || *end || parsed > maximum) - return -1; - *value = parsed; - return 0; -} - -static void usage(const char *program) -{ - printf("Usage: %s [--pid PID] [--min-ms MS] [--duration SEC] [--top N] [--demo]\n", - program); -} - -static int parse_options(int argc, char **argv, struct options *options) -{ - static const struct option long_options[] = { - { "pid", required_argument, NULL, 'p' }, - { "min-ms", required_argument, NULL, 'm' }, - { "duration", required_argument, NULL, 'd' }, - { "top", required_argument, NULL, 't' }, - { "demo", no_argument, NULL, 'D' }, - { "help", no_argument, NULL, 'h' }, - {}, - }; - int option; - - while ((option = getopt_long(argc, argv, "p:m:d:t:Dh", long_options, - NULL)) != -1) { - switch (option) { - case 'p': - if (parse_uint(optarg, 0xffffffffU, &options->pid)) - return -1; - break; - case 'm': - if (parse_uint(optarg, 60000, &options->minimum_ms)) - return -1; - break; - case 'd': - if (parse_uint(optarg, 86400, &options->duration_seconds)) - return -1; - break; - case 't': - if (parse_uint(optarg, MAX_STATS, &options->top)) - return -1; - break; - case 'D': options->demo = true; break; - case 'h': usage(argv[0]); exit(0); - default: return -1; - } - } - return optind == argc && options->minimum_ms && options->top ? 0 : -1; -} - -static pid_t start_demo(int gate[2], int data_pipe[2]) -{ - pid_t child = fork(); - char byte; - - if (child) - return child; - close(gate[1]); - close(data_pipe[1]); - if (read(gate[0], &byte, 1) != 1) - _exit(2); - if (read(data_pipe[0], &byte, 1) != 1) - _exit(3); - _exit(0); -} - -int main(int argc, char **argv) -{ - struct options options = { .minimum_ms = 10, .top = 10 }; - struct slow_syscall_index_bpf *skel = NULL; - struct ring_buffer *ring = NULL; - unsigned long long deadline = 0; - int gate[2] = { -1, -1 }, data_pipe[2] = { -1, -1 }; - pid_t demo_child = -1; - int demo_status = 0; - int err = 1; - - setvbuf(stdout, NULL, _IONBF, 0); - if (parse_options(argc, argv, &options)) { - usage(argv[0]); - return 2; - } - if (options.demo) { - if (options.pid) { - fprintf(stderr, "--demo and --pid cannot be combined\n"); - return 2; - } - if (pipe(gate) || pipe(data_pipe)) { - perror("pipe"); - goto cleanup; - } - demo_child = start_demo(gate, data_pipe); - if (demo_child < 0) { - perror("fork"); - goto cleanup; - } - options.pid = demo_child; - } - - skel = slow_syscall_index_bpf__open(); - if (!skel) - goto cleanup; - skel->rodata->minimum_ns = (unsigned long long)options.minimum_ms * 1000000ULL; - skel->rodata->target_tgid = options.pid; - skel->rodata->ignored_tgid = getpid(); - if (slow_syscall_index_bpf__load(skel) || - slow_syscall_index_bpf__attach(skel)) { - fprintf(stderr, "failed to load and attach slow syscall tracer\n"); - goto cleanup; - } - ring = ring_buffer__new(bpf_map__fd(skel->maps.events), handle_event, - NULL, NULL); - if (!ring) - goto cleanup; - - signal(SIGINT, handle_signal); - signal(SIGTERM, handle_signal); - if (options.pid) - printf("Tracing PID %u syscalls slower than %u ms. Press Ctrl-C to stop.\n", - options.pid, options.minimum_ms); - else - printf("Tracing all syscalls slower than %u ms. Press Ctrl-C to stop.\n", - options.minimum_ms); - if (options.duration_seconds) - deadline = monotonic_ns() + - (unsigned long long)options.duration_seconds * 1000000000ULL; - - if (options.demo) { - char byte = 'x'; - struct timespec delay = { - .tv_sec = options.minimum_ms / 1000, - .tv_nsec = (long)(options.minimum_ms % 1000 + 20) * 1000000L, - }; - - if (delay.tv_nsec >= 1000000000L) { - delay.tv_sec++; - delay.tv_nsec -= 1000000000L; - } - close(gate[0]); gate[0] = -1; - close(data_pipe[0]); data_pipe[0] = -1; - write(gate[1], &byte, 1); - nanosleep(&delay, NULL); - write(data_pipe[1], &byte, 1); - waitpid(demo_child, &demo_status, 0); - demo_child = -1; - for (int i = 0; i < 20 && !received_events; i++) - ring_buffer__poll(ring, 50); - } else { - while (!stop && (!deadline || monotonic_ns() < deadline)) { - int poll_result = ring_buffer__poll(ring, 100); - - if (poll_result < 0 && poll_result != -EINTR) { - fprintf(stderr, "ring buffer poll failed: %d\n", poll_result); - goto cleanup; - } - } - } - - print_summary(options.top); - printf("completed=%llu slow=%llu dropped=%llu unaggregated=%u\n", - (unsigned long long)skel->bss->completed_syscalls, - (unsigned long long)skel->bss->slow_syscalls, - (unsigned long long)skel->bss->dropped_events, - unaggregated_events); - if (options.demo && (!WIFEXITED(demo_status) || WEXITSTATUS(demo_status) || - !received_events)) - goto cleanup; - err = 0; - -cleanup: - if (demo_child > 0) { - kill(demo_child, SIGKILL); - waitpid(demo_child, NULL, 0); - } - for (size_t i = 0; i < 2; i++) { - if (gate[i] >= 0) close(gate[i]); - if (data_pipe[i] >= 0) close(data_pipe[i]); - } - ring_buffer__free(ring); - slow_syscall_index_bpf__destroy(skel); - return err; -} diff --git a/src/56-slow-syscall-index/slow_syscall_index.h b/src/56-slow-syscall-index/slow_syscall_index.h deleted file mode 100644 index f4af45a..0000000 --- a/src/56-slow-syscall-index/slow_syscall_index.h +++ /dev/null @@ -1,17 +0,0 @@ -/* SPDX-License-Identifier: (LGPL-2.1 OR BSD-2-Clause) */ -#ifndef __SLOW_SYSCALL_INDEX_H -#define __SLOW_SYSCALL_INDEX_H - -#define SLOW_SYSCALL_COMM_LEN 16 - -struct slow_syscall_event { - unsigned long long timestamp_ns; - unsigned long long duration_ns; - long long return_value; - unsigned int tgid; - unsigned int tid; - unsigned int syscall_id; - char comm[SLOW_SYSCALL_COMM_LEN]; -}; - -#endif /* __SLOW_SYSCALL_INDEX_H */ diff --git a/src/56-slow-syscall-index/tests/test_slow_syscall_index.py b/src/56-slow-syscall-index/tests/test_slow_syscall_index.py deleted file mode 100644 index 61d4013..0000000 --- a/src/56-slow-syscall-index/tests/test_slow_syscall_index.py +++ /dev/null @@ -1,25 +0,0 @@ -#!/usr/bin/env python3 -import subprocess -import sys - - -def main(): - result = subprocess.run( - [sys.argv[1], "--demo", "--min-ms", "10"], - text=True, - capture_output=True, - check=False, - timeout=30, - ) - output = result.stdout + result.stderr - if result.returncode: - raise AssertionError(output) - assert "syscall=read(" in output, output - assert "Slow syscall index, ranked by total latency:" in output, output - assert "completed=" in output and "slow=1" in output, output - assert "unaggregated=0" in output, output - print("slow syscall index integration test: PASS") - - -if __name__ == "__main__": - main() diff --git a/src/56-tc-flow-index/.config b/src/56-tc-flow-index/.config new file mode 100644 index 0000000..940e6f6 --- /dev/null +++ b/src/56-tc-flow-index/.config @@ -0,0 +1,10 @@ +level=Depth +type=Networking +kernel_min=6.16 +kernel_min_basis=required-feature +architectures=x86_64 +btf=required +kernel_config=CONFIG_BPF=y,CONFIG_BPF_SYSCALL=y,CONFIG_BPF_JIT=y,CONFIG_DEBUG_INFO_BTF=y,CONFIG_NET_SCHED=y,CONFIG_NET_CLS_BPF=y +hardware=network-interface +root=required +test_status=ci-build diff --git a/src/56-tc-flow-index/.gitignore b/src/56-tc-flow-index/.gitignore new file mode 100644 index 0000000..ff08a9d --- /dev/null +++ b/src/56-tc-flow-index/.gitignore @@ -0,0 +1,2 @@ +.output/ +tc_flow_index diff --git a/src/56-slow-syscall-index/Makefile b/src/56-tc-flow-index/Makefile similarity index 97% rename from src/56-slow-syscall-index/Makefile rename to src/56-tc-flow-index/Makefile index 0f5c8c3..7e0a404 100644 --- a/src/56-slow-syscall-index/Makefile +++ b/src/56-tc-flow-index/Makefile @@ -17,7 +17,7 @@ VMLINUX := ../third_party/vmlinux/$(ARCH)/vmlinux.h INCLUDES := -I$(OUTPUT) -I../third_party/libbpf/include/uapi -I$(dir $(VMLINUX)) -I. CFLAGS := -g -Wall -Wextra -D_GNU_SOURCE ALL_LDFLAGS := $(LDFLAGS) $(EXTRA_LDFLAGS) -APPS = slow_syscall_index +APPS = tc_flow_index CLANG_BPF_SYS_INCLUDES ?= $(shell $(CLANG) -v -E - &1 \ | sed -n '/<...> search starts here:/,/End of search list./{ s| \(/.*\)|-idirafter \1|p }') @@ -53,7 +53,7 @@ test: all echo "test requires root; run inside the documented KVM guest" >&2; \ exit 1; \ fi - python3 tests/test_slow_syscall_index.py ./slow_syscall_index + python3 tests/test_tc_flow_index.py ./tc_flow_index $(OUTPUT) $(OUTPUT)/libbpf $(BPFTOOL_OUTPUT): $(call msg,MKDIR,$@) diff --git a/src/56-tc-flow-index/bpf_experimental.h b/src/56-tc-flow-index/bpf_experimental.h new file mode 100644 index 0000000..16cd8d2 --- /dev/null +++ b/src/56-tc-flow-index/bpf_experimental.h @@ -0,0 +1,40 @@ +/* SPDX-License-Identifier: (LGPL-2.1 OR BSD-2-Clause) */ +#ifndef __TC_FLOW_INDEX_EXPERIMENTAL_H +#define __TC_FLOW_INDEX_EXPERIMENTAL_H + +#include + +#define __contains(name, node) \ + __attribute__((btf_decl_tag("contains:" #name ":" #node))) + +extern void *bpf_obj_new_impl(__u64 local_type_id, void *meta) __ksym; +#define bpf_obj_new(type) \ + ((type *)bpf_obj_new_impl(bpf_core_type_id_local(type), NULL)) + +extern void bpf_obj_drop_impl(void *kptr, void *meta) __ksym; +#define bpf_obj_drop(kptr) bpf_obj_drop_impl(kptr, NULL) + +extern void *bpf_refcount_acquire_impl(void *kptr, void *meta) __ksym; +#define bpf_refcount_acquire(kptr) \ + bpf_refcount_acquire_impl(kptr, NULL) + +extern int bpf_rbtree_add_impl(struct bpf_rb_root *root, + struct bpf_rb_node *node, + bool (*less)(struct bpf_rb_node *, + const struct bpf_rb_node *), + void *meta, __u64 off) __ksym; +#define bpf_rbtree_add(root, node, less) \ + bpf_rbtree_add_impl(root, node, less, NULL, 0) + +extern struct bpf_rb_node * +bpf_rbtree_remove(struct bpf_rb_root *root, struct bpf_rb_node *node) __ksym; +extern struct bpf_rb_node * +bpf_rbtree_first(struct bpf_rb_root *root) __ksym; +extern struct bpf_rb_node * +bpf_rbtree_root(struct bpf_rb_root *root) __ksym; +extern struct bpf_rb_node * +bpf_rbtree_left(struct bpf_rb_root *root, struct bpf_rb_node *node) __ksym; +extern struct bpf_rb_node * +bpf_rbtree_right(struct bpf_rb_root *root, struct bpf_rb_node *node) __ksym; + +#endif /* __TC_FLOW_INDEX_EXPERIMENTAL_H */ diff --git a/src/56-tc-flow-index/tc_flow_index.bpf.c b/src/56-tc-flow-index/tc_flow_index.bpf.c new file mode 100644 index 0000000..ede33f7 --- /dev/null +++ b/src/56-tc-flow-index/tc_flow_index.bpf.c @@ -0,0 +1,286 @@ +// SPDX-License-Identifier: GPL-2.0 +#include "vmlinux.h" +#include +#include +#include "bpf_experimental.h" +#include "tc_flow_index.h" + +char LICENSE[] SEC("license") = "GPL"; + +#define ETH_P_IP 0x0800 +#define IPPROTO_TCP 6 +#define IPPROTO_UDP 17 +#define IP_MF 0x2000 +#define IP_OFFSET 0x1fff +#define INDEX_SEARCH_DEPTH 32 +#define TC_ACT_OK 0 + +const volatile __u32 maximum_entries = 4096; + +struct flow_entry { + struct bpf_refcount ref; + struct bpf_rb_node by_identity; + struct bpf_rb_node by_traffic; + struct flow_key key; + __u64 packets; + __u64 bytes; + __u64 last_seen_ns; + char comm[FLOW_COMM_LEN]; +}; + +#define private(name) \ + SEC(".data." #name) __hidden __attribute__((aligned(8))) + +private(index) struct bpf_spin_lock index_lock; +private(index) struct bpf_rb_root identity_root + __contains(flow_entry, by_identity); +private(index) struct bpf_rb_root traffic_root + __contains(flow_entry, by_traffic); + +struct flow_cursor snapshot_cursor; +struct flow_snapshot snapshot_result; +__u64 observed_packets; +__u64 indexed_flows; +__u64 dropped_new_flows; +__u64 allocation_failures; +__u64 refcount_failures; +__u64 rank_update_failures; + +static __always_inline int compare_key(const struct flow_key *left, + const struct flow_key *right) +{ + if (left->source_ip != right->source_ip) + return left->source_ip < right->source_ip ? -1 : 1; + if (left->destination_ip != right->destination_ip) + return left->destination_ip < right->destination_ip ? -1 : 1; + if (left->source_port != right->source_port) + return left->source_port < right->source_port ? -1 : 1; + if (left->destination_port != right->destination_port) + return left->destination_port < right->destination_port ? -1 : 1; + if (left->protocol != right->protocol) + return left->protocol < right->protocol ? -1 : 1; + return 0; +} + +static bool identity_less(struct bpf_rb_node *a, + const struct bpf_rb_node *b) +{ + struct flow_entry *left = container_of(a, struct flow_entry, + by_identity); + struct flow_entry *right = container_of(b, struct flow_entry, + by_identity); + + return compare_key(&left->key, &right->key) < 0; +} + +static __always_inline bool traffic_before(__u64 left_bytes, + __u64 left_packets, + const struct flow_key *left_key, + __u64 right_bytes, + __u64 right_packets, + const struct flow_key *right_key) +{ + if (left_bytes != right_bytes) + return left_bytes > right_bytes; + if (left_packets != right_packets) + return left_packets > right_packets; + return compare_key(left_key, right_key) < 0; +} + +static bool traffic_less(struct bpf_rb_node *a, + const struct bpf_rb_node *b) +{ + struct flow_entry *left = container_of(a, struct flow_entry, by_traffic); + struct flow_entry *right = container_of(b, struct flow_entry, by_traffic); + + return traffic_before(left->bytes, left->packets, &left->key, + right->bytes, right->packets, &right->key); +} + +static __always_inline struct flow_entry * +find_flow_locked(const struct flow_key *key) +{ + struct bpf_rb_node *rb = bpf_rbtree_root(&identity_root); + + for (int i = 0; i < INDEX_SEARCH_DEPTH && rb; i++) { + struct flow_entry *entry = + container_of(rb, struct flow_entry, by_identity); + int comparison = compare_key(key, &entry->key); + + if (!comparison) + return entry; + if (comparison < 0) + rb = bpf_rbtree_left(&identity_root, rb); + else + rb = bpf_rbtree_right(&identity_root, rb); + } + return NULL; +} + +static __always_inline bool update_existing_locked(struct flow_entry *entry, + __u32 packet_bytes, + __u64 now_ns) +{ + struct flow_entry *owner; + struct bpf_rb_node *removed; + + removed = bpf_rbtree_remove(&traffic_root, &entry->by_traffic); + if (!removed) + return false; + owner = container_of(removed, struct flow_entry, by_traffic); + owner->packets++; + owner->bytes += packet_bytes; + owner->last_seen_ns = now_ns; + bpf_rbtree_add(&traffic_root, &owner->by_traffic, traffic_less); + return true; +} + +static __always_inline void update_flow(const struct flow_key *key, + __u32 packet_bytes, __u64 now_ns, + const char comm[FLOW_COMM_LEN]) +{ + struct flow_entry *entry, *new_entry, *traffic_owner; + + bpf_spin_lock(&index_lock); + entry = find_flow_locked(key); + if (entry) { + if (!update_existing_locked(entry, packet_bytes, now_ns)) + rank_update_failures++; + bpf_spin_unlock(&index_lock); + return; + } + bpf_spin_unlock(&index_lock); + + new_entry = bpf_obj_new(typeof(*new_entry)); + if (!new_entry) { + __sync_fetch_and_add(&allocation_failures, 1); + return; + } + traffic_owner = bpf_refcount_acquire(new_entry); + if (!traffic_owner) { + __sync_fetch_and_add(&refcount_failures, 1); + bpf_obj_drop(new_entry); + return; + } + new_entry->key.source_ip = key->source_ip; + new_entry->key.destination_ip = key->destination_ip; + new_entry->key.source_port = key->source_port; + new_entry->key.destination_port = key->destination_port; + new_entry->key.protocol = key->protocol; + new_entry->packets = 1; + new_entry->bytes = packet_bytes; + new_entry->last_seen_ns = now_ns; + __builtin_memcpy(new_entry->comm, comm, sizeof(new_entry->comm)); + + bpf_spin_lock(&index_lock); + entry = find_flow_locked(key); + if (entry) { + if (!update_existing_locked(entry, packet_bytes, now_ns)) + rank_update_failures++; + bpf_spin_unlock(&index_lock); + bpf_obj_drop(new_entry); + bpf_obj_drop(traffic_owner); + return; + } + if (indexed_flows >= maximum_entries) { + dropped_new_flows++; + bpf_spin_unlock(&index_lock); + bpf_obj_drop(new_entry); + bpf_obj_drop(traffic_owner); + return; + } + bpf_rbtree_add(&identity_root, &new_entry->by_identity, identity_less); + bpf_rbtree_add(&traffic_root, &traffic_owner->by_traffic, traffic_less); + indexed_flows++; + bpf_spin_unlock(&index_lock); +} + +static __always_inline bool parse_flow(struct __sk_buff *skb, + struct flow_key *key) +{ + struct ethhdr ethernet; + struct iphdr ip; + __u32 transport_offset; + __be16 ports[2]; + + if (bpf_skb_load_bytes(skb, 0, ðernet, sizeof(ethernet)) || + ethernet.h_proto != bpf_htons(ETH_P_IP)) + return false; + if (bpf_skb_load_bytes(skb, sizeof(ethernet), &ip, sizeof(ip)) || + ip.version != 4 || ip.ihl < 5 || + (bpf_ntohs(ip.frag_off) & (IP_MF | IP_OFFSET)) || + (ip.protocol != IPPROTO_TCP && ip.protocol != IPPROTO_UDP)) + return false; + transport_offset = sizeof(ethernet) + ip.ihl * 4; + if (bpf_skb_load_bytes(skb, transport_offset, ports, sizeof(ports))) + return false; + key->source_ip = ip.saddr; + key->destination_ip = ip.daddr; + key->source_port = ports[0]; + key->destination_port = ports[1]; + key->protocol = ip.protocol; + return true; +} + +SEC("tc") +int index_egress_flow(struct __sk_buff *skb) +{ + struct flow_key key = {}; + char comm[FLOW_COMM_LEN]; + + if (!parse_flow(skb, &key)) + return TC_ACT_OK; + __sync_fetch_and_add(&observed_packets, 1); + bpf_get_current_comm(comm, sizeof(comm)); + update_flow(&key, skb->len, bpf_ktime_get_ns(), comm); + return TC_ACT_OK; +} + +static __always_inline bool cursor_before_entry(const struct flow_cursor *cursor, + const struct flow_entry *entry) +{ + return traffic_before(cursor->bytes, cursor->packets, &cursor->key, + entry->bytes, entry->packets, &entry->key); +} + +SEC("syscall") +int snapshot_next(void *ctx) +{ + struct flow_entry *entry = NULL; + struct bpf_rb_node *candidate = NULL; + struct bpf_rb_node *rb; + + (void)ctx; + __builtin_memset(&snapshot_result, 0, sizeof(snapshot_result)); + bpf_spin_lock(&index_lock); + if (!snapshot_cursor.valid) { + candidate = bpf_rbtree_first(&traffic_root); + } else { + rb = bpf_rbtree_root(&traffic_root); + for (int i = 0; i < INDEX_SEARCH_DEPTH && rb; i++) { + entry = container_of(rb, struct flow_entry, by_traffic); + if (cursor_before_entry(&snapshot_cursor, entry)) { + candidate = rb; + rb = bpf_rbtree_left(&traffic_root, rb); + } else { + rb = bpf_rbtree_right(&traffic_root, rb); + } + } + } + if (candidate) { + entry = container_of(candidate, struct flow_entry, by_traffic); + snapshot_result.packets = entry->packets; + snapshot_result.bytes = entry->bytes; + snapshot_result.last_seen_ns = entry->last_seen_ns; + snapshot_result.key.source_ip = entry->key.source_ip; + snapshot_result.key.destination_ip = entry->key.destination_ip; + snapshot_result.key.source_port = entry->key.source_port; + snapshot_result.key.destination_port = entry->key.destination_port; + snapshot_result.key.protocol = entry->key.protocol; + snapshot_result.found = 1; + __builtin_memcpy(snapshot_result.comm, entry->comm, + sizeof(snapshot_result.comm)); + } + bpf_spin_unlock(&index_lock); + return 0; +} diff --git a/src/56-tc-flow-index/tc_flow_index.c b/src/56-tc-flow-index/tc_flow_index.c new file mode 100644 index 0000000..b64d2cc --- /dev/null +++ b/src/56-tc-flow-index/tc_flow_index.c @@ -0,0 +1,316 @@ +// SPDX-License-Identifier: (LGPL-2.1 OR BSD-2-Clause) +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "tc_flow_index.h" +#include "tc_flow_index.skel.h" + +#define MAX_TOP 64 + +struct options { + const char *interface; + unsigned int duration_seconds; + unsigned int top; + bool demo; +}; + +static volatile sig_atomic_t stop; + +static void handle_signal(int signal_number) +{ + (void)signal_number; + stop = 1; +} + +static unsigned long long monotonic_ns(void) +{ + struct timespec now; + + clock_gettime(CLOCK_MONOTONIC, &now); + return (unsigned long long)now.tv_sec * 1000000000ULL + now.tv_nsec; +} + +static int parse_uint(const char *text, unsigned int maximum, + unsigned int *value) +{ + char *end = NULL; + unsigned long parsed; + + errno = 0; + parsed = strtoul(text, &end, 10); + if (errno || !*text || *end || !parsed || parsed > maximum) + return -1; + *value = parsed; + return 0; +} + +static void usage(const char *program) +{ + printf("Usage: %s --interface IFACE [--duration SEC] [--top N]\n" + " %s --demo [--top N]\n", program, program); +} + +static int parse_options(int argc, char **argv, struct options *options) +{ + static const struct option long_options[] = { + { "interface", required_argument, NULL, 'i' }, + { "duration", required_argument, NULL, 'd' }, + { "top", required_argument, NULL, 't' }, + { "demo", no_argument, NULL, 'D' }, + { "help", no_argument, NULL, 'h' }, + {}, + }; + int option; + + while ((option = getopt_long(argc, argv, "i:d:t:Dh", long_options, + NULL)) != -1) { + switch (option) { + case 'i': options->interface = optarg; break; + case 'd': + if (parse_uint(optarg, 86400, &options->duration_seconds)) + return -1; + break; + case 't': + if (parse_uint(optarg, MAX_TOP, &options->top)) + return -1; + break; + case 'D': options->demo = true; break; + case 'h': usage(argv[0]); exit(0); + default: return -1; + } + } + if (options->demo) { + if (options->interface) + return -1; + options->interface = "lo"; + } + return optind == argc && options->interface ? 0 : -1; +} + +static int send_demo_flow(unsigned int datagrams, size_t payload_size) +{ + struct sockaddr_in receiver_address = { + .sin_family = AF_INET, + .sin_addr.s_addr = htonl(INADDR_LOOPBACK), + }; + struct sockaddr_in sender_address = { + .sin_family = AF_INET, + .sin_addr.s_addr = htonl(INADDR_LOOPBACK), + }; + socklen_t address_length = sizeof(receiver_address); + char payload[1000] = {}; + int receiver = -1, sender = -1; + int err = -1; + + if (payload_size > sizeof(payload)) + return -1; + receiver = socket(AF_INET, SOCK_DGRAM | SOCK_CLOEXEC, 0); + sender = socket(AF_INET, SOCK_DGRAM | SOCK_CLOEXEC, 0); + if (receiver < 0 || sender < 0 || + bind(receiver, (struct sockaddr *)&receiver_address, + sizeof(receiver_address)) || + getsockname(receiver, (struct sockaddr *)&receiver_address, + &address_length) || + bind(sender, (struct sockaddr *)&sender_address, + sizeof(sender_address))) + goto cleanup; + for (unsigned int i = 0; i < datagrams; i++) + if (sendto(sender, payload, payload_size, 0, + (struct sockaddr *)&receiver_address, + sizeof(receiver_address)) != (ssize_t)payload_size) + goto cleanup; + err = 0; + +cleanup: + if (receiver >= 0) close(receiver); + if (sender >= 0) close(sender); + return err; +} + +static int run_demo_traffic(void) +{ + return send_demo_flow(2, 100) || + send_demo_flow(4, 300) || + send_demo_flow(6, 700); +} + +static int snapshot_next(struct tc_flow_index_bpf *skel, + struct flow_snapshot *result) +{ + LIBBPF_OPTS(bpf_test_run_opts, opts); + + if (bpf_prog_test_run_opts(bpf_program__fd(skel->progs.snapshot_next), + &opts)) { + fprintf(stderr, "failed to query BPF flow index: %s\n", + strerror(errno)); + return -1; + } + *result = skel->bss->snapshot_result; + if (result->found) { + skel->bss->snapshot_cursor.bytes = result->bytes; + skel->bss->snapshot_cursor.packets = result->packets; + skel->bss->snapshot_cursor.key = result->key; + skel->bss->snapshot_cursor.valid = 1; + } + return 0; +} + +static int print_flows(struct tc_flow_index_bpf *skel, unsigned int top) +{ + struct flow_snapshot entry; + + memset(&skel->bss->snapshot_cursor, 0, + sizeof(skel->bss->snapshot_cursor)); + printf("\nTop egress flows, ranked in the BPF rbtree by bytes:\n"); + printf("%-21s %-21s %-5s %10s %12s %-16s\n", + "SOURCE", "DESTINATION", "PROTO", "PACKETS", "BYTES", "COMM"); + for (unsigned int i = 0; i < top; i++) { + char source_ip[INET_ADDRSTRLEN], destination_ip[INET_ADDRSTRLEN]; + char source[64], destination[64]; + + if (snapshot_next(skel, &entry)) + return -1; + if (!entry.found) + break; + inet_ntop(AF_INET, &entry.key.source_ip, source_ip, + sizeof(source_ip)); + inet_ntop(AF_INET, &entry.key.destination_ip, destination_ip, + sizeof(destination_ip)); + snprintf(source, sizeof(source), "%s:%u", source_ip, + ntohs(entry.key.source_port)); + snprintf(destination, sizeof(destination), "%s:%u", + destination_ip, ntohs(entry.key.destination_port)); + printf("%-21s %-21s %-5s %10llu %12llu %-16s\n", + source, destination, + entry.key.protocol == IPPROTO_TCP ? "TCP" : "UDP", + entry.packets, entry.bytes, entry.comm); + } + return 0; +} + +static int attach_tc_program(struct bpf_tc_hook *hook, + struct bpf_tc_opts *attach, + bool *hook_created, bool *attached) +{ + int err = bpf_tc_hook_create(hook); + + if (!err) + *hook_created = true; + else if (err != -EEXIST) { + fprintf(stderr, "failed to create clsact hook: %s\n", strerror(-err)); + return -1; + } + err = bpf_tc_attach(hook, attach); + if (err) { + fprintf(stderr, "failed to attach TC program: %s\n", strerror(-err)); + return -1; + } + *attached = true; + return 0; +} + +static int capture_traffic(const struct options *options) +{ + unsigned long long deadline; + + if (options->demo) + return run_demo_traffic(); + signal(SIGINT, handle_signal); + signal(SIGTERM, handle_signal); + deadline = monotonic_ns() + + (unsigned long long)options->duration_seconds * 1000000000ULL; + while (!stop && monotonic_ns() < deadline) { + struct timespec pause = { .tv_nsec = 100000000 }; + + nanosleep(&pause, NULL); + } + return 0; +} + +static int detach_tc_program(struct bpf_tc_hook *hook, + struct bpf_tc_opts *detach, bool *attached) +{ + int err = bpf_tc_detach(hook, detach); + + if (err) { + fprintf(stderr, "failed to detach TC program: %s\n", strerror(-err)); + return -1; + } + *attached = false; + return 0; +} + +int main(int argc, char **argv) +{ + struct options options = { .duration_seconds = 10, .top = 10 }; + struct tc_flow_index_bpf *skel = NULL; + LIBBPF_OPTS(bpf_tc_hook, hook, .attach_point = BPF_TC_EGRESS); + LIBBPF_OPTS(bpf_tc_opts, attach, .handle = 1, .priority = 1); + LIBBPF_OPTS(bpf_tc_opts, detach, .handle = 1, .priority = 1); + bool hook_created = false; + bool attached = false; + int err = 1; + + setvbuf(stdout, NULL, _IONBF, 0); + if (parse_options(argc, argv, &options)) { + usage(argv[0]); + return 2; + } + hook.ifindex = if_nametoindex(options.interface); + if (!hook.ifindex) { + fprintf(stderr, "interface does not exist: %s\n", options.interface); + return 2; + } + + skel = tc_flow_index_bpf__open_and_load(); + if (!skel) { + fprintf(stderr, "failed to load TC flow index\n"); + goto cleanup; + } + attach.prog_fd = bpf_program__fd(skel->progs.index_egress_flow); + if (attach_tc_program(&hook, &attach, &hook_created, &attached)) + goto cleanup; + printf("Indexing IPv4 TCP/UDP egress flows on %s for %u seconds.\n", + options.interface, options.demo ? 0 : options.duration_seconds); + + if (capture_traffic(&options)) { + fprintf(stderr, "failed to generate traffic\n"); + goto cleanup; + } + + if (detach_tc_program(&hook, &detach, &attached)) + goto cleanup; + if (print_flows(skel, options.top)) + goto cleanup; + printf("observed_packets=%llu indexed_flows=%llu dropped_new=%llu " + "allocation_failures=%llu refcount_failures=%llu rank_update_failures=%llu\n", + (unsigned long long)skel->bss->observed_packets, + (unsigned long long)skel->bss->indexed_flows, + (unsigned long long)skel->bss->dropped_new_flows, + (unsigned long long)skel->bss->allocation_failures, + (unsigned long long)skel->bss->refcount_failures, + (unsigned long long)skel->bss->rank_update_failures); + if (options.demo && + (skel->bss->indexed_flows != 3 || skel->bss->rank_update_failures)) + goto cleanup; + err = 0; + +cleanup: + if (attached) + bpf_tc_detach(&hook, &detach); + if (hook_created) + bpf_tc_hook_destroy(&hook); + tc_flow_index_bpf__destroy(skel); + return err; +} diff --git a/src/56-tc-flow-index/tc_flow_index.h b/src/56-tc-flow-index/tc_flow_index.h new file mode 100644 index 0000000..1fc33ca --- /dev/null +++ b/src/56-tc-flow-index/tc_flow_index.h @@ -0,0 +1,32 @@ +/* SPDX-License-Identifier: (LGPL-2.1 OR BSD-2-Clause) */ +#ifndef __TC_FLOW_INDEX_H +#define __TC_FLOW_INDEX_H + +#define FLOW_COMM_LEN 16 + +struct flow_key { + unsigned int source_ip; + unsigned int destination_ip; + unsigned short source_port; + unsigned short destination_port; + unsigned char protocol; + unsigned char padding[3]; +}; + +struct flow_snapshot { + unsigned long long packets; + unsigned long long bytes; + unsigned long long last_seen_ns; + struct flow_key key; + unsigned int found; + char comm[FLOW_COMM_LEN]; +}; + +struct flow_cursor { + unsigned long long bytes; + unsigned long long packets; + struct flow_key key; + unsigned int valid; +}; + +#endif /* __TC_FLOW_INDEX_H */ diff --git a/src/56-tc-flow-index/tests/test_tc_flow_index.py b/src/56-tc-flow-index/tests/test_tc_flow_index.py new file mode 100644 index 0000000..0de5a1a --- /dev/null +++ b/src/56-tc-flow-index/tests/test_tc_flow_index.py @@ -0,0 +1,44 @@ +#!/usr/bin/env python3 +import re +import subprocess +import sys + + +def run_demo(): + result = subprocess.run( + [sys.argv[1], "--demo", "--top", "3"], + text=True, + capture_output=True, + check=False, + timeout=30, + ) + output = result.stdout + result.stderr + if result.returncode: + raise AssertionError(output) + assert "ranked in the BPF rbtree by bytes:" in output, output + assert "indexed_flows=3" in output, output + assert "allocation_failures=0" in output, output + assert "refcount_failures=0" in output, output + assert "rank_update_failures=0" in output, output + byte_counts = [int(value) for value in re.findall(r"UDP\s+\d+\s+(\d+)", output)] + assert len(byte_counts) == 3, output + assert byte_counts == sorted(byte_counts, reverse=True), output + + +def main(): + run_demo() + run_demo() + invalid = subprocess.run( + [sys.argv[1], "--interface", "no-such-interface"], + text=True, + capture_output=True, + check=False, + timeout=5, + ) + assert invalid.returncode == 2, invalid.stdout + invalid.stderr + assert "interface does not exist" in invalid.stderr, invalid.stderr + print("TC flow index integration test: PASS") + + +if __name__ == "__main__": + main() diff --git a/src/57-oom-watch/.config b/src/57-oom-watch/.config new file mode 100644 index 0000000..61fe7f7 --- /dev/null +++ b/src/57-oom-watch/.config @@ -0,0 +1,10 @@ +level=Depth +type=Tracing +kernel_min=7.1 +kernel_min_basis=required-feature +architectures=x86_64 +btf=required +kernel_config=CONFIG_BPF=y,CONFIG_BPF_SYSCALL=y,CONFIG_BPF_JIT=y,CONFIG_BPF_EVENTS=y,CONFIG_DEBUG_INFO_BTF=y,CONFIG_MEMCG=y +hardware=none +root=required +test_status=ci-build diff --git a/src/57-oom-watch/oom_watch.bpf.c b/src/57-oom-watch/oom_watch.bpf.c index fc9fefc..d1c5d34 100644 --- a/src/57-oom-watch/oom_watch.bpf.c +++ b/src/57-oom-watch/oom_watch.bpf.c @@ -8,16 +8,16 @@ char LICENSE[] SEC("license") = "GPL"; const volatile __u64 target_cgroup_id; +const volatile __u32 sample_every = 1; extern struct task_struct *bpf_task_from_pid(__s32 pid) __ksym; extern void bpf_task_release(struct task_struct *task) __ksym; -struct reclaim_profile { - __u64 begin_count; - __u64 end_count; - __u64 reclaimed_pages; - __u64 cross_cgroup_reclaims; - __u64 last_reclaim_ns; +struct active_reclaim { + __u64 started_ns; + __u64 cgroup_id; + __s32 stack_id; + __u32 padding; }; struct victim_state { @@ -33,6 +33,27 @@ struct { __type(value, struct reclaim_profile); } profiles SEC(".maps"); +struct { + __uint(type, BPF_MAP_TYPE_LRU_HASH); + __uint(max_entries, 4096); + __type(key, __u64); + __type(value, struct active_reclaim); +} active_reclaims SEC(".maps"); + +struct { + __uint(type, BPF_MAP_TYPE_STACK_TRACE); + __uint(max_entries, 1024); + __uint(key_size, sizeof(__u32)); + __uint(value_size, OOM_STACK_DEPTH * sizeof(__u64)); +} stack_traces SEC(".maps"); + +struct { + __uint(type, BPF_MAP_TYPE_LRU_HASH); + __uint(max_entries, 8192); + __type(key, struct reclaim_stack_key); + __type(value, struct reclaim_stack_profile); +} stack_profiles SEC(".maps"); + struct { __uint(type, BPF_MAP_TYPE_HASH); __uint(max_entries, 1024); @@ -46,6 +67,7 @@ struct { } events SEC(".maps"); __u64 dropped_victim_states; +__u64 dropped_reclaim_states; static __always_inline bool selected_cgroup(__u64 cgroup_id) { @@ -89,12 +111,68 @@ static __always_inline struct reclaim_profile *get_profile(__u64 cgroup_id) return bpf_map_lookup_elem(&profiles, &cgroup_id); } +static __always_inline __u32 latency_bucket(__u64 duration_ns) +{ + __u64 microseconds = duration_ns / 1000; + __u32 bucket = 0; + + for (int i = 0; i < OOM_RECLAIM_BUCKETS - 1; i++) { + if (microseconds < 2) + break; + microseconds >>= 1; + bucket++; + } + return bucket; +} + +static __always_inline void update_maximum(__u64 *maximum, __u64 value) +{ + __u64 previous = *maximum; + + for (int i = 0; i < 8 && previous < value; i++) { + __u64 observed = __sync_val_compare_and_swap(maximum, previous, + value); + + if (observed == previous) + break; + previous = observed; + } +} + +static __always_inline void update_stack_profile(__u64 cgroup_id, + __s32 stack_id, + __u64 duration_ns, + __u64 reclaimed) +{ + struct reclaim_stack_key key = { + .cgroup_id = cgroup_id, + .stack_id = stack_id, + }; + struct reclaim_stack_profile initial = {}; + struct reclaim_stack_profile *profile; + + profile = bpf_map_lookup_elem(&stack_profiles, &key); + if (!profile) { + bpf_map_update_elem(&stack_profiles, &key, &initial, BPF_NOEXIST); + profile = bpf_map_lookup_elem(&stack_profiles, &key); + } + if (!profile) + return; + __sync_fetch_and_add(&profile->samples, 1); + __sync_fetch_and_add(&profile->total_ns, duration_ns); + __sync_fetch_and_add(&profile->reclaimed_pages, reclaimed); + update_maximum(&profile->maximum_ns, duration_ns); +} + SEC("tp_btf/mm_vmscan_memcg_reclaim_begin") -int BPF_PROG(track_reclaim_begin, gfp_t gfp_flags, int order, +int BPF_PROG(profile_reclaim_begin, gfp_t gfp_flags, int order, struct mem_cgroup *memcg) { + struct active_reclaim active = { .stack_id = -1 }; struct reclaim_profile *profile; + __u64 pid_tgid = bpf_get_current_pid_tgid(); __u64 cgroup_id = memcg_cgroup_id(memcg); + __u64 sequence; (void)gfp_flags; (void)order; @@ -103,28 +181,56 @@ int BPF_PROG(track_reclaim_begin, gfp_t gfp_flags, int order, profile = get_profile(cgroup_id); if (!profile) return 0; - __sync_fetch_and_add(&profile->begin_count, 1); + sequence = __sync_fetch_and_add(&profile->begin_count, 1); if (bpf_get_current_cgroup_id() != cgroup_id) __sync_fetch_and_add(&profile->cross_cgroup_reclaims, 1); profile->last_reclaim_ns = bpf_ktime_get_ns(); + active.started_ns = profile->last_reclaim_ns; + active.cgroup_id = cgroup_id; + if (!sample_every || sequence % sample_every == 0) { + active.stack_id = bpf_get_stackid(ctx, &stack_traces, + BPF_F_FAST_STACK_CMP | 2); + if (active.stack_id >= 0) + __sync_fetch_and_add(&profile->stack_samples, 1); + else + __sync_fetch_and_add(&profile->stack_failures, 1); + } + if (bpf_map_update_elem(&active_reclaims, &pid_tgid, &active, BPF_ANY)) + __sync_fetch_and_add(&dropped_reclaim_states, 1); return 0; } SEC("tp_btf/mm_vmscan_memcg_reclaim_end") -int BPF_PROG(track_reclaim_end, unsigned long reclaimed, +int BPF_PROG(profile_reclaim_end, unsigned long reclaimed, struct mem_cgroup *memcg) { + struct active_reclaim *active; struct reclaim_profile *profile; + __u64 pid_tgid = bpf_get_current_pid_tgid(); __u64 cgroup_id = memcg_cgroup_id(memcg); + __u64 duration_ns; + __s32 stack_id; if (!selected_cgroup(cgroup_id)) return 0; - profile = get_profile(cgroup_id); - if (!profile) + active = bpf_map_lookup_elem(&active_reclaims, &pid_tgid); + if (!active || active->cgroup_id != cgroup_id) return 0; - __sync_fetch_and_add(&profile->end_count, 1); - __sync_fetch_and_add(&profile->reclaimed_pages, reclaimed); - profile->last_reclaim_ns = bpf_ktime_get_ns(); + duration_ns = bpf_ktime_get_ns() - active->started_ns; + stack_id = active->stack_id; + profile = get_profile(cgroup_id); + if (profile) { + __sync_fetch_and_add(&profile->end_count, 1); + __sync_fetch_and_add(&profile->reclaimed_pages, reclaimed); + __sync_fetch_and_add(&profile->total_reclaim_ns, duration_ns); + __sync_fetch_and_add(&profile->latency_slots[latency_bucket(duration_ns)], + 1); + update_maximum(&profile->maximum_reclaim_ns, duration_ns); + profile->last_reclaim_ns = bpf_ktime_get_ns(); + } + if (stack_id >= 0) + update_stack_profile(cgroup_id, stack_id, duration_ns, reclaimed); + bpf_map_delete_elem(&active_reclaims, &pid_tgid); return 0; } @@ -166,13 +272,8 @@ int capture_oom_victim(struct trace_event_raw_mark_victim *ctx) (void *)ctx + (ctx->__data_loc_comm & 0xffff)); profile = bpf_map_lookup_elem(&profiles, &cgroup_id); - if (profile) { - event->reclaim_begin_count = profile->begin_count; - event->reclaim_end_count = profile->end_count; - event->reclaimed_pages = profile->reclaimed_pages; - event->cross_cgroup_reclaims = profile->cross_cgroup_reclaims; - event->last_reclaim_ns = profile->last_reclaim_ns; - } + if (profile) + __builtin_memcpy(&event->profile, profile, sizeof(event->profile)); bpf_ringbuf_submit(event, 0); return 0; } diff --git a/src/57-oom-watch/oom_watch.c b/src/57-oom-watch/oom_watch.c index b5eb512..a33d776 100644 --- a/src/57-oom-watch/oom_watch.c +++ b/src/57-oom-watch/oom_watch.c @@ -6,6 +6,7 @@ #include #include #include +#include #include #include #include @@ -15,6 +16,7 @@ #include #include #include +#include #include #include "oom_watch.h" #include "oom_watch.skel.h" @@ -22,6 +24,7 @@ struct options { const char *cgroup_path; unsigned int duration_seconds; + unsigned int sample_every; bool demo; }; @@ -33,6 +36,51 @@ static unsigned int observed_victim_pid; static unsigned int observed_victim_tid; static unsigned long long observed_reclaims; static unsigned long long observed_cross_cgroup_reclaims; +static unsigned long long observed_stack_samples; + +struct kernel_symbol { + unsigned long long address; + char *name; +}; + +struct kernel_symbols { + struct kernel_symbol *items; + size_t count; + size_t capacity; +}; + +struct runtime_context { + int profiles_fd; + int stack_profiles_fd; + int stack_traces_fd; + struct kernel_symbols symbols; +}; + +struct oom_runtime { + struct oom_watch_bpf *skel; + struct ring_buffer *ring; + struct runtime_context context; +}; + +struct selected_cgroup { + char demo_path[256]; + const char *path; + struct stat metadata; + bool demo_created; + bool memory_enabled_by_demo; +}; + +struct demo_process { + pid_t child; + int ready_pipe[2]; + int continue_pipe[2]; + int status; +}; + +struct ranked_stack { + struct reclaim_stack_key key; + struct reclaim_stack_profile profile; +}; struct allocation_context { int ready_fd; @@ -88,20 +136,203 @@ static int memory_controller_enabled(bool *enabled) return 0; } +static int compare_symbols(const void *left, const void *right) +{ + const struct kernel_symbol *a = left; + const struct kernel_symbol *b = right; + + return a->address < b->address ? -1 : a->address > b->address ? 1 : 0; +} + +static int load_kernel_symbols(struct kernel_symbols *symbols) +{ + char name[256]; + char type; + unsigned long long address; + FILE *file = fopen("/proc/kallsyms", "r"); + + if (!file) + return -1; + while (fscanf(file, "%llx %c %255s%*[^\n]\n", &address, &type, + name) == 3) { + struct kernel_symbol *item; + + (void)type; + if (symbols->count == symbols->capacity) { + size_t capacity = symbols->capacity ? symbols->capacity * 2 : 4096; + void *items = realloc(symbols->items, + capacity * sizeof(*symbols->items)); + + if (!items) + goto error; + symbols->items = items; + symbols->capacity = capacity; + } + item = &symbols->items[symbols->count++]; + item->address = address; + item->name = strdup(name); + if (!item->name) + goto error; + } + fclose(file); + qsort(symbols->items, symbols->count, sizeof(*symbols->items), + compare_symbols); + return symbols->count ? 0 : -1; + +error: + fclose(file); + return -1; +} + +static void free_kernel_symbols(struct kernel_symbols *symbols) +{ + for (size_t i = 0; i < symbols->count; i++) + free(symbols->items[i].name); + free(symbols->items); +} + +static const struct kernel_symbol *find_kernel_symbol( + const struct kernel_symbols *symbols, unsigned long long address) +{ + size_t low = 0, high = symbols->count; + + while (low < high) { + size_t middle = low + (high - low) / 2; + + if (symbols->items[middle].address <= address) + low = middle + 1; + else + high = middle; + } + return low ? &symbols->items[low - 1] : NULL; +} + +static void insert_ranked_stack(struct ranked_stack top[5], size_t *count, + const struct reclaim_stack_key *key, + const struct reclaim_stack_profile *profile) +{ + size_t position = 0; + + while (position < *count && + top[position].profile.total_ns >= profile->total_ns) + position++; + if (position >= 5) + return; + if (*count < 5) + (*count)++; + for (size_t i = *count - 1; i > position; i--) + top[i] = top[i - 1]; + top[position].key = *key; + top[position].profile = *profile; +} + +static void print_reclaim_stacks(struct runtime_context *runtime, + __u64 cgroup_id) +{ + struct ranked_stack top[5] = {}; + struct reclaim_stack_key previous, next; + bool have_previous = false; + size_t count = 0; + + while (!bpf_map_get_next_key(runtime->stack_profiles_fd, + have_previous ? &previous : NULL, &next)) { + struct reclaim_stack_profile profile; + + if (next.cgroup_id == cgroup_id && + !bpf_map_lookup_elem(runtime->stack_profiles_fd, &next, + &profile)) + insert_ranked_stack(top, &count, &next, &profile); + previous = next; + have_previous = true; + } + + for (size_t rank = 0; rank < count; rank++) { + unsigned long long addresses[OOM_STACK_DEPTH] = {}; + + printf("reclaim_stack rank=%zu samples=%llu total_ms=%.3f " + "max_ms=%.3f reclaimed_pages=%llu\n", + rank + 1, top[rank].profile.samples, + top[rank].profile.total_ns / 1000000.0, + top[rank].profile.maximum_ns / 1000000.0, + top[rank].profile.reclaimed_pages); + if (bpf_map_lookup_elem(runtime->stack_traces_fd, + &top[rank].key.stack_id, addresses)) + continue; + for (size_t frame = 0; frame < OOM_STACK_DEPTH && addresses[frame]; + frame++) { + const struct kernel_symbol *symbol = + find_kernel_symbol(&runtime->symbols, addresses[frame]); + + if (symbol && symbol->address) + printf(" #%zu %s+0x%llx\n", frame, symbol->name, + addresses[frame] - symbol->address); + else + printf(" #%zu 0x%llx\n", frame, addresses[frame]); + } + } +} + +static void print_reclaim_profile(struct runtime_context *runtime, + __u64 cgroup_id, + const struct reclaim_profile *profile) +{ + printf("reclaim_profile cgroup_id=%llu cycles=%llu completed=%llu " + "total_ms=%.3f max_ms=%.3f reclaimed_pages=%llu " + "cross_cgroup=%llu stack_samples=%llu stack_failures=%llu\n", + (unsigned long long)cgroup_id, profile->begin_count, + profile->end_count, profile->total_reclaim_ns / 1000000.0, + profile->maximum_reclaim_ns / 1000000.0, + profile->reclaimed_pages, profile->cross_cgroup_reclaims, + profile->stack_samples, profile->stack_failures); + for (unsigned int bucket = 0; bucket < OOM_RECLAIM_BUCKETS; bucket++) { + unsigned long long low, high; + + if (!profile->latency_slots[bucket]) + continue; + low = bucket ? 1ULL << bucket : 0; + high = (1ULL << (bucket + 1)) - 1; + if (bucket == OOM_RECLAIM_BUCKETS - 1) + printf("reclaim_latency_us=>=%llu count=%llu\n", low, + profile->latency_slots[bucket]); + else + printf("reclaim_latency_us=%llu-%llu count=%llu\n", low, + high, profile->latency_slots[bucket]); + } + print_reclaim_stacks(runtime, cgroup_id); +} + +static void print_live_profiles(struct runtime_context *runtime) +{ + __u64 previous, next; + bool have_previous = false; + + while (!bpf_map_get_next_key(runtime->profiles_fd, + have_previous ? &previous : NULL, &next)) { + struct reclaim_profile profile; + + if (!bpf_map_lookup_elem(runtime->profiles_fd, &next, &profile)) + print_reclaim_profile(runtime, next, &profile); + previous = next; + have_previous = true; + } +} + static int handle_event(void *ctx, void *data, size_t size) { const struct oom_watch_event *event = data; + struct runtime_context *runtime = ctx; - (void)ctx; if (size != sizeof(*event)) return 0; if (event->type == OOM_VICTIM_MARKED) { victim_events++; observed_cgroup_id = event->cgroup_id; observed_victim_pid = event->victim_pid; - observed_reclaims = event->reclaim_begin_count; + observed_reclaims = event->profile.begin_count; observed_victim_tid = event->victim_tid; - observed_cross_cgroup_reclaims = event->cross_cgroup_reclaims; + observed_cross_cgroup_reclaims = + event->profile.cross_cgroup_reclaims; + observed_stack_samples = event->profile.stack_samples; printf("event=oom-victim pid=%u tid=%u comm=%s trigger_pid=%u cgroup_id=%llu " "anon_rss_kb=%llu file_rss_kb=%llu total_vm_kb=%llu " "reclaim_cycles=%llu cross_cgroup_reclaims=%llu " @@ -112,9 +343,10 @@ static int handle_event(void *ctx, void *data, size_t size) (unsigned long long)event->anon_rss_kb, (unsigned long long)event->file_rss_kb, (unsigned long long)event->total_vm_kb, - (unsigned long long)event->reclaim_begin_count, - (unsigned long long)event->cross_cgroup_reclaims, - (unsigned long long)event->reclaimed_pages); + (unsigned long long)event->profile.begin_count, + (unsigned long long)event->profile.cross_cgroup_reclaims, + (unsigned long long)event->profile.reclaimed_pages); + print_reclaim_profile(runtime, event->cgroup_id, &event->profile); } else if (event->type == OOM_VICTIM_EXITED) { exit_events++; printf("event=victim-exit pid=%u tid=%u cgroup_id=%llu exit_code=%d\n", @@ -124,14 +356,15 @@ static int handle_event(void *ctx, void *data, size_t size) return 0; } -static int parse_uint(const char *text, unsigned int *value) +static int parse_uint(const char *text, unsigned int maximum, + unsigned int *value) { char *end = NULL; unsigned long parsed; errno = 0; parsed = strtoul(text, &end, 10); - if (errno || !*text || *end || parsed > 86400) + if (errno || !*text || *end || !parsed || parsed > maximum) return -1; *value = parsed; return 0; @@ -139,8 +372,8 @@ static int parse_uint(const char *text, unsigned int *value) static void usage(const char *program) { - printf("Usage: %s [--cgroup PATH] [--duration SEC]\n" - " %s --demo\n", program, program); + printf("Usage: %s [--cgroup PATH] [--duration SEC] [--sample-every N]\n" + " %s --demo [--sample-every N]\n", program, program); } static int parse_options(int argc, char **argv, struct options *options) @@ -148,18 +381,23 @@ static int parse_options(int argc, char **argv, struct options *options) static const struct option long_options[] = { { "cgroup", required_argument, NULL, 'c' }, { "duration", required_argument, NULL, 'd' }, + { "sample-every", required_argument, NULL, 's' }, { "demo", no_argument, NULL, 'D' }, { "help", no_argument, NULL, 'h' }, {}, }; int option; - while ((option = getopt_long(argc, argv, "c:d:Dh", long_options, + while ((option = getopt_long(argc, argv, "c:d:s:Dh", long_options, NULL)) != -1) { switch (option) { case 'c': options->cgroup_path = optarg; break; case 'd': - if (parse_uint(optarg, &options->duration_seconds)) + if (parse_uint(optarg, 86400, &options->duration_seconds)) + return -1; + break; + case 's': + if (parse_uint(optarg, 1000000, &options->sample_every)) return -1; break; case 'D': options->demo = true; break; @@ -249,158 +487,253 @@ static int trigger_cross_cgroup_reclaim(const char *cgroup_path) return -1; } +static int select_cgroup(const struct options *options, + struct selected_cgroup *selected) +{ + if (options->demo) { + snprintf(selected->demo_path, sizeof(selected->demo_path), + "/sys/fs/cgroup/ebpf-oom-watch-%d", getpid()); + if (configure_demo_cgroup(selected->demo_path, + &selected->demo_created, + &selected->memory_enabled_by_demo)) { + fprintf(stderr, "failed to configure demo memory cgroup: %s\n", + strerror(errno)); + return -1; + } + selected->path = selected->demo_path; + } else { + selected->path = options->cgroup_path; + } + if (!selected->path) + return 0; + if (!stat(selected->path, &selected->metadata)) + return 0; + fprintf(stderr, "failed to stat cgroup %s: %s\n", selected->path, + strerror(errno)); + return -1; +} + +static void cleanup_selected_cgroup(struct selected_cgroup *selected, + int *result) +{ + if (selected->demo_created && rmdir(selected->demo_path) && !*result) + *result = 1; + if (selected->memory_enabled_by_demo && + write_text("/sys/fs/cgroup/cgroup.subtree_control", "-memory") && + !*result) + *result = 1; +} + +static int prepare_runtime(struct oom_runtime *runtime, + const struct options *options, + const struct selected_cgroup *selected) +{ + runtime->skel = oom_watch_bpf__open(); + if (!runtime->skel) + return -1; + runtime->skel->rodata->target_cgroup_id = + selected->path ? selected->metadata.st_ino : 0; + runtime->skel->rodata->sample_every = options->sample_every; + if (oom_watch_bpf__load(runtime->skel) || + oom_watch_bpf__attach(runtime->skel)) { + fprintf(stderr, "failed to load and attach OOM watcher\n"); + return -1; + } + runtime->context.profiles_fd = + bpf_map__fd(runtime->skel->maps.profiles); + runtime->context.stack_profiles_fd = + bpf_map__fd(runtime->skel->maps.stack_profiles); + runtime->context.stack_traces_fd = + bpf_map__fd(runtime->skel->maps.stack_traces); + if (load_kernel_symbols(&runtime->context.symbols)) + fprintf(stderr, "warning: kernel symbols unavailable; printing raw stack addresses\n"); + runtime->ring = ring_buffer__new( + bpf_map__fd(runtime->skel->maps.events), handle_event, + &runtime->context, NULL); + return runtime->ring ? 0 : -1; +} + +static void destroy_runtime(struct oom_runtime *runtime) +{ + ring_buffer__free(runtime->ring); + free_kernel_symbols(&runtime->context.symbols); + oom_watch_bpf__destroy(runtime->skel); +} + +static void init_demo_process(struct demo_process *demo) +{ + memset(demo, 0, sizeof(*demo)); + demo->child = -1; + demo->ready_pipe[0] = -1; + demo->ready_pipe[1] = -1; + demo->continue_pipe[0] = -1; + demo->continue_pipe[1] = -1; +} + +static void close_demo_pipe(int *fd) +{ + if (*fd >= 0) + close(*fd); + *fd = -1; +} + +static void cleanup_demo_process(struct demo_process *demo) +{ + if (demo->child > 0) { + kill(demo->child, SIGKILL); + waitpid(demo->child, NULL, 0); + } + close_demo_pipe(&demo->ready_pipe[0]); + close_demo_pipe(&demo->ready_pipe[1]); + close_demo_pipe(&demo->continue_pipe[0]); + close_demo_pipe(&demo->continue_pipe[1]); +} + +static int start_demo_process(struct demo_process *demo, + const char *cgroup_path) +{ + struct pollfd ready = { .events = POLLIN }; + struct timespec leader_exit_delay = { .tv_nsec = 100000000 }; + char byte = 'x'; + + if (pipe(demo->ready_pipe) || pipe(demo->continue_pipe)) + return -1; + demo->child = fork(); + if (demo->child < 0) + return -1; + if (!demo->child) { + close(demo->ready_pipe[0]); + close(demo->continue_pipe[1]); + allocate_until_killed(cgroup_path, demo->ready_pipe[1], + demo->continue_pipe[0]); + } + close_demo_pipe(&demo->ready_pipe[1]); + close_demo_pipe(&demo->continue_pipe[0]); + ready.fd = demo->ready_pipe[0]; + if (poll(&ready, 1, 5000) != 1 || + read(demo->ready_pipe[0], &byte, 1) != 1 || + trigger_cross_cgroup_reclaim(cgroup_path)) + return -1; + nanosleep(&leader_exit_delay, NULL); + if (write(demo->continue_pipe[1], &byte, 1) != 1) + return -1; + close_demo_pipe(&demo->ready_pipe[0]); + close_demo_pipe(&demo->continue_pipe[1]); + return 0; +} + +static int collect_demo_events(struct demo_process *demo, + struct ring_buffer *ring) +{ + for (int i = 0; i < 200; i++) { + pid_t waited; + + ring_buffer__poll(ring, 50); + waited = waitpid(demo->child, &demo->status, WNOHANG); + if (waited == demo->child) { + demo->child = -1; + break; + } + } + for (int i = 0; i < 10 && exit_events < 1; i++) + ring_buffer__poll(ring, 50); + return demo->child < 0 ? 0 : -1; +} + +static bool valid_demo_observation(const struct demo_process *demo, + unsigned long long cgroup_id, + const struct oom_watch_bpf *skel) +{ + return WIFSIGNALED(demo->status) && + WTERMSIG(demo->status) == SIGKILL && victim_events == 1 && + exit_events == 1 && observed_cgroup_id == cgroup_id && + observed_victim_pid && observed_victim_tid && + observed_victim_pid != observed_victim_tid && observed_reclaims && + observed_cross_cgroup_reclaims && observed_stack_samples && + !skel->bss->dropped_victim_states && + !skel->bss->dropped_reclaim_states; +} + +static int run_demo(struct oom_runtime *runtime, + const struct selected_cgroup *selected) +{ + struct demo_process demo; + int result = -1; + + init_demo_process(&demo); + if (start_demo_process(&demo, selected->path) || + collect_demo_events(&demo, runtime->ring)) + goto cleanup; + printf("demo workload signaled=%d signal=%d\n", + WIFSIGNALED(demo.status), + WIFSIGNALED(demo.status) ? WTERMSIG(demo.status) : 0); + if (!valid_demo_observation(&demo, selected->metadata.st_ino, + runtime->skel)) + goto cleanup; + printf("demo result=matched-profile-to-victim\n"); + result = 0; + +cleanup: + cleanup_demo_process(&demo); + return result; +} + +static int watch_profiles(struct oom_runtime *runtime, + unsigned int duration_seconds) +{ + unsigned long long deadline = 0; + + signal(SIGINT, handle_signal); + signal(SIGTERM, handle_signal); + if (duration_seconds) + deadline = monotonic_ns() + + (unsigned long long)duration_seconds * 1000000000ULL; + while (!stop && (!deadline || monotonic_ns() < deadline)) { + int result = ring_buffer__poll(runtime->ring, 100); + + if (result < 0 && result != -EINTR) { + fprintf(stderr, "ring buffer poll failed: %d\n", result); + return -1; + } + } + print_live_profiles(&runtime->context); + return 0; +} + int main(int argc, char **argv) { - struct options options = {}; - struct oom_watch_bpf *skel = NULL; - struct ring_buffer *ring = NULL; - char demo_cgroup_path[256]; - unsigned long long deadline = 0; - struct stat cgroup_stat = {}; - const char *selected_path = NULL; - pid_t child = -1; - int ready_pipe[2] = { -1, -1 }; - int continue_pipe[2] = { -1, -1 }; - int status = 0; + struct options options = { .sample_every = 1 }; + struct selected_cgroup selected = {}; + struct oom_runtime runtime = {}; int err = 1; - bool demo_cgroup_created = false; - bool memory_enabled_by_demo = false; setvbuf(stdout, NULL, _IONBF, 0); if (parse_options(argc, argv, &options)) { usage(argv[0]); return 2; } - if (options.demo) { - snprintf(demo_cgroup_path, sizeof(demo_cgroup_path), - "/sys/fs/cgroup/ebpf-oom-watch-%d", getpid()); - if (configure_demo_cgroup(demo_cgroup_path, - &demo_cgroup_created, - &memory_enabled_by_demo)) { - fprintf(stderr, "failed to configure demo memory cgroup: %s\n", - strerror(errno)); - goto cleanup; - } - selected_path = demo_cgroup_path; - } else { - selected_path = options.cgroup_path; - } - if (selected_path && stat(selected_path, &cgroup_stat)) { - fprintf(stderr, "failed to stat cgroup %s: %s\n", selected_path, - strerror(errno)); - goto cleanup; - } - - skel = oom_watch_bpf__open(); - if (!skel) - goto cleanup; - skel->rodata->target_cgroup_id = selected_path ? cgroup_stat.st_ino : 0; - if (oom_watch_bpf__load(skel) || oom_watch_bpf__attach(skel)) { - fprintf(stderr, "failed to load and attach OOM watcher\n"); - goto cleanup; - } - ring = ring_buffer__new(bpf_map__fd(skel->maps.events), handle_event, - NULL, NULL); - if (!ring) + if (select_cgroup(&options, &selected) || + prepare_runtime(&runtime, &options, &selected)) goto cleanup; - if (selected_path) + if (selected.path) printf("oom-watch tracing cgroup=%s cgroup_id=%llu\n", - selected_path, (unsigned long long)cgroup_stat.st_ino); + selected.path, + (unsigned long long)selected.metadata.st_ino); else printf("oom-watch tracing all cgroups\n"); - if (options.demo) { - struct pollfd ready = { .events = POLLIN }; - struct timespec leader_exit_delay = { .tv_nsec = 100000000 }; - char byte = 'x'; - - if (pipe(ready_pipe) || pipe(continue_pipe)) - goto cleanup; - child = fork(); - if (child < 0) - goto cleanup; - if (!child) { - close(ready_pipe[0]); - close(continue_pipe[1]); - allocate_until_killed(selected_path, ready_pipe[1], - continue_pipe[0]); - } - close(ready_pipe[1]); ready_pipe[1] = -1; - close(continue_pipe[0]); continue_pipe[0] = -1; - ready.fd = ready_pipe[0]; - if (poll(&ready, 1, 5000) != 1 || - read(ready_pipe[0], &byte, 1) != 1 || - trigger_cross_cgroup_reclaim(selected_path)) - goto cleanup; - nanosleep(&leader_exit_delay, NULL); - if (write(continue_pipe[1], &byte, 1) != 1) - goto cleanup; - close(ready_pipe[0]); ready_pipe[0] = -1; - close(continue_pipe[1]); continue_pipe[1] = -1; - - for (int i = 0; i < 200; i++) { - pid_t waited; - - ring_buffer__poll(ring, 50); - waited = waitpid(child, &status, WNOHANG); - if (waited == child) { - child = -1; - break; - } - } - for (int i = 0; i < 10 && exit_events < 1; i++) - ring_buffer__poll(ring, 50); - - printf("demo workload signaled=%d signal=%d\n", - WIFSIGNALED(status), - WIFSIGNALED(status) ? WTERMSIG(status) : 0); - if (!WIFSIGNALED(status) || WTERMSIG(status) != SIGKILL || - victim_events != 1 || exit_events != 1 || - observed_cgroup_id != (unsigned long long)cgroup_stat.st_ino || - !observed_victim_pid || !observed_victim_tid || - observed_victim_pid == observed_victim_tid || - !observed_reclaims || !observed_cross_cgroup_reclaims || - skel->bss->dropped_victim_states) - goto cleanup; - printf("demo result=matched-profile-to-victim\n"); - } else { - signal(SIGINT, handle_signal); - signal(SIGTERM, handle_signal); - if (options.duration_seconds) - deadline = monotonic_ns() + - (unsigned long long)options.duration_seconds * - 1000000000ULL; - while (!stop && (!deadline || monotonic_ns() < deadline)) { - int poll_result = ring_buffer__poll(ring, 100); - - if (poll_result < 0 && poll_result != -EINTR) { - fprintf(stderr, "ring buffer poll failed: %d\n", - poll_result); - goto cleanup; - } - } - } - printf("dropped_victim_states=%llu\n", - (unsigned long long)skel->bss->dropped_victim_states); + if ((options.demo && run_demo(&runtime, &selected)) || + (!options.demo && watch_profiles(&runtime, + options.duration_seconds))) + goto cleanup; + printf("dropped_victim_states=%llu dropped_reclaim_states=%llu\n", + (unsigned long long)runtime.skel->bss->dropped_victim_states, + (unsigned long long)runtime.skel->bss->dropped_reclaim_states); err = 0; cleanup: - if (child > 0) { - kill(child, SIGKILL); - waitpid(child, NULL, 0); - } - for (size_t i = 0; i < 2; i++) { - if (ready_pipe[i] >= 0) close(ready_pipe[i]); - if (continue_pipe[i] >= 0) close(continue_pipe[i]); - } - ring_buffer__free(ring); - oom_watch_bpf__destroy(skel); - if (demo_cgroup_created && rmdir(demo_cgroup_path) && !err) - err = 1; - if (memory_enabled_by_demo && - write_text("/sys/fs/cgroup/cgroup.subtree_control", "-memory") && - !err) - err = 1; + destroy_runtime(&runtime); + cleanup_selected_cgroup(&selected, &err); return err; } diff --git a/src/57-oom-watch/oom_watch.h b/src/57-oom-watch/oom_watch.h index 2288ed0..f2a71f9 100644 --- a/src/57-oom-watch/oom_watch.h +++ b/src/57-oom-watch/oom_watch.h @@ -2,19 +2,44 @@ #ifndef __OOM_WATCH_H #define __OOM_WATCH_H +#define OOM_RECLAIM_BUCKETS 20 +#define OOM_STACK_DEPTH 127 + enum oom_watch_event_type { OOM_VICTIM_MARKED = 1, OOM_VICTIM_EXITED = 2, }; -struct oom_watch_event { - unsigned long long timestamp_ns; - unsigned long long cgroup_id; - unsigned long long reclaim_begin_count; - unsigned long long reclaim_end_count; +struct reclaim_profile { + unsigned long long begin_count; + unsigned long long end_count; unsigned long long reclaimed_pages; unsigned long long cross_cgroup_reclaims; unsigned long long last_reclaim_ns; + unsigned long long total_reclaim_ns; + unsigned long long maximum_reclaim_ns; + unsigned long long stack_samples; + unsigned long long stack_failures; + unsigned long long latency_slots[OOM_RECLAIM_BUCKETS]; +}; + +struct reclaim_stack_key { + unsigned long long cgroup_id; + signed int stack_id; + unsigned int padding; +}; + +struct reclaim_stack_profile { + unsigned long long samples; + unsigned long long total_ns; + unsigned long long maximum_ns; + unsigned long long reclaimed_pages; +}; + +struct oom_watch_event { + unsigned long long timestamp_ns; + unsigned long long cgroup_id; + struct reclaim_profile profile; unsigned long long total_vm_kb; unsigned long long anon_rss_kb; unsigned long long file_rss_kb; diff --git a/src/57-oom-watch/tests/test_oom_watch.py b/src/57-oom-watch/tests/test_oom_watch.py index 07d134f..3f19873 100644 --- a/src/57-oom-watch/tests/test_oom_watch.py +++ b/src/57-oom-watch/tests/test_oom_watch.py @@ -1,7 +1,7 @@ #!/usr/bin/env python3 +import re import subprocess import sys -import re from pathlib import Path @@ -25,10 +25,14 @@ def main(): assert match and match.group(1) != match.group(2), output assert "reclaim_cycles=" in output, output assert "cross_cgroup_reclaims=" in output, output + assert "reclaim_profile cgroup_id=" in output, output + assert "reclaim_latency_us=" in output, output + assert "reclaim_stack rank=1" in output, output + assert "try_to_free_mem_cgroup_pages" in output, output assert "event=victim-exit" in output, output assert "demo workload signaled=1 signal=9" in output, output assert "demo result=matched-profile-to-victim" in output, output - assert "dropped_victim_states=0" in output, output + assert "dropped_victim_states=0 dropped_reclaim_states=0" in output, output print("OOM watch integration test: PASS") diff --git a/src/60-afxdp-dump/.config b/src/60-afxdp-dump/.config new file mode 100644 index 0000000..ca6da83 --- /dev/null +++ b/src/60-afxdp-dump/.config @@ -0,0 +1,10 @@ +level=Depth +type=Networking +kernel_min=5.3 +kernel_min_basis=required-feature +architectures=x86_64 +btf=required +kernel_config=CONFIG_BPF=y,CONFIG_BPF_SYSCALL=y,CONFIG_BPF_JIT=y,CONFIG_DEBUG_INFO_BTF=y,CONFIG_XDP_SOCKETS=y +hardware=network-interface +root=required +test_status=ci-build diff --git a/src/60-afxdp-dump/afxdp_dump.c b/src/60-afxdp-dump/afxdp_dump.c index 5176d9f..f4b1427 100644 --- a/src/60-afxdp-dump/afxdp_dump.c +++ b/src/60-afxdp-dump/afxdp_dump.c @@ -199,43 +199,73 @@ static int recycle_frame(struct xsk_state *xsk, unsigned long long address) return 0; } +static const struct iphdr *parse_ipv4(const void *packet, unsigned int length, + unsigned int *ip_header_length, + unsigned int *ip_length) +{ + const struct ethhdr *ethernet = packet; + const struct iphdr *ip; + unsigned int available_ip; + + if (length < sizeof(*ethernet) + sizeof(*ip) || + ethernet->h_proto != htons(ETH_P_IP)) + return NULL; + ip = packet + sizeof(*ethernet); + *ip_header_length = ip->ihl * 4; + if (ip->version != 4 || ip->protocol != IPPROTO_UDP || + (ntohs(ip->frag_off) & + (IPV4_MORE_FRAGMENTS | IPV4_FRAGMENT_OFFSET)) || + *ip_header_length < sizeof(*ip) || + length < sizeof(*ethernet) + *ip_header_length) + return NULL; + *ip_length = ntohs(ip->tot_len); + available_ip = length - sizeof(*ethernet); + if (*ip_length < *ip_header_length + sizeof(struct udphdr) || + *ip_length > available_ip) + return NULL; + return ip; +} + +static const struct udphdr *parse_udp(const void *packet, + unsigned int length, + unsigned int ip_header_length, + unsigned int ip_length, + const unsigned char **payload, + unsigned int *payload_length) +{ + const struct udphdr *udp = packet + sizeof(struct ethhdr) + + ip_header_length; + unsigned int available_payload; + unsigned int udp_length; + + *payload = (const unsigned char *)(udp + 1); + udp_length = ntohs(udp->len); + available_payload = length - (*payload - (const unsigned char *)packet); + if (udp_length < sizeof(*udp) || + udp_length > ip_length - ip_header_length || + udp_length - sizeof(*udp) > available_payload) + return NULL; + *payload_length = udp_length - sizeof(*udp); + return udp; +} + static void dump_packet(const void *packet, unsigned int length, unsigned int packet_number) { - const struct ethhdr *ethernet = packet; const struct iphdr *ip; const struct udphdr *udp; const unsigned char *payload; char source[INET_ADDRSTRLEN], destination[INET_ADDRSTRLEN]; char preview[PAYLOAD_PREVIEW + 1]; - unsigned int available_ip, available_payload, ip_header_length; - unsigned int ip_length, payload_length, preview_length, udp_length; + unsigned int ip_header_length, ip_length, payload_length, preview_length; - if (length < sizeof(*ethernet) + sizeof(*ip) || - ethernet->h_proto != htons(ETH_P_IP)) + ip = parse_ipv4(packet, length, &ip_header_length, &ip_length); + if (!ip) return; - ip = packet + sizeof(*ethernet); - ip_header_length = ip->ihl * 4; - if (ip->version != 4 || ip->protocol != IPPROTO_UDP || - (ntohs(ip->frag_off) & - (IPV4_MORE_FRAGMENTS | IPV4_FRAGMENT_OFFSET)) || - ip_header_length < sizeof(*ip) || - length < sizeof(*ethernet) + ip_header_length + sizeof(*udp)) + udp = parse_udp(packet, length, ip_header_length, ip_length, &payload, + &payload_length); + if (!udp) return; - ip_length = ntohs(ip->tot_len); - available_ip = length - sizeof(*ethernet); - if (ip_length < ip_header_length + sizeof(*udp) || - ip_length > available_ip) - return; - udp = packet + sizeof(*ethernet) + ip_header_length; - payload = (const unsigned char *)(udp + 1); - udp_length = ntohs(udp->len); - available_payload = length - (payload - (const unsigned char *)packet); - if (udp_length < sizeof(*udp) || - udp_length > ip_length - ip_header_length || - udp_length - sizeof(*udp) > available_payload) - return; - payload_length = udp_length - sizeof(*udp); preview_length = payload_length < PAYLOAD_PREVIEW ? payload_length : PAYLOAD_PREVIEW; for (unsigned int i = 0; i < preview_length; i++)