Deploying to gh-pages from @ eunomia-bpf/bpf-developer-tutorial@52ae3ae26d 🚀

9-runqlat/.gitignore

@@ -0,0 +1,7 @@
+.vscode
+package.json
+*.o
+*.skel.json
+*.skel.yaml
+package.yaml
+ecli

9-runqlat/bits.bpf.h

@@ -0,0 +1,31 @@
+/* SPDX-License-Identifier: (LGPL-2.1 OR BSD-2-Clause) */
+#ifndef __BITS_BPF_H
+#define __BITS_BPF_H
+
+#define READ_ONCE(x) (*(volatile typeof(x) *)&(x))
+#define WRITE_ONCE(x, val) ((*(volatile typeof(x) *)&(x)) = val)
+
+static __always_inline u64 log2(u32 v)
+{
+	u32 shift, r;
+
+	r = (v > 0xFFFF) << 4; v >>= r;
+	shift = (v > 0xFF) << 3; v >>= shift; r |= shift;
+	shift = (v > 0xF) << 2; v >>= shift; r |= shift;
+	shift = (v > 0x3) << 1; v >>= shift; r |= shift;
+	r |= (v >> 1);
+
+	return r;
+}
+
+static __always_inline u64 log2l(u64 v)
+{
+	u32 hi = v >> 32;
+
+	if (hi)
+		return log2(hi) + 32;
+	else
+		return log2(v);
+}
+
+#endif /* __BITS_BPF_H */

9-runqlat/core_fixes.bpf.h

@@ -0,0 +1,112 @@
+/* SPDX-License-Identifier: (LGPL-2.1 OR BSD-2-Clause) */
+/* Copyright (c) 2021 Hengqi Chen */
+
+#ifndef __CORE_FIXES_BPF_H
+#define __CORE_FIXES_BPF_H
+
+#include <vmlinux.h>
+#include <bpf/bpf_core_read.h>
+
+/**
+ * commit 2f064a59a1 ("sched: Change task_struct::state") changes
+ * the name of task_struct::state to task_struct::__state
+ * see:
+ *     https://github.com/torvalds/linux/commit/2f064a59a1
+ */
+struct task_struct___o {
+	volatile long int state;
+} __attribute__((preserve_access_index));
+
+struct task_struct___x {
+	unsigned int __state;
+} __attribute__((preserve_access_index));
+
+static __always_inline __s64 get_task_state(void *task)
+{
+	struct task_struct___x *t = task;
+
+	if (bpf_core_field_exists(t->__state))
+		return BPF_CORE_READ(t, __state);
+	return BPF_CORE_READ((struct task_struct___o *)task, state);
+}
+
+/**
+ * commit 309dca309fc3 ("block: store a block_device pointer in struct bio")
+ * adds a new member bi_bdev which is a pointer to struct block_device
+ * see:
+ *     https://github.com/torvalds/linux/commit/309dca309fc3
+ */
+struct bio___o {
+	struct gendisk *bi_disk;
+} __attribute__((preserve_access_index));
+
+struct bio___x {
+	struct block_device *bi_bdev;
+} __attribute__((preserve_access_index));
+
+static __always_inline struct gendisk *get_gendisk(void *bio)
+{
+	struct bio___x *b = bio;
+
+	if (bpf_core_field_exists(b->bi_bdev))
+		return BPF_CORE_READ(b, bi_bdev, bd_disk);
+	return BPF_CORE_READ((struct bio___o *)bio, bi_disk);
+}
+
+/**
+ * commit d5869fdc189f ("block: introduce block_rq_error tracepoint")
+ * adds a new tracepoint block_rq_error and it shares the same arguments
+ * with tracepoint block_rq_complete. As a result, the kernel BTF now has
+ * a `struct trace_event_raw_block_rq_completion` instead of
+ * `struct trace_event_raw_block_rq_complete`.
+ * see:
+ *     https://github.com/torvalds/linux/commit/d5869fdc189f
+ */
+struct trace_event_raw_block_rq_complete___x {
+	dev_t dev;
+	sector_t sector;
+	unsigned int nr_sector;
+} __attribute__((preserve_access_index));
+
+struct trace_event_raw_block_rq_completion___x {
+	dev_t dev;
+	sector_t sector;
+	unsigned int nr_sector;
+} __attribute__((preserve_access_index));
+
+static __always_inline bool has_block_rq_completion()
+{
+	if (bpf_core_type_exists(struct trace_event_raw_block_rq_completion___x))
+		return true;
+	return false;
+}
+
+/**
+ * commit d152c682f03c ("block: add an explicit ->disk backpointer to the
+ * request_queue") and commit f3fa33acca9f ("block: remove the ->rq_disk
+ * field in struct request") make some changes to `struct request` and
+ * `struct request_queue`. Now, to get the `struct gendisk *` field in a CO-RE
+ * way, we need both `struct request` and `struct request_queue`.
+ * see:
+ *     https://github.com/torvalds/linux/commit/d152c682f03c
+ *     https://github.com/torvalds/linux/commit/f3fa33acca9f
+ */
+struct request_queue___x {
+	struct gendisk *disk;
+} __attribute__((preserve_access_index));
+
+struct request___x {
+	struct request_queue___x *q;
+	struct gendisk *rq_disk;
+} __attribute__((preserve_access_index));
+
+static __always_inline struct gendisk *get_disk(void *request)
+{
+	struct request___x *r = request;
+
+	if (bpf_core_field_exists(r->rq_disk))
+		return BPF_CORE_READ(r, rq_disk);
+	return BPF_CORE_READ(r, q, disk);
+}
+
+#endif /* __CORE_FIXES_BPF_H */

9-runqlat/maps.bpf.h

@@ -0,0 +1,26 @@
+// SPDX-License-Identifier: (LGPL-2.1 OR BSD-2-Clause)
+// Copyright (c) 2020 Anton Protopopov
+#ifndef __MAPS_BPF_H
+#define __MAPS_BPF_H
+
+#include <bpf/bpf_helpers.h>
+#include <asm-generic/errno.h>
+
+static __always_inline void *
+bpf_map_lookup_or_try_init(void *map, const void *key, const void *init)
+{
+	void *val;
+	long err;
+
+	val = bpf_map_lookup_elem(map, key);
+	if (val)
+		return val;
+
+	err = bpf_map_update_elem(map, key, init, BPF_NOEXIST);
+	if (err && err != -EEXIST)
+		return 0;
+
+	return bpf_map_lookup_elem(map, key);
+}
+
+#endif /* __MAPS_BPF_H */

9-runqlat/runqlat.bpf.c

@@ -0,0 +1,152 @@
+// SPDX-License-Identifier: GPL-2.0
+// Copyright (c) 2020 Wenbo Zhang
+#include <vmlinux.h>
+#include <bpf/bpf_helpers.h>
+#include <bpf/bpf_core_read.h>
+#include <bpf/bpf_tracing.h>
+#include "runqlat.h"
+#include "bits.bpf.h"
+#include "maps.bpf.h"
+#include "core_fixes.bpf.h"
+
+#define MAX_ENTRIES	10240
+#define TASK_RUNNING 	0
+
+const volatile bool filter_cg = false;
+const volatile bool targ_per_process = false;
+const volatile bool targ_per_thread = false;
+const volatile bool targ_per_pidns = false;
+const volatile bool targ_ms = false;
+const volatile pid_t targ_tgid = 0;
+
+struct {
+	__uint(type, BPF_MAP_TYPE_CGROUP_ARRAY);
+	__type(key, u32);
+	__type(value, u32);
+	__uint(max_entries, 1);
+} cgroup_map SEC(".maps");
+
+struct {
+	__uint(type, BPF_MAP_TYPE_HASH);
+	__uint(max_entries, MAX_ENTRIES);
+	__type(key, u32);
+	__type(value, u64);
+} start SEC(".maps");
+
+static struct hist zero;
+
+/// @sample {"interval": 1000, "type" : "log2_hist"}
+struct {
+	__uint(type, BPF_MAP_TYPE_HASH);
+	__uint(max_entries, MAX_ENTRIES);
+	__type(key, u32);
+	__type(value, struct hist);
+} hists SEC(".maps");
+
+static int trace_enqueue(u32 tgid, u32 pid)
+{
+	u64 ts;
+
+	if (!pid)
+		return 0;
+	if (targ_tgid && targ_tgid != tgid)
+		return 0;
+
+	ts = bpf_ktime_get_ns();
+	bpf_map_update_elem(&start, &pid, &ts, BPF_ANY);
+	return 0;
+}
+
+static unsigned int pid_namespace(struct task_struct *task)
+{
+	struct pid *pid;
+	unsigned int level;
+	struct upid upid;
+	unsigned int inum;
+
+	/*  get the pid namespace by following task_active_pid_ns(),
+	 *  pid->numbers[pid->level].ns
+	 */
+	pid = BPF_CORE_READ(task, thread_pid);
+	level = BPF_CORE_READ(pid, level);
+	bpf_core_read(&upid, sizeof(upid), &pid->numbers[level]);
+	inum = BPF_CORE_READ(upid.ns, ns.inum);
+
+	return inum;
+}
+
+static int handle_switch(bool preempt, struct task_struct *prev, struct task_struct *next)
+{
+	struct hist *histp;
+	u64 *tsp, slot;
+	u32 pid, hkey;
+	s64 delta;
+
+	if (filter_cg && !bpf_current_task_under_cgroup(&cgroup_map, 0))
+		return 0;
+
+	if (get_task_state(prev) == TASK_RUNNING)
+		trace_enqueue(BPF_CORE_READ(prev, tgid), BPF_CORE_READ(prev, pid));
+
+	pid = BPF_CORE_READ(next, pid);
+
+	tsp = bpf_map_lookup_elem(&start, &pid);
+	if (!tsp)
+		return 0;
+	delta = bpf_ktime_get_ns() - *tsp;
+	if (delta < 0)
+		goto cleanup;
+
+	if (targ_per_process)
+		hkey = BPF_CORE_READ(next, tgid);
+	else if (targ_per_thread)
+		hkey = pid;
+	else if (targ_per_pidns)
+		hkey = pid_namespace(next);
+	else
+		hkey = -1;
+	histp = bpf_map_lookup_or_try_init(&hists, &hkey, &zero);
+	if (!histp)
+		goto cleanup;
+	if (!histp->comm[0])
+		bpf_probe_read_kernel_str(&histp->comm, sizeof(histp->comm),
+					next->comm);
+	if (targ_ms)
+		delta /= 1000000U;
+	else
+		delta /= 1000U;
+	slot = log2l(delta);
+	if (slot >= MAX_SLOTS)
+		slot = MAX_SLOTS - 1;
+	__sync_fetch_and_add(&histp->slots[slot], 1);
+
+cleanup:
+	bpf_map_delete_elem(&start, &pid);
+	return 0;
+}
+
+SEC("raw_tp/sched_wakeup")
+int BPF_PROG(handle_sched_wakeup, struct task_struct *p)
+{
+	if (filter_cg && !bpf_current_task_under_cgroup(&cgroup_map, 0))
+		return 0;
+
+	return trace_enqueue(BPF_CORE_READ(p, tgid), BPF_CORE_READ(p, pid));
+}
+
+SEC("raw_tp/sched_wakeup_new")
+int BPF_PROG(handle_sched_wakeup_new, struct task_struct *p)
+{
+	if (filter_cg && !bpf_current_task_under_cgroup(&cgroup_map, 0))
+		return 0;
+
+	return trace_enqueue(BPF_CORE_READ(p, tgid), BPF_CORE_READ(p, pid));
+}
+
+SEC("raw_tp/sched_switch")
+int BPF_PROG(handle_sched_switch, bool preempt, struct task_struct *prev, struct task_struct *next)
+{
+	return handle_switch(preempt, prev, next);
+}
+
+char LICENSE[] SEC("license") = "GPL";

9-runqlat/runqlat.h

@@ -0,0 +1,14 @@
+
+/* SPDX-License-Identifier: (LGPL-2.1 OR BSD-2-Clause) */
+#ifndef __RUNQLAT_H
+#define __RUNQLAT_H
+
+#define TASK_COMM_LEN	16
+#define MAX_SLOTS	26
+
+struct hist {
+	__u32 slots[MAX_SLOTS];
+	char comm[TASK_COMM_LEN];
+};
+
+#endif /* __RUNQLAT_H */