mirror of
https://github.com/eunomia-bpf/bpf-developer-tutorial.git
synced 2026-02-04 02:34:16 +08:00
513 lines
38 KiB
HTML
513 lines
38 KiB
HTML
<!DOCTYPE HTML>
|
||
<html lang="en" class="sidebar-visible no-js light">
|
||
<head>
|
||
<!-- Book generated using mdBook -->
|
||
<meta charset="UTF-8">
|
||
<title>bcc Tutorial - bpf-developer-tutorial</title>
|
||
|
||
|
||
<!-- Custom HTML head -->
|
||
|
||
<meta name="description" content="">
|
||
<meta name="viewport" content="width=device-width, initial-scale=1">
|
||
<meta name="theme-color" content="#ffffff" />
|
||
|
||
<link rel="icon" href="../favicon.svg">
|
||
<link rel="shortcut icon" href="../favicon.png">
|
||
<link rel="stylesheet" href="../css/variables.css">
|
||
<link rel="stylesheet" href="../css/general.css">
|
||
<link rel="stylesheet" href="../css/chrome.css">
|
||
<link rel="stylesheet" href="../css/print.css" media="print">
|
||
|
||
<!-- Fonts -->
|
||
<link rel="stylesheet" href="../FontAwesome/css/font-awesome.css">
|
||
<link rel="stylesheet" href="../fonts/fonts.css">
|
||
|
||
<!-- Highlight.js Stylesheets -->
|
||
<link rel="stylesheet" href="../highlight.css">
|
||
<link rel="stylesheet" href="../tomorrow-night.css">
|
||
<link rel="stylesheet" href="../ayu-highlight.css">
|
||
|
||
<!-- Custom theme stylesheets -->
|
||
|
||
</head>
|
||
<body>
|
||
<div id="body-container">
|
||
<!-- Provide site root to javascript -->
|
||
<script>
|
||
var path_to_root = "../";
|
||
var default_theme = window.matchMedia("(prefers-color-scheme: dark)").matches ? "navy" : "light";
|
||
</script>
|
||
|
||
<!-- Work around some values being stored in localStorage wrapped in quotes -->
|
||
<script>
|
||
try {
|
||
var theme = localStorage.getItem('mdbook-theme');
|
||
var sidebar = localStorage.getItem('mdbook-sidebar');
|
||
|
||
if (theme.startsWith('"') && theme.endsWith('"')) {
|
||
localStorage.setItem('mdbook-theme', theme.slice(1, theme.length - 1));
|
||
}
|
||
|
||
if (sidebar.startsWith('"') && sidebar.endsWith('"')) {
|
||
localStorage.setItem('mdbook-sidebar', sidebar.slice(1, sidebar.length - 1));
|
||
}
|
||
} catch (e) { }
|
||
</script>
|
||
|
||
<!-- Set the theme before any content is loaded, prevents flash -->
|
||
<script>
|
||
var theme;
|
||
try { theme = localStorage.getItem('mdbook-theme'); } catch(e) { }
|
||
if (theme === null || theme === undefined) { theme = default_theme; }
|
||
var html = document.querySelector('html');
|
||
html.classList.remove('no-js')
|
||
html.classList.remove('light')
|
||
html.classList.add(theme);
|
||
html.classList.add('js');
|
||
</script>
|
||
|
||
<!-- Hide / unhide sidebar before it is displayed -->
|
||
<script>
|
||
var html = document.querySelector('html');
|
||
var sidebar = null;
|
||
if (document.body.clientWidth >= 1080) {
|
||
try { sidebar = localStorage.getItem('mdbook-sidebar'); } catch(e) { }
|
||
sidebar = sidebar || 'visible';
|
||
} else {
|
||
sidebar = 'hidden';
|
||
}
|
||
html.classList.remove('sidebar-visible');
|
||
html.classList.add("sidebar-" + sidebar);
|
||
</script>
|
||
|
||
<nav id="sidebar" class="sidebar" aria-label="Table of contents">
|
||
<div class="sidebar-scrollbox">
|
||
<ol class="chapter"><li class="chapter-item expanded affix "><li class="part-title">eBPF 实践教程：基于 libbpf 和 CO-RE</li><li class="chapter-item expanded "><a href="../0-introduce/index.html"><strong aria-hidden="true">1.</strong> eBPF 入门开发实践教程一：介绍 eBPF 的基本概念、常见的开发工具</a></li><li class="chapter-item expanded "><a href="../1-helloworld/index.html"><strong aria-hidden="true">2.</strong> eBPF 入门开发实践教程二：Hello World，基本框架和开发流程</a></li><li class="chapter-item expanded "><a href="../2-kprobe-unlink/index.html"><strong aria-hidden="true">3.</strong> eBPF 入门开发实践教程二：在 eBPF 中使用 kprobe 监测捕获 unlink 系统调用</a></li><li class="chapter-item expanded "><a href="../3-fentry-unlink/index.html"><strong aria-hidden="true">4.</strong> eBPF 入门开发实践教程三：在 eBPF 中使用 fentry 监测捕获 unlink 系统调用</a></li><li class="chapter-item expanded "><a href="../4-opensnoop/index.html"><strong aria-hidden="true">5.</strong> eBPF 入门开发实践教程四：在 eBPF 中捕获进程打开文件的系统调用集合，使用全局变量过滤进程 pid</a></li><li class="chapter-item expanded "><a href="../5-uprobe-bashreadline/index.html"><strong aria-hidden="true">6.</strong> eBPF 入门开发实践教程五：在 eBPF 中使用 uprobe 捕获 bash 的 readline 函数调用</a></li><li class="chapter-item expanded "><a href="../6-sigsnoop/index.html"><strong aria-hidden="true">7.</strong> eBPF 入门开发实践教程六：捕获进程发送信号的系统调用集合，使用 hash map 保存状态</a></li><li class="chapter-item expanded "><a href="../7-execsnoop/index.html"><strong aria-hidden="true">8.</strong> eBPF 入门实践教程七：捕获进程执行/退出时间，通过 perf event array 向用户态打印输出</a></li><li class="chapter-item expanded "><a href="../8-exitsnoop/index.html"><strong aria-hidden="true">9.</strong> eBPF 入门开发实践教程八：在 eBPF 中使用 exitsnoop 监控进程退出事件，使用 ring buffer 向用户态打印输出</a></li><li class="chapter-item expanded "><a href="../9-runqlat/index.html"><strong aria-hidden="true">10.</strong> eBPF 入门开发实践教程九：一个 Linux 内核 BPF 程序，通过柱状图来总结调度程序运行队列延迟，显示任务等待运行在 CPU 上的时间长度</a></li><li class="chapter-item expanded "><a href="../10-hardirqs/index.html"><strong aria-hidden="true">11.</strong> eBPF 入门开发实践教程十：在 eBPF 中使用 hardirqs 或 softirqs 捕获中断事件</a></li><li class="chapter-item expanded "><a href="../11-bootstrap/index.html"><strong aria-hidden="true">12.</strong> eBPF 入门开发实践教程十一：在 eBPF 中使用 bootstrap 开发用户态程序并跟踪 exec() 和 exit() 系统调用</a></li><li class="chapter-item expanded "><a href="../13-tcpconnlat/index.html"><strong aria-hidden="true">13.</strong> eBPF入门实践教程：使用 libbpf-bootstrap 开发程序统计 TCP 连接延时</a></li><li class="chapter-item expanded "><a href="../13-tcpconnlat/tcpconnlat.html"><strong aria-hidden="true">14.</strong> eBPF 入门实践教程：编写 eBPF 程序 tcpconnlat 测量 tcp 连接延时</a></li><li class="chapter-item expanded "><a href="../14-tcpstates/index.html"><strong aria-hidden="true">15.</strong> eBPF入门实践教程：使用 libbpf-bootstrap 开发程序统计 TCP 连接延时</a></li><li class="chapter-item expanded "><a href="../15-tcprtt/index.html"><strong aria-hidden="true">16.</strong> eBPF 入门实践教程：编写 eBPF 程序 Tcprtt 测量 TCP 连接的往返时间</a></li><li class="chapter-item expanded "><a href="../16-memleak/index.html"><strong aria-hidden="true">17.</strong> eBPF 入门实践教程：编写 eBPF 程序 Memleak 监控内存泄漏</a></li><li class="chapter-item expanded "><a href="../17-biopattern/index.html"><strong aria-hidden="true">18.</strong> eBPF 入门实践教程：编写 eBPF 程序 Biopattern: 统计随机/顺序磁盘 I/O</a></li><li class="chapter-item expanded "><a href="../18-further-reading/index.html"><strong aria-hidden="true">19.</strong> 更多的参考资料</a></li><li class="chapter-item expanded "><a href="../19-lsm-connect/index.html"><strong aria-hidden="true">20.</strong> eBPF 入门实践教程：使用 LSM 进行安全检测防御</a></li><li class="chapter-item expanded "><a href="../20-tc/index.html"><strong aria-hidden="true">21.</strong> eBPF 入门实践教程：使用 eBPF 进行 tc 流量控制</a></li><li class="chapter-item expanded affix "><li class="part-title">bcc 开发者教程</li><li class="chapter-item expanded "><a href="../bcc-documents/kernel-versions.html"><strong aria-hidden="true">22.</strong> BPF Features by Linux Kernel Version</a></li><li class="chapter-item expanded "><a href="../bcc-documents/kernel_config.html"><strong aria-hidden="true">23.</strong> Kernel Configuration for BPF Features</a></li><li class="chapter-item expanded "><a href="../bcc-documents/reference_guide.html"><strong aria-hidden="true">24.</strong> bcc Reference Guide</a></li><li class="chapter-item expanded "><a href="../bcc-documents/special_filtering.html"><strong aria-hidden="true">25.</strong> Special Filtering</a></li><li class="chapter-item expanded "><a href="../bcc-documents/tutorial.html" class="active"><strong aria-hidden="true">26.</strong> bcc Tutorial</a></li><li class="chapter-item expanded "><a href="../bcc-documents/tutorial_bcc_python_developer.html"><strong aria-hidden="true">27.</strong> bcc Python Developer Tutorial</a></li></ol>
|
||
</div>
|
||
<div id="sidebar-resize-handle" class="sidebar-resize-handle"></div>
|
||
</nav>
|
||
|
||
<div id="page-wrapper" class="page-wrapper">
|
||
|
||
<div class="page">
|
||
<div id="menu-bar-hover-placeholder"></div>
|
||
<div id="menu-bar" class="menu-bar sticky bordered">
|
||
<div class="left-buttons">
|
||
<button id="sidebar-toggle" class="icon-button" type="button" title="Toggle Table of Contents" aria-label="Toggle Table of Contents" aria-controls="sidebar">
|
||
<i class="fa fa-bars"></i>
|
||
</button>
|
||
<button id="theme-toggle" class="icon-button" type="button" title="Change theme" aria-label="Change theme" aria-haspopup="true" aria-expanded="false" aria-controls="theme-list">
|
||
<i class="fa fa-paint-brush"></i>
|
||
</button>
|
||
<ul id="theme-list" class="theme-popup" aria-label="Themes" role="menu">
|
||
<li role="none"><button role="menuitem" class="theme" id="light">Light</button></li>
|
||
<li role="none"><button role="menuitem" class="theme" id="rust">Rust</button></li>
|
||
<li role="none"><button role="menuitem" class="theme" id="coal">Coal</button></li>
|
||
<li role="none"><button role="menuitem" class="theme" id="navy">Navy</button></li>
|
||
<li role="none"><button role="menuitem" class="theme" id="ayu">Ayu</button></li>
|
||
</ul>
|
||
<button id="search-toggle" class="icon-button" type="button" title="Search. (Shortkey: s)" aria-label="Toggle Searchbar" aria-expanded="false" aria-keyshortcuts="S" aria-controls="searchbar">
|
||
<i class="fa fa-search"></i>
|
||
</button>
|
||
</div>
|
||
|
||
<h1 class="menu-title">bpf-developer-tutorial</h1>
|
||
|
||
<div class="right-buttons">
|
||
<a href="../print.html" title="Print this book" aria-label="Print this book">
|
||
<i id="print-button" class="fa fa-print"></i>
|
||
</a>
|
||
|
||
</div>
|
||
</div>
|
||
|
||
<div id="search-wrapper" class="hidden">
|
||
<form id="searchbar-outer" class="searchbar-outer">
|
||
<input type="search" id="searchbar" name="searchbar" placeholder="Search this book ..." aria-controls="searchresults-outer" aria-describedby="searchresults-header">
|
||
</form>
|
||
<div id="searchresults-outer" class="searchresults-outer hidden">
|
||
<div id="searchresults-header" class="searchresults-header"></div>
|
||
<ul id="searchresults">
|
||
</ul>
|
||
</div>
|
||
</div>
|
||
|
||
<!-- Apply ARIA attributes after the sidebar and the sidebar toggle button are added to the DOM -->
|
||
<script>
|
||
document.getElementById('sidebar-toggle').setAttribute('aria-expanded', sidebar === 'visible');
|
||
document.getElementById('sidebar').setAttribute('aria-hidden', sidebar !== 'visible');
|
||
Array.from(document.querySelectorAll('#sidebar a')).forEach(function(link) {
|
||
link.setAttribute('tabIndex', sidebar === 'visible' ? 0 : -1);
|
||
});
|
||
</script>
|
||
|
||
<div id="content" class="content">
|
||
<main>
|
||
<h1 id="bcc-tutorial"><a class="header" href="#bcc-tutorial">bcc Tutorial</a></h1>
|
||
<p>This tutorial covers how to use <a href="https://github.com/iovisor/bcc">bcc</a> tools to quickly solve performance, troubleshooting, and networking issues. If you want to develop new bcc tools, see <a href="tutorial_bcc_python_developer.html">tutorial_bcc_python_developer.md</a> for that tutorial.</p>
|
||
<p>It is assumed for this tutorial that bcc is already installed, and you can run tools like execsnoop successfully. See <a href="https://github.com/iovisor/bcc/tree/master/INSTALL.md">INSTALL.md</a>. This uses enhancements added to the Linux 4.x series.</p>
|
||
<h2 id="observability"><a class="header" href="#observability">Observability</a></h2>
|
||
<p>Some quick wins.</p>
|
||
<h3 id="0-before-bcc"><a class="header" href="#0-before-bcc">0. Before bcc</a></h3>
|
||
<p>Before using bcc, you should start with the Linux basics. One reference is the <a href="https://netflixtechblog.com/linux-performance-analysis-in-60-000-milliseconds-accc10403c55">Linux Performance Analysis in 60,000 Milliseconds</a> post, which covers these commands:</p>
|
||
<ol>
|
||
<li>uptime</li>
|
||
<li>dmesg | tail</li>
|
||
<li>vmstat 1</li>
|
||
<li>mpstat -P ALL 1</li>
|
||
<li>pidstat 1</li>
|
||
<li>iostat -xz 1</li>
|
||
<li>free -m</li>
|
||
<li>sar -n DEV 1</li>
|
||
<li>sar -n TCP,ETCP 1</li>
|
||
<li>top</li>
|
||
</ol>
|
||
<h3 id="1-general-performance"><a class="header" href="#1-general-performance">1. General Performance</a></h3>
|
||
<p>Here is a generic checklist for performance investigations with bcc, first as a list, then in detail:</p>
|
||
<ol>
|
||
<li>execsnoop</li>
|
||
<li>opensnoop</li>
|
||
<li>ext4slower (or btrfs*, xfs*, zfs*)</li>
|
||
<li>biolatency</li>
|
||
<li>biosnoop</li>
|
||
<li>cachestat</li>
|
||
<li>tcpconnect</li>
|
||
<li>tcpaccept</li>
|
||
<li>tcpretrans</li>
|
||
<li>runqlat</li>
|
||
<li>profile</li>
|
||
</ol>
|
||
<p>These tools may be installed on your system under /usr/share/bcc/tools, or you can run them from the bcc github repo under /tools where they have a .py extension. Browse the 50+ tools available for more analysis options.</p>
|
||
<h4 id="11-execsnoop"><a class="header" href="#11-execsnoop">1.1 execsnoop</a></h4>
|
||
<pre><code class="language-sh"># ./execsnoop
|
||
PCOMM PID RET ARGS
|
||
supervise 9660 0 ./run
|
||
supervise 9661 0 ./run
|
||
mkdir 9662 0 /bin/mkdir -p ./main
|
||
run 9663 0 ./run
|
||
[...]
|
||
</code></pre>
|
||
<p>execsnoop prints one line of output for each new process. Check for short-lived processes. These can consume CPU resources, but not show up in most monitoring tools that periodically take snapshots of which processes are running.</p>
|
||
<p>It works by tracing exec(), not the fork(), so it will catch many types of new processes but not all (eg, it won't see an application launching working processes, that doesn't exec() anything else).</p>
|
||
<p>More <a href="https://github.com/iovisor/bcc/tree/master/tools/execsnoop_example.txt">examples</a>.</p>
|
||
<h4 id="12-opensnoop"><a class="header" href="#12-opensnoop">1.2. opensnoop</a></h4>
|
||
<pre><code class="language-sh"># ./opensnoop
|
||
PID COMM FD ERR PATH
|
||
1565 redis-server 5 0 /proc/1565/stat
|
||
1565 redis-server 5 0 /proc/1565/stat
|
||
1565 redis-server 5 0 /proc/1565/stat
|
||
1603 snmpd 9 0 /proc/net/dev
|
||
1603 snmpd 11 0 /proc/net/if_inet6
|
||
1603 snmpd -1 2 /sys/class/net/eth0/device/vendor
|
||
1603 snmpd 11 0 /proc/sys/net/ipv4/neigh/eth0/retrans_time_ms
|
||
1603 snmpd 11 0 /proc/sys/net/ipv6/neigh/eth0/retrans_time_ms
|
||
1603 snmpd 11 0 /proc/sys/net/ipv6/conf/eth0/forwarding
|
||
[...]
|
||
</code></pre>
|
||
<p>opensnoop prints one line of output for each open() syscall, including details.</p>
|
||
<p>Files that are opened can tell you a lot about how applications work: identifying their data files, config files, and log files. Sometimes applications can misbehave, and perform poorly, when they are constantly attempting to read files that do not exist. opensnoop gives you a quick look.</p>
|
||
<p>More <a href="https://github.com/iovisor/bcc/tree/master/tools/opensnoop_example.txt">examples</a>.</p>
|
||
<h4 id="13-ext4slower-or-btrfs-xfs-zfs"><a class="header" href="#13-ext4slower-or-btrfs-xfs-zfs">1.3. ext4slower (or btrfs*, xfs*, zfs*)</a></h4>
|
||
<pre><code class="language-sh"># ./ext4slower
|
||
Tracing ext4 operations slower than 10 ms
|
||
TIME COMM PID T BYTES OFF_KB LAT(ms) FILENAME
|
||
06:35:01 cron 16464 R 1249 0 16.05 common-auth
|
||
06:35:01 cron 16463 R 1249 0 16.04 common-auth
|
||
06:35:01 cron 16465 R 1249 0 16.03 common-auth
|
||
06:35:01 cron 16465 R 4096 0 10.62 login.defs
|
||
06:35:01 cron 16464 R 4096 0 10.61 login.defs
|
||
</code></pre>
|
||
<p>ext4slower traces the ext4 file system and times common operations, and then only prints those that exceed a threshold.</p>
|
||
<p>This is great for identifying or exonerating one type of performance issue: show individually slow disk i/O via the file system. Disks process I/O asynchronously, and it can be difficult to associate latency at that layer with the latency applications experience. Tracing higher up in the kernel stack, at the VFS -> file system interface, will more closely match what an application suffers. Use this tool to identify if file system latency exceeds a given threshold.</p>
|
||
<p>Similar tools exist in bcc for other file systems: btrfsslower, xfsslower, and zfsslower. There is also fileslower, which works at the VFS layer and traces everything (although at some higher overhead).</p>
|
||
<p>More <a href="https://github.com/iovisor/bcc/tree/master/tools/ext4slower_example.txt">examples</a>.</p>
|
||
<h4 id="14-biolatency"><a class="header" href="#14-biolatency">1.4. biolatency</a></h4>
|
||
<pre><code class="language-sh"># ./biolatency
|
||
Tracing block device I/O... Hit Ctrl-C to end.
|
||
^C
|
||
usecs : count distribution
|
||
0 -> 1 : 0 | |
|
||
2 -> 3 : 0 | |
|
||
4 -> 7 : 0 | |
|
||
8 -> 15 : 0 | |
|
||
16 -> 31 : 0 | |
|
||
32 -> 63 : 0 | |
|
||
64 -> 127 : 1 | |
|
||
128 -> 255 : 12 |******** |
|
||
256 -> 511 : 15 |********** |
|
||
512 -> 1023 : 43 |******************************* |
|
||
1024 -> 2047 : 52 |**************************************|
|
||
2048 -> 4095 : 47 |********************************** |
|
||
4096 -> 8191 : 52 |**************************************|
|
||
8192 -> 16383 : 36 |************************** |
|
||
16384 -> 32767 : 15 |********** |
|
||
32768 -> 65535 : 2 |* |
|
||
65536 -> 131071 : 2 |* |
|
||
</code></pre>
|
||
<p>biolatency traces disk I/O latency (time from device issue to completion), and when the tool ends (Ctrl-C, or a given interval), it prints a histogram summary of the latency.</p>
|
||
<p>This is great for understanding disk I/O latency beyond the average times given by tools like iostat. I/O latency outliers will be visible at the end of the distribution, as well as multi-mode distributions.</p>
|
||
<p>More <a href="https://github.com/iovisor/bcc/tree/master/tools/biolatency_example.txt">examples</a>.</p>
|
||
<h4 id="15-biosnoop"><a class="header" href="#15-biosnoop">1.5. biosnoop</a></h4>
|
||
<pre><code class="language-sh"># ./biosnoop
|
||
TIME(s) COMM PID DISK T SECTOR BYTES LAT(ms)
|
||
0.000004001 supervise 1950 xvda1 W 13092560 4096 0.74
|
||
0.000178002 supervise 1950 xvda1 W 13092432 4096 0.61
|
||
0.001469001 supervise 1956 xvda1 W 13092440 4096 1.24
|
||
0.001588002 supervise 1956 xvda1 W 13115128 4096 1.09
|
||
1.022346001 supervise 1950 xvda1 W 13115272 4096 0.98
|
||
1.022568002 supervise 1950 xvda1 W 13188496 4096 0.93
|
||
[...]
|
||
</code></pre>
|
||
<p>biosnoop prints a line of output for each disk I/O, with details including latency (time from device issue to completion).</p>
|
||
<p>This allows you to examine disk I/O in more detail, and look for time-ordered patterns (eg, reads queueing behind writes). Note that the output will be verbose if your system performs disk I/O at a high rate.</p>
|
||
<p>More <a href="https://github.com/iovisor/bcc/tree/master/tools/biosnoop_example.txt">examples</a>.</p>
|
||
<h4 id="16-cachestat"><a class="header" href="#16-cachestat">1.6. cachestat</a></h4>
|
||
<pre><code class="language-sh"># ./cachestat
|
||
HITS MISSES DIRTIES READ_HIT% WRITE_HIT% BUFFERS_MB CACHED_MB
|
||
1074 44 13 94.9% 2.9% 1 223
|
||
2195 170 8 92.5% 6.8% 1 143
|
||
182 53 56 53.6% 1.3% 1 143
|
||
62480 40960 20480 40.6% 19.8% 1 223
|
||
7 2 5 22.2% 22.2% 1 223
|
||
348 0 0 100.0% 0.0% 1 223
|
||
[...]
|
||
</code></pre>
|
||
<p>cachestat prints a one line summary every second (or every custom interval) showing statistics from the file system cache.</p>
|
||
<p>Use this to identify a low cache hit ratio, and a high rate of misses: which gives one lead for performance tuning.</p>
|
||
<p>More <a href="https://github.com/iovisor/bcc/tree/master/tools/cachestat_example.txt">examples</a>.</p>
|
||
<h4 id="17-tcpconnect"><a class="header" href="#17-tcpconnect">1.7. tcpconnect</a></h4>
|
||
<pre><code class="language-sh"># ./tcpconnect
|
||
PID COMM IP SADDR DADDR DPORT
|
||
1479 telnet 4 127.0.0.1 127.0.0.1 23
|
||
1469 curl 4 10.201.219.236 54.245.105.25 80
|
||
1469 curl 4 10.201.219.236 54.67.101.145 80
|
||
1991 telnet 6 ::1 ::1 23
|
||
2015 ssh 6 fe80::2000:bff:fe82:3ac fe80::2000:bff:fe82:3ac 22
|
||
[...]
|
||
</code></pre>
|
||
<p>tcpconnect prints one line of output for every active TCP connection (eg, via connect()), with details including source and destination addresses.</p>
|
||
<p>Look for unexpected connections that may point to inefficiencies in application configuration, or an intruder.</p>
|
||
<p>More <a href="https://github.com/iovisor/bcc/tree/master/tools/tcpconnect_example.txt">examples</a>.</p>
|
||
<h4 id="18-tcpaccept"><a class="header" href="#18-tcpaccept">1.8. tcpaccept</a></h4>
|
||
<pre><code class="language-sh"># ./tcpaccept
|
||
PID COMM IP RADDR LADDR LPORT
|
||
907 sshd 4 192.168.56.1 192.168.56.102 22
|
||
907 sshd 4 127.0.0.1 127.0.0.1 22
|
||
5389 perl 6 1234:ab12:2040:5020:2299:0:5:0 1234:ab12:2040:5020:2299:0:5:0 7001
|
||
[...]
|
||
</code></pre>
|
||
<p>tcpaccept prints one line of output for every passive TCP connection (eg, via accept()), with details including source and destination addresses.</p>
|
||
<p>Look for unexpected connections that may point to inefficiencies in application configuration, or an intruder.</p>
|
||
<p>More <a href="https://github.com/iovisor/bcc/tree/master/tools/tcpaccept_example.txt">examples</a>.</p>
|
||
<h4 id="19-tcpretrans"><a class="header" href="#19-tcpretrans">1.9. tcpretrans</a></h4>
|
||
<pre><code class="language-sh"># ./tcpretrans
|
||
TIME PID IP LADDR:LPORT T> RADDR:RPORT STATE
|
||
01:55:05 0 4 10.153.223.157:22 R> 69.53.245.40:34619 ESTABLISHED
|
||
01:55:05 0 4 10.153.223.157:22 R> 69.53.245.40:34619 ESTABLISHED
|
||
01:55:17 0 4 10.153.223.157:22 R> 69.53.245.40:22957 ESTABLISHED
|
||
[...]
|
||
</code></pre>
|
||
<p>tcprerans prints one line of output for every TCP retransmit packet, with details including source and destination addresses, and kernel state of the TCP connection.</p>
|
||
<p>TCP retransmissions cause latency and throughput issues. For ESTABLISHED retransmits, look for patterns with networks. For SYN_SENT, this may point to target kernel CPU saturation and kernel packet drops.</p>
|
||
<p>More <a href="https://github.com/iovisor/bcc/tree/master/tools/tcpretrans_example.txt">examples</a>.</p>
|
||
<h4 id="110-runqlat"><a class="header" href="#110-runqlat">1.10. runqlat</a></h4>
|
||
<pre><code class="language-sh"># ./runqlat
|
||
Tracing run queue latency... Hit Ctrl-C to end.
|
||
^C
|
||
usecs : count distribution
|
||
0 -> 1 : 233 |*********** |
|
||
2 -> 3 : 742 |************************************ |
|
||
4 -> 7 : 203 |********** |
|
||
8 -> 15 : 173 |******** |
|
||
16 -> 31 : 24 |* |
|
||
32 -> 63 : 0 | |
|
||
64 -> 127 : 30 |* |
|
||
128 -> 255 : 6 | |
|
||
256 -> 511 : 3 | |
|
||
512 -> 1023 : 5 | |
|
||
1024 -> 2047 : 27 |* |
|
||
2048 -> 4095 : 30 |* |
|
||
4096 -> 8191 : 20 | |
|
||
8192 -> 16383 : 29 |* |
|
||
16384 -> 32767 : 809 |****************************************|
|
||
32768 -> 65535 : 64 |*** |
|
||
</code></pre>
|
||
<p>runqlat times how long threads were waiting on the CPU run queues, and prints this as a histogram.</p>
|
||
<p>This can help quantify time lost waiting for a turn on CPU, during periods of CPU saturation.</p>
|
||
<p>More <a href="https://github.com/iovisor/bcc/tree/master/tools/runqlat_example.txt">examples</a>.</p>
|
||
<h4 id="111-profile"><a class="header" href="#111-profile">1.11. profile</a></h4>
|
||
<pre><code class="language-sh"># ./profile
|
||
Sampling at 49 Hertz of all threads by user + kernel stack... Hit Ctrl-C to end.
|
||
^C
|
||
00007f31d76c3251 [unknown]
|
||
47a2c1e752bf47f7 [unknown]
|
||
- sign-file (8877)
|
||
1
|
||
|
||
ffffffff813d0af8 __clear_user
|
||
ffffffff813d5277 iov_iter_zero
|
||
ffffffff814ec5f2 read_iter_zero
|
||
ffffffff8120be9d __vfs_read
|
||
ffffffff8120c385 vfs_read
|
||
ffffffff8120d786 sys_read
|
||
ffffffff817cc076 entry_SYSCALL_64_fastpath
|
||
00007fc5652ad9b0 read
|
||
- dd (25036)
|
||
4
|
||
|
||
0000000000400542 func_a
|
||
0000000000400598 main
|
||
00007f12a133e830 __libc_start_main
|
||
083e258d4c544155 [unknown]
|
||
- func_ab (13549)
|
||
5
|
||
|
||
[...]
|
||
|
||
ffffffff8105eb66 native_safe_halt
|
||
ffffffff8103659e default_idle
|
||
ffffffff81036d1f arch_cpu_idle
|
||
ffffffff810bba5a default_idle_call
|
||
ffffffff810bbd07 cpu_startup_entry
|
||
ffffffff8104df55 start_secondary
|
||
- swapper/1 (0)
|
||
75
|
||
</code></pre>
|
||
<p>profile is a CPU profiler, which takes samples of stack traces at timed intervals, and prints a summary of unique stack traces and a count of their occurrence.</p>
|
||
<p>Use this tool to understand the code paths that are consuming CPU resources.</p>
|
||
<p>More <a href="https://github.com/iovisor/bcc/tree/master/tools/profile_example.txt">examples</a>.</p>
|
||
<h3 id="2-observability-with-generic-tools"><a class="header" href="#2-observability-with-generic-tools">2. Observability with Generic Tools</a></h3>
|
||
<p>In addition to the above tools for performance tuning, below is a checklist for bcc generic tools, first as a list, and in detail:</p>
|
||
<ol>
|
||
<li>trace</li>
|
||
<li>argdist</li>
|
||
<li>funccount</li>
|
||
</ol>
|
||
<p>These generic tools may be useful to provide visibility to solve your specific problems.</p>
|
||
<h4 id="21-trace"><a class="header" href="#21-trace">2.1. trace</a></h4>
|
||
<h5 id="example-1"><a class="header" href="#example-1">Example 1</a></h5>
|
||
<p>Suppose you want to track file ownership change. There are three syscalls, <code>chown</code>, <code>fchown</code> and <code>lchown</code> which users can use to change file ownership. The corresponding syscall entry is <code>SyS_[f|l]chown</code>. The following command can be used to print out syscall parameters and the calling process user id. You can use <code>id</code> command to find the uid of a particular user.</p>
|
||
<pre><code class="language-sh">$ trace.py \
|
||
'p::SyS_chown "file = %s, to_uid = %d, to_gid = %d, from_uid = %d", arg1, arg2, arg3, $uid' \
|
||
'p::SyS_fchown "fd = %d, to_uid = %d, to_gid = %d, from_uid = %d", arg1, arg2, arg3, $uid' \
|
||
'p::SyS_lchown "file = %s, to_uid = %d, to_gid = %d, from_uid = %d", arg1, arg2, arg3, $uid'
|
||
PID TID COMM FUNC -
|
||
1269255 1269255 python3.6 SyS_lchown file = /tmp/dotsync-usisgezu/tmp, to_uid = 128203, to_gid = 100, from_uid = 128203
|
||
1269441 1269441 zstd SyS_chown file = /tmp/dotsync-vic7ygj0/dotsync-package.zst, to_uid = 128203, to_gid = 100, from_uid = 128203
|
||
1269255 1269255 python3.6 SyS_lchown file = /tmp/dotsync-a40zd7ev/tmp, to_uid = 128203, to_gid = 100, from_uid = 128203
|
||
1269442 1269442 zstd SyS_chown file = /tmp/dotsync-gzp413o_/dotsync-package.zst, to_uid = 128203, to_gid = 100, from_uid = 128203
|
||
1269255 1269255 python3.6 SyS_lchown file = /tmp/dotsync-whx4fivm/tmp/.bash_profile, to_uid = 128203, to_gid = 100, from_uid = 128203
|
||
</code></pre>
|
||
<h5 id="example-2"><a class="header" href="#example-2">Example 2</a></h5>
|
||
<p>Suppose you want to count nonvoluntary context switches (<code>nvcsw</code>) in your bpf based performance monitoring tools and you do not know what is the proper method. <code>/proc/<pid>/status</code> already tells you the number (<code>nonvoluntary_ctxt_switches</code>) for a pid and you can use <code>trace.py</code> to do a quick experiment to verify your method. With kernel source code, the <code>nvcsw</code> is counted at file <code>linux/kernel/sched/core.c</code> function <code>__schedule</code> and under condition</p>
|
||
<pre><code class="language-c">!(!preempt && prev->state) // i.e., preempt || !prev->state
|
||
</code></pre>
|
||
<p>The <code>__schedule</code> function is marked as <code>notrace</code>, and the best place to evaluate the above condition seems in <code>sched/sched_switch</code> tracepoint called inside function <code>__schedule</code> and defined in <code>linux/include/trace/events/sched.h</code>. <code>trace.py</code> already has <code>args</code> being the pointer to the tracepoint <code>TP_STRUCT__entry</code>. The above condition in function <code>__schedule</code> can be represented as</p>
|
||
<pre><code class="language-c">args->prev_state == TASK_STATE_MAX || args->prev_state == 0
|
||
</code></pre>
|
||
<p>The below command can be used to count the involuntary context switches (per process or per pid) and compare to <code>/proc/<pid>/status</code> or <code>/proc/<pid>/task/<task_id>/status</code> for correctness, as in typical cases, involuntary context switches are not very common.</p>
|
||
<pre><code class="language-sh">$ trace.py -p 1134138 't:sched:sched_switch (args->prev_state == TASK_STATE_MAX || args->prev_state == 0)'
|
||
PID TID COMM FUNC
|
||
1134138 1134140 contention_test sched_switch
|
||
1134138 1134142 contention_test sched_switch
|
||
...
|
||
$ trace.py -L 1134140 't:sched:sched_switch (args->prev_state == TASK_STATE_MAX || args->prev_state == 0)'
|
||
PID TID COMM FUNC
|
||
1134138 1134140 contention_test sched_switch
|
||
1134138 1134140 contention_test sched_switch
|
||
...
|
||
</code></pre>
|
||
<h5 id="example-3"><a class="header" href="#example-3">Example 3</a></h5>
|
||
<p>This example is related to issue <a href="https://github.com/iovisor/bcc/issues/1231">1231</a> and <a href="https://github.com/iovisor/bcc/issues/1516">1516</a> where uprobe does not work at all in certain cases. First, you can do a <code>strace</code> as below</p>
|
||
<pre><code class="language-sh">$ strace trace.py 'r:bash:readline "%s", retval'
|
||
...
|
||
perf_event_open(0x7ffd968212f0, -1, 0, -1, 0x8 /* PERF_FLAG_??? */) = -1 EIO (Input/output error)
|
||
...
|
||
</code></pre>
|
||
<p>The <code>perf_event_open</code> syscall returns <code>-EIO</code>. Digging into kernel uprobe related codes in <code>/kernel/trace</code> and <code>/kernel/events</code> directories to search <code>EIO</code>, the function <code>uprobe_register</code> is the most suspicious. Let us find whether this function is called or not and what is the return value if it is called. In one terminal using the following command to print out the return value of uprobe_register,</p>
|
||
<pre><code class="language-sh">$ trace.py 'r::uprobe_register "ret = %d", retval'
|
||
</code></pre>
|
||
<p>In another terminal run the same bash uretprobe tracing example, and you should get</p>
|
||
<pre><code class="language-sh">$ trace.py 'r::uprobe_register "ret = %d", retval'
|
||
PID TID COMM FUNC -
|
||
1041401 1041401 python2.7 uprobe_register ret = -5
|
||
</code></pre>
|
||
<p>The <code>-5</code> error code is EIO. This confirms that the following code in function <code>uprobe_register</code> is the most suspicious culprit.</p>
|
||
<pre><code class="language-c"> if (!inode->i_mapping->a_ops->readpage && !shmem_mapping(inode->i_mapping))
|
||
return -EIO;
|
||
</code></pre>
|
||
<p>The <code>shmem_mapping</code> function is defined as</p>
|
||
<pre><code class="language-c">bool shmem_mapping(struct address_space *mapping)
|
||
{
|
||
return mapping->a_ops == &shmem_aops;
|
||
}
|
||
</code></pre>
|
||
<p>To confirm the theory, find what is <code>inode->i_mapping->a_ops</code> with the following command</p>
|
||
<pre><code class="language-sh">$ trace.py -I 'linux/fs.h' 'p::uprobe_register(struct inode *inode) "a_ops = %llx", inode->i_mapping->a_ops'
|
||
PID TID COMM FUNC -
|
||
814288 814288 python2.7 uprobe_register a_ops = ffffffff81a2adc0
|
||
^C$ grep ffffffff81a2adc0 /proc/kallsyms
|
||
ffffffff81a2adc0 R empty_aops
|
||
</code></pre>
|
||
<p>The kernel symbol <code>empty_aops</code> does not have <code>readpage</code> defined and hence the above suspicious condition is true. Further examining the kernel source code shows that <code>overlayfs</code> does not provide its own <code>a_ops</code> while some other file systems (e.g., ext4) define their own <code>a_ops</code> (e.g., <code>ext4_da_aops</code>), and <code>ext4_da_aops</code> defines <code>readpage</code>. Hence, uprobe works fine on ext4 while not on overlayfs.</p>
|
||
<p>More <a href="https://github.com/iovisor/bcc/tree/master/tools/trace_example.txt">examples</a>.</p>
|
||
<h4 id="22-argdist"><a class="header" href="#22-argdist">2.2. argdist</a></h4>
|
||
<p>More <a href="https://github.com/iovisor/bcc/tree/master/tools/argdist_example.txt">examples</a>.</p>
|
||
<h4 id="23-funccount"><a class="header" href="#23-funccount">2.3. funccount</a></h4>
|
||
<p>More <a href="https://github.com/iovisor/bcc/tree/master/tools/funccount_example.txt">examples</a>.</p>
|
||
<h2 id="networking"><a class="header" href="#networking">Networking</a></h2>
|
||
<p>To do.</p>
|
||
|
||
</main>
|
||
|
||
<nav class="nav-wrapper" aria-label="Page navigation">
|
||
<!-- Mobile navigation buttons -->
|
||
<a rel="prev" href="../bcc-documents/special_filtering.html" class="mobile-nav-chapters previous" title="Previous chapter" aria-label="Previous chapter" aria-keyshortcuts="Left">
|
||
<i class="fa fa-angle-left"></i>
|
||
</a>
|
||
|
||
<a rel="next" href="../bcc-documents/tutorial_bcc_python_developer.html" class="mobile-nav-chapters next" title="Next chapter" aria-label="Next chapter" aria-keyshortcuts="Right">
|
||
<i class="fa fa-angle-right"></i>
|
||
</a>
|
||
|
||
<div style="clear: both"></div>
|
||
</nav>
|
||
</div>
|
||
</div>
|
||
|
||
<nav class="nav-wide-wrapper" aria-label="Page navigation">
|
||
<a rel="prev" href="../bcc-documents/special_filtering.html" class="nav-chapters previous" title="Previous chapter" aria-label="Previous chapter" aria-keyshortcuts="Left">
|
||
<i class="fa fa-angle-left"></i>
|
||
</a>
|
||
|
||
<a rel="next" href="../bcc-documents/tutorial_bcc_python_developer.html" class="nav-chapters next" title="Next chapter" aria-label="Next chapter" aria-keyshortcuts="Right">
|
||
<i class="fa fa-angle-right"></i>
|
||
</a>
|
||
</nav>
|
||
|
||
</div>
|
||
|
||
|
||
|
||
|
||
<script>
|
||
window.playground_copyable = true;
|
||
</script>
|
||
|
||
|
||
<script src="../elasticlunr.min.js"></script>
|
||
<script src="../mark.min.js"></script>
|
||
<script src="../searcher.js"></script>
|
||
|
||
<script src="../clipboard.min.js"></script>
|
||
<script src="../highlight.js"></script>
|
||
<script src="../book.js"></script>
|
||
|
||
<!-- Custom JS scripts -->
|
||
|
||
|
||
</div>
|
||
</body>
|
||
</html>
|