mirror of
https://github.com/eunomia-bpf/bpf-developer-tutorial.git
synced 2026-02-04 02:34:16 +08:00
529 lines
39 KiB
HTML
529 lines
39 KiB
HTML
<!DOCTYPE HTML>
|
||
<html lang="en" class="sidebar-visible no-js light">
|
||
<head>
|
||
<!-- Book generated using mdBook -->
|
||
<meta charset="UTF-8">
|
||
<title>bcc Tutorial - bpf-developer-tutorial</title>
|
||
|
||
|
||
<!-- Custom HTML head -->
|
||
|
||
<meta name="description" content="">
|
||
<meta name="viewport" content="width=device-width, initial-scale=1">
|
||
<meta name="theme-color" content="#ffffff" />
|
||
|
||
<link rel="icon" href="../favicon.svg">
|
||
<link rel="shortcut icon" href="../favicon.png">
|
||
<link rel="stylesheet" href="../css/variables.css">
|
||
<link rel="stylesheet" href="../css/general.css">
|
||
<link rel="stylesheet" href="../css/chrome.css">
|
||
<link rel="stylesheet" href="../css/print.css" media="print">
|
||
|
||
<!-- Fonts -->
|
||
<link rel="stylesheet" href="../FontAwesome/css/font-awesome.css">
|
||
<link rel="stylesheet" href="../fonts/fonts.css">
|
||
|
||
<!-- Highlight.js Stylesheets -->
|
||
<link rel="stylesheet" href="../highlight.css">
|
||
<link rel="stylesheet" href="../tomorrow-night.css">
|
||
<link rel="stylesheet" href="../ayu-highlight.css">
|
||
|
||
<!-- Custom theme stylesheets -->
|
||
|
||
</head>
|
||
<body>
|
||
<div id="body-container">
|
||
<!-- Provide site root to javascript -->
|
||
<script>
|
||
var path_to_root = "../";
|
||
var default_theme = window.matchMedia("(prefers-color-scheme: dark)").matches ? "navy" : "light";
|
||
</script>
|
||
|
||
<!-- Work around some values being stored in localStorage wrapped in quotes -->
|
||
<script>
|
||
try {
|
||
var theme = localStorage.getItem('mdbook-theme');
|
||
var sidebar = localStorage.getItem('mdbook-sidebar');
|
||
|
||
if (theme.startsWith('"') && theme.endsWith('"')) {
|
||
localStorage.setItem('mdbook-theme', theme.slice(1, theme.length - 1));
|
||
}
|
||
|
||
if (sidebar.startsWith('"') && sidebar.endsWith('"')) {
|
||
localStorage.setItem('mdbook-sidebar', sidebar.slice(1, sidebar.length - 1));
|
||
}
|
||
} catch (e) { }
|
||
</script>
|
||
|
||
<!-- Set the theme before any content is loaded, prevents flash -->
|
||
<script>
|
||
var theme;
|
||
try { theme = localStorage.getItem('mdbook-theme'); } catch(e) { }
|
||
if (theme === null || theme === undefined) { theme = default_theme; }
|
||
var html = document.querySelector('html');
|
||
html.classList.remove('no-js')
|
||
html.classList.remove('light')
|
||
html.classList.add(theme);
|
||
html.classList.add('js');
|
||
</script>
|
||
|
||
<!-- Hide / unhide sidebar before it is displayed -->
|
||
<script>
|
||
var html = document.querySelector('html');
|
||
var sidebar = null;
|
||
if (document.body.clientWidth >= 1080) {
|
||
try { sidebar = localStorage.getItem('mdbook-sidebar'); } catch(e) { }
|
||
sidebar = sidebar || 'visible';
|
||
} else {
|
||
sidebar = 'hidden';
|
||
}
|
||
html.classList.remove('sidebar-visible');
|
||
html.classList.add("sidebar-" + sidebar);
|
||
</script>
|
||
|
||
<nav id="sidebar" class="sidebar" aria-label="Table of contents">
|
||
<div class="sidebar-scrollbox">
|
||
<ol class="chapter"><li class="chapter-item expanded affix "><li class="part-title">eBPF 实践教程：基于 libbpf 和 CO-RE</li><li class="chapter-item expanded "><a href="../0-introduce/index.html"><strong aria-hidden="true">1.</strong> 介绍 eBPF 的基本概念、常见的开发工具</a></li><li class="chapter-item expanded "><a href="../1-helloworld/index.html"><strong aria-hidden="true">2.</strong> eBPF Hello World，基本框架和开发流程</a></li><li class="chapter-item expanded "><a href="../2-kprobe-unlink/index.html"><strong aria-hidden="true">3.</strong> 使用 kprobe 监测捕获 unlink 系统调用</a></li><li class="chapter-item expanded "><a href="../3-fentry-unlink/index.html"><strong aria-hidden="true">4.</strong> 使用 fentry 监测捕获 unlink 系统调用</a></li><li class="chapter-item expanded "><a href="../4-opensnoop/index.html"><strong aria-hidden="true">5.</strong> 捕获进程打开文件的系统调用集合，使用全局变量过滤进程 pid</a></li><li class="chapter-item expanded "><a href="../5-uprobe-bashreadline/index.html"><strong aria-hidden="true">6.</strong> 使用 uprobe 捕获 bash 的 readline 函数调用</a></li><li class="chapter-item expanded "><a href="../6-sigsnoop/index.html"><strong aria-hidden="true">7.</strong> 捕获进程发送信号的系统调用集合，使用 hash map 保存状态</a></li><li class="chapter-item expanded "><a href="../7-execsnoop/index.html"><strong aria-hidden="true">8.</strong> 捕获进程执行/退出时间，通过 perf event array 向用户态打印输出</a></li><li class="chapter-item expanded "><a href="../8-exitsnoop/index.html"><strong aria-hidden="true">9.</strong> 使用 exitsnoop 监控进程退出事件，使用 ring buffer 向用户态打印输出</a></li><li class="chapter-item expanded "><a href="../9-runqlat/index.html"><strong aria-hidden="true">10.</strong> 一个 Linux 内核 BPF 程序，通过柱状图来总结调度程序运行队列延迟，显示任务等待运行在 CPU 上的时间长度</a></li><li class="chapter-item expanded "><a href="../10-hardirqs/index.html"><strong aria-hidden="true">11.</strong> 使用 hardirqs 或 softirqs 捕获中断事件</a></li><li class="chapter-item expanded "><a href="../11-bootstrap/index.html"><strong aria-hidden="true">12.</strong> 使用 bootstrap 开发用户态程序并跟踪 exec() 和 exit() 系统调用</a></li><li class="chapter-item expanded "><a href="../13-tcpconnlat/index.html"><strong aria-hidden="true">13.</strong> 使用 libbpf-bootstrap 开发程序统计 TCP 连接延时</a></li><li class="chapter-item expanded "><a href="../14-tcpstates/index.html"><strong aria-hidden="true">14.</strong> 使用 libbpf-bootstrap 记录 TCP 连接状态与 TCP RTT</a></li><li class="chapter-item expanded "><a href="../15-javagc/index.html"><strong aria-hidden="true">15.</strong> 使用 USDT 捕获用户态 Java GC 事件耗时</a></li><li class="chapter-item expanded "><a href="../16-memleak/index.html"><strong aria-hidden="true">16.</strong> 编写 eBPF 程序 Memleak 监控内存泄漏</a></li><li class="chapter-item expanded "><a href="../17-biopattern/index.html"><strong aria-hidden="true">17.</strong> 编写 eBPF 程序 Biopattern 统计随机/顺序磁盘 I/O</a></li><li class="chapter-item expanded "><a href="../18-further-reading/index.html"><strong aria-hidden="true">18.</strong> 更多的参考资料</a></li><li class="chapter-item expanded "><a href="../19-lsm-connect/index.html"><strong aria-hidden="true">19.</strong> 使用 LSM 进行安全检测防御</a></li><li class="chapter-item expanded "><a href="../20-tc/index.html"><strong aria-hidden="true">20.</strong> 使用 eBPF 进行 tc 流量控制</a></li><li class="chapter-item expanded affix "><li class="part-title">eBPF 高级特性与进阶主题</li><li class="chapter-item expanded "><a href="../22-android/index.html"><strong aria-hidden="true">21.</strong> 在 Android 上使用 eBPF 程序</a></li><li class="chapter-item expanded "><a href="../23-http/index.html"><strong aria-hidden="true">22.</strong> 使用 eBPF 追踪 HTTP 请求或其他七层协议</a></li><li class="chapter-item expanded "><a href="../30-sslsniff/index.html"><strong aria-hidden="true">23.</strong> 使用 uprobe 捕获多种库的 SSL/TLS 明文数据</a></li><li class="chapter-item expanded "><a href="../29-sockops/index.html"><strong aria-hidden="true">24.</strong> 使用 sockops 加速网络请求转发</a></li><li class="chapter-item expanded "><a href="../24-hide/index.html"><strong aria-hidden="true">25.</strong> 使用 eBPF 隐藏进程或文件信息</a></li><li class="chapter-item expanded "><a href="../25-signal/index.html"><strong aria-hidden="true">26.</strong> 使用 bpf_send_signal 发送信号终止进程</a></li><li class="chapter-item expanded "><a href="../26-sudo/index.html"><strong aria-hidden="true">27.</strong> 使用 eBPF 添加 sudo 用户</a></li><li class="chapter-item expanded "><a href="../27-replace/index.html"><strong aria-hidden="true">28.</strong> 使用 eBPF 替换任意程序读取或写入的文本</a></li><li class="chapter-item expanded "><a href="../28-detach/index.html"><strong aria-hidden="true">29.</strong> BPF的生命周期：使用 Detached 模式在用户态应用退出后持续运行 eBPF 程序</a></li><li class="chapter-item expanded "><a href="../30-sslsniff/index.html"><strong aria-hidden="true">30.</strong> 使用 eBPF 用户态捕获多种库的 SSL/TLS 明文数据</a></li><li class="chapter-item expanded affix "><li class="part-title">bcc tutorial</li><li class="chapter-item expanded "><a href="../bcc-documents/kernel-versions.html"><strong aria-hidden="true">31.</strong> BPF Features by Linux Kernel Version</a></li><li class="chapter-item expanded "><a href="../bcc-documents/kernel_config.html"><strong aria-hidden="true">32.</strong> Kernel Configuration for BPF Features</a></li><li class="chapter-item expanded "><a href="../bcc-documents/reference_guide.html"><strong aria-hidden="true">33.</strong> bcc Reference Guide</a></li><li class="chapter-item expanded "><a href="../bcc-documents/special_filtering.html"><strong aria-hidden="true">34.</strong> Special Filtering</a></li><li class="chapter-item expanded "><a href="../bcc-documents/tutorial.html" class="active"><strong aria-hidden="true">35.</strong> bcc Tutorial</a></li><li class="chapter-item expanded "><a href="../bcc-documents/tutorial_bcc_python_developer.html"><strong aria-hidden="true">36.</strong> bcc Python Developer Tutorial</a></li></ol>
|
||
</div>
|
||
<div id="sidebar-resize-handle" class="sidebar-resize-handle"></div>
|
||
</nav>
|
||
|
||
<!-- Track and set sidebar scroll position -->
|
||
<script>
|
||
var sidebarScrollbox = document.querySelector('#sidebar .sidebar-scrollbox');
|
||
sidebarScrollbox.addEventListener('click', function(e) {
|
||
if (e.target.tagName === 'A') {
|
||
sessionStorage.setItem('sidebar-scroll', sidebarScrollbox.scrollTop);
|
||
}
|
||
}, { passive: true });
|
||
var sidebarScrollTop = sessionStorage.getItem('sidebar-scroll');
|
||
sessionStorage.removeItem('sidebar-scroll');
|
||
if (sidebarScrollTop) {
|
||
// preserve sidebar scroll position when navigating via links within sidebar
|
||
sidebarScrollbox.scrollTop = sidebarScrollTop;
|
||
} else {
|
||
// scroll sidebar to current active section when navigating via "next/previous chapter" buttons
|
||
var activeSection = document.querySelector('#sidebar .active');
|
||
if (activeSection) {
|
||
activeSection.scrollIntoView({ block: 'center' });
|
||
}
|
||
}
|
||
</script>
|
||
|
||
<div id="page-wrapper" class="page-wrapper">
|
||
|
||
<div class="page">
|
||
<div id="menu-bar-hover-placeholder"></div>
|
||
<div id="menu-bar" class="menu-bar sticky">
|
||
<div class="left-buttons">
|
||
<button id="sidebar-toggle" class="icon-button" type="button" title="Toggle Table of Contents" aria-label="Toggle Table of Contents" aria-controls="sidebar">
|
||
<i class="fa fa-bars"></i>
|
||
</button>
|
||
<button id="theme-toggle" class="icon-button" type="button" title="Change theme" aria-label="Change theme" aria-haspopup="true" aria-expanded="false" aria-controls="theme-list">
|
||
<i class="fa fa-paint-brush"></i>
|
||
</button>
|
||
<ul id="theme-list" class="theme-popup" aria-label="Themes" role="menu">
|
||
<li role="none"><button role="menuitem" class="theme" id="light">Light</button></li>
|
||
<li role="none"><button role="menuitem" class="theme" id="rust">Rust</button></li>
|
||
<li role="none"><button role="menuitem" class="theme" id="coal">Coal</button></li>
|
||
<li role="none"><button role="menuitem" class="theme" id="navy">Navy</button></li>
|
||
<li role="none"><button role="menuitem" class="theme" id="ayu">Ayu</button></li>
|
||
</ul>
|
||
<button id="search-toggle" class="icon-button" type="button" title="Search. (Shortkey: s)" aria-label="Toggle Searchbar" aria-expanded="false" aria-keyshortcuts="S" aria-controls="searchbar">
|
||
<i class="fa fa-search"></i>
|
||
</button>
|
||
</div>
|
||
|
||
<h1 class="menu-title">bpf-developer-tutorial</h1>
|
||
|
||
<div class="right-buttons">
|
||
<a href="../print.html" title="Print this book" aria-label="Print this book">
|
||
<i id="print-button" class="fa fa-print"></i>
|
||
</a>
|
||
|
||
</div>
|
||
</div>
|
||
|
||
<div id="search-wrapper" class="hidden">
|
||
<form id="searchbar-outer" class="searchbar-outer">
|
||
<input type="search" id="searchbar" name="searchbar" placeholder="Search this book ..." aria-controls="searchresults-outer" aria-describedby="searchresults-header">
|
||
</form>
|
||
<div id="searchresults-outer" class="searchresults-outer hidden">
|
||
<div id="searchresults-header" class="searchresults-header"></div>
|
||
<ul id="searchresults">
|
||
</ul>
|
||
</div>
|
||
</div>
|
||
|
||
<!-- Apply ARIA attributes after the sidebar and the sidebar toggle button are added to the DOM -->
|
||
<script>
|
||
document.getElementById('sidebar-toggle').setAttribute('aria-expanded', sidebar === 'visible');
|
||
document.getElementById('sidebar').setAttribute('aria-hidden', sidebar !== 'visible');
|
||
Array.from(document.querySelectorAll('#sidebar a')).forEach(function(link) {
|
||
link.setAttribute('tabIndex', sidebar === 'visible' ? 0 : -1);
|
||
});
|
||
</script>
|
||
|
||
<div id="content" class="content">
|
||
<main>
|
||
<h1 id="bcc-教程"><a class="header" href="#bcc-教程">bcc 教程</a></h1>
|
||
<p>本教程介绍如何使用<a href="https://github.com/iovisor/bcc">bcc</a>工具快速解决性能、故障排除和网络问题。如果你想开发新的bcc工具，请参考<a href="tutorial_bcc_python_developer.html">tutorial_bcc_python_developer.md</a>教程。</p>
|
||
<p>本教程假设bcc已经安装好，并且你可以成功运行像execsnoop这样的工具。参见<a href="https://github.com/iovisor/bcc/tree/master/INSTALL.md">INSTALL.md</a>。这些功能是在Linux 4.x系列中增加的。</p>
|
||
<h2 id="可观察性"><a class="header" href="#可观察性">可观察性</a></h2>
|
||
<p>一些快速的收获。</p>
|
||
<h3 id="0-使用bcc之前"><a class="header" href="#0-使用bcc之前">0. 使用bcc之前</a></h3>
|
||
<p>在使用bcc之前，你应该从Linux基础知识开始。可以参考<a href="https://netflixtechblog.com/linux-performance-analysis-in-60-000-milliseconds-accc10403c55">Linux Performance Analysis in 60,000 Milliseconds</a>文章，其中介绍了以下命令：</p>
|
||
<ol>
|
||
<li>uptime</li>
|
||
<li>dmesg | tail</li>
|
||
<li>vmstat 1</li>
|
||
<li>mpstat -P ALL 1</li>
|
||
<li>pidstat 1</li>
|
||
<li>iostat -xz 1</li>
|
||
<li>free -m</li>
|
||
<li>sar -n DEV 1</li>
|
||
<li>sar -n TCP,ETCP 1</li>
|
||
<li>top</li>
|
||
</ol>
|
||
<h3 id="1-性能分析"><a class="header" href="#1-性能分析">1. 性能分析</a></h3>
|
||
<p>这是一个用于性能调查的通用检查清单，首先有一个列表，然后详细描述：</p>
|
||
<ol>
|
||
<li>execsnoop</li>
|
||
<li>opensnoop</li>
|
||
<li>ext4slower（或btrfs*，xfs*，zfs*）</li>
|
||
<li>biolatency</li>
|
||
<li>biosnoop</li>
|
||
<li>cachestat</li>
|
||
<li>tcpconnect</li>
|
||
<li>tcpaccept</li>
|
||
<li>tcpretrans</li>
|
||
<li>runqlat</li>
|
||
<li>profile</li>
|
||
</ol>
|
||
<p>这些工具可能已经安装在你的系统的/usr/share/bcc/tools目录下，或者你可以从bcc github仓库的/tools目录中运行它们，这些工具使用.py扩展名。浏览50多个可用的工具，获得更多的分析选项。</p>
|
||
<h4 id="11-execsnoop"><a class="header" href="#11-execsnoop">1.1 execsnoop</a></h4>
|
||
<pre><code class="language-sh"># ./execsnoop
|
||
PCOMM PID RET ARGS
|
||
supervise 9660 0 ./run
|
||
supervise 9661 0 ./run
|
||
mkdir 9662 0 /bin/mkdir -p ./main
|
||
run 9663 0 ./run
|
||
[...]
|
||
</code></pre>
|
||
<p>execsnoop对于每个新进程打印一行输出。检查短生命周期的进程。这些进程可能会消耗CPU资源，但不会在大多数周期性运行的进程监控工具中显示出来。它通过跟踪<code>exec()</code>来工作，而不是<code>fork()</code>，所以它可以捕获许多类型的新进程，但不是所有类型（例如，它不会看到启动工作进程的应用程序，该应用程序没有<code>exec()</code>其他任何内容）。</p>
|
||
<p>更多<a href="https://github.com/iovisor/bcc/tree/master/tools/execsnoop_example.txt">例子</a>。</p>
|
||
<h4 id="12-opensnoop"><a class="header" href="#12-opensnoop">1.2. opensnoop</a></h4>
|
||
<pre><code class="language-sh"># ./opensnoop
|
||
PID COMM FD ERR PATH
|
||
1565 redis-server 5 0 /proc/1565/stat
|
||
1565 redis-server 5 0 /proc/1565/stat
|
||
1565 redis-server 5 0 /proc/1565/stat
|
||
1603 snmpd 9 0 /proc/net/dev
|
||
1603 snmpd 11 0 /proc/net/if_inet6
|
||
1603 snmpd -1 2 /sys/class/net/eth0/device/vendor
|
||
1603 snmpd 11 0 /proc/sys/net/ipv4/neigh/eth0/retrans_time_ms
|
||
1603 snmpd 11 0 /proc/sys/net/ipv6/neigh/eth0/retrans_time_ms
|
||
1603 snmpd 11 0 /proc/sys/net/ipv6/conf/eth0/forwarding
|
||
[...]
|
||
</code></pre>
|
||
<p>opensnoop每次open() syscall执行时打印一行输出，包括详细信息。</p>
|
||
<p>打开的文件可以告诉你很多关于应用程序的工作方式的信息：它们的数据文件、配置文件和日志文件。有时候应用程序可能会表现不正常，当它们不断尝试读取不存在的文件时则会表现得很差。opensnoop能够快速帮助你查看。</p>
|
||
<p>更多<a href="https://github.com/iovisor/bcc/tree/master/tools/opensnoop_example.txt">例子</a>。</p>
|
||
<h4 id="13-ext4slower或btrfsxfszfs"><a class="header" href="#13-ext4slower或btrfsxfszfs">1.3. ext4slower（或btrfs*，xfs*，zfs*）</a></h4>
|
||
<pre><code class="language-sh"># ./ext4slower
|
||
追踪超过10毫秒的ext4操作
|
||
时间 进程 进程ID T 字节数 偏移KB 延迟(ms) 文件名
|
||
06:35:01 cron 16464 R 1249 0 16.05 common-auth
|
||
06:35:01 cron 16463 R 1249 0 16.04 common-auth
|
||
06:35:01 cron 16465 R 1249 0 16.03 common-auth
|
||
06:35:01 cron 16465 R 4096 0 10.62 login.defs
|
||
06:35:01 cron 16464 R 4096 0 10.61 login.defs
|
||
</code></pre>
|
||
<p>ext4slower跟踪ext4文件系统，并计时常见操作，然后只打印超过阈值的操作。这对于识别或证明一种性能问题非常方便：通过文件系统单独显示较慢的磁盘 I/O。磁盘以异步方式处理 I/O，很难将该层的延迟与应用程序所经历的延迟关联起来。在内核堆栈中更高层的追踪，即在 VFS -> 文件系统接口中，会更接近应用程序遭受的延迟。使用此工具来判断文件系统的延迟是否超过了给定的阈值。</p>
|
||
<p>在 bcc 中存在其他文件系统的类似工具：btrfsslower、xfsslower 和 zfsslower。还有一个名为 fileslower 的工具，它在 VFS 层工作并跟踪所有内容（尽管会有更高的开销）。</p>
|
||
<p>更多<a href="https://github.com/iovisor/bcc/tree/master/tools/ext4slower_example.txt">示例</a>。</p>
|
||
<h4 id="14-biolatency"><a class="header" href="#14-biolatency">1.4. biolatency</a></h4>
|
||
<pre><code class="language-sh"># ./biolatency
|
||
跟踪块设备的 I/O... 按 Ctrl-C 结束。
|
||
^C
|
||
微秒 : 数量 分布
|
||
0 -> 1 : 0 | |
|
||
2 -> 3 : 0 | |
|
||
4 -> 7 : 0 | |
|
||
8 -> 15 : 0 | |
|
||
16 -> 31 : 0 | |
|
||
32 -> 63 : 0 | |
|
||
64 -> 127 : 1 | |
|
||
128 -> 255 : 12 |******** |
|
||
256 -> 511 : 15 |********** |
|
||
512 -> 1023 : 43 |******************************* |
|
||
1024 -> 2047 : 52 |**************************************|
|
||
2048 -> 4095 : 47 |********************************** |
|
||
4096 -> 8191 : 52 |**************************************|
|
||
8192 -> 16383 : 36 |************************** |
|
||
16384 -> 32767 : 15 |********** |。32768 -> 65535 : 2 |* |
|
||
65536 -> 131071 : 2 |* |
|
||
</code></pre>
|
||
<p>biolatency跟踪磁盘I/O延迟（从设备执行到完成的时间），当工具结束（Ctrl-C，或给定的间隔）时，它会打印延迟的直方图摘要。</p>
|
||
<p>这对于了解超出iostat等工具提供的平均时间的磁盘I/O延迟非常有用。在分布的末尾将可见I/O延迟的异常值，以及多种模式的分布。</p>
|
||
<p>更多<a href="https://github.com/iovisor/bcc/tree/master/tools/biolatency_example.txt">示例</a>。</p>
|
||
<h4 id="15-biosnoop"><a class="header" href="#15-biosnoop">1.5. biosnoop</a></h4>
|
||
<pre><code class="language-sh"># ./biosnoop
|
||
TIME(s) COMM PID DISK T SECTOR BYTES LAT(ms)
|
||
0.000004001 supervise 1950 xvda1 W 13092560 4096 0.74
|
||
0.000178002 supervise 1950 xvda1 W 13092432 4096 0.61
|
||
0.001469001 supervise 1956 xvda1 W 13092440 4096 1.24
|
||
0.001588002 supervise 1956 xvda1 W 13115128 4096 1.09
|
||
1.022346001 supervise 1950 xvda1 W 13115272 4096 0.98
|
||
1.022568002 supervise 1950 xvda1 W 13188496 4096 0.93
|
||
[...]
|
||
</code></pre>
|
||
<p>biosnoop为每个磁盘I/O打印一行输出，其中包括延迟（从设备执行到完成的时间）等详细信息。</p>
|
||
<p>这让您可以更详细地研究磁盘I/O，并寻找按时间排序的模式（例如，读取在写入后排队）。请注意，如果您的系统以高速率执行磁盘I/O，则输出将冗长。</p>
|
||
<p>更多<a href="https://github.com/iovisor/bcc/tree/master/tools/biosnoop_example.txt">示例</a>。</p>
|
||
<h4 id="16-cachestat"><a class="header" href="#16-cachestat">1.6. cachestat</a></h4>
|
||
<pre><code class="language-sh"># ./cachestat
|
||
HITS MISSES DIRTIES READ_HIT% WRITE_HIT% BUFFERS_MB CACHED_MB
|
||
1074 44 13 94.9% 2.9% 1 223
|
||
2195 170 8 92.5% 6.8% 1 143
|
||
182 53 56 53.6% 1.3% 1 143
|
||
62480 40960 20480 40.6% 19.8% 1 223"。
|
||
格式：仅返回翻译后的内容，不包括原始文本。```
|
||
7 2 5 22.2% 22.2% 1 223
|
||
348 0 0 100.0% 0.0% 1 223
|
||
[...]
|
||
</code></pre>
|
||
<p>cachestat 每秒（或每个自定义时间间隔）打印一行摘要，显示文件系统缓存的统计信息。</p>
|
||
<p>可以用它来识别低缓存命中率和高缺失率，这是性能调优的线索之一。</p>
|
||
<p>更多 <a href="https://github.com/iovisor/bcc/tree/master/tools/cachestat_example.txt">示例</a>。</p>
|
||
<h4 id="17-tcpconnect"><a class="header" href="#17-tcpconnect">1.7. tcpconnect</a></h4>
|
||
<pre><code class="language-sh"># ./tcpconnect
|
||
PID COMM IP SADDR DADDR DPORT
|
||
1479 telnet 4 127.0.0.1 127.0.0.1 23
|
||
1469 curl 4 10.201.219.236 54.245.105.25 80
|
||
1469 curl 4 10.201.219.236 54.67.101.145 80
|
||
1991 telnet 6 ::1 ::1 23
|
||
2015 ssh 6 fe80::2000:bff:fe82:3ac fe80::2000:bff:fe82:3ac 22
|
||
[...]
|
||
</code></pre>
|
||
<p>tcpconnect 每个活动的 TCP 连接（例如通过 connect()）打印一行输出，包括源地址和目标地址的详细信息。</p>
|
||
<p>寻找可能指向应用程序配置问题或入侵者的意外连接。</p>
|
||
<p>更多 <a href="https://github.com/iovisor/bcc/tree/master/tools/tcpconnect_example.txt">示例</a>。</p>
|
||
<h4 id="18-tcpaccept"><a class="header" href="#18-tcpaccept">1.8. tcpaccept</a></h4>
|
||
<pre><code class="language-sh"># ./tcpaccept
|
||
PID COMM IP RADDR LADDR LPORT
|
||
907 sshd 4 192.168.56.1 192.168.56.102 22
|
||
907 sshd 4 127.0.0.1 127.0.0.1 22
|
||
5389 perl 6 1234:ab12:2040:5020:2299:0:5:0 1234:ab12:2040:5020:2299:0:5:0 7001
|
||
[...]
|
||
</code></pre>
|
||
<p>tcpaccept 每个被动的 TCP 连接（例如通过 accept()）打印一行输出，包括源地址和目标地址的详细信息。</p>
|
||
<p>寻找可能指向应用程序配置问题或入侵者的意外连接。</p>
|
||
<p>更多 <a href="https://github.com/iovisor/bcc/tree/master/tools/tcpaccept_example.txt">示例</a>。</p>
|
||
<h4 id="19-tcpretrans"><a class="header" href="#19-tcpretrans">1.9. tcpretrans</a></h4>
|
||
<pre><code class="language-sh"># ./tcpretrans".
|
||
```时间 PID IP LADDR:LPORT T> RADDR:RPORT 状态
|
||
01:55:05 0 4 10.153.223.157:22 R> 69.53.245.40:34619 已建立
|
||
01:55:05 0 4 10.153.223.157:22 R> 69.53.245.40:34619 已建立
|
||
01:55:17 0 4 10.153.223.157:22 R> 69.53.245.40:22957 已建立
|
||
[...]
|
||
</code></pre>
|
||
<p>tcpretrans为每个TCP重传数据包打印一行输出，其中包括源地址、目的地址以及TCP连接的内核状态。</p>
|
||
<p>TCP重传会导致延迟和吞吐量问题。对于已建立的重传，可以查找与网络有关的模式。对于SYN_SENT，可能指向目标内核CPU饱和和内核数据包丢失。</p>
|
||
<p>更多<a href="https://github.com/iovisor/bcc/tree/master/tools/tcpretrans_example.txt">示例</a>。</p>
|
||
<h4 id="110-runqlat"><a class="header" href="#110-runqlat">1.10. runqlat</a></h4>
|
||
<pre><code class="language-sh"># ./runqlat
|
||
跟踪运行队列延迟... 按Ctrl-C结束。
|
||
^C
|
||
微秒数 : 计数 分布
|
||
0 -> 1 : 233 |*********** |
|
||
2 -> 3 : 742 |************************************ |
|
||
4 -> 7 : 203 |********** |
|
||
8 -> 15 : 173 |******** |
|
||
16 -> 31 : 24 |* |
|
||
32 -> 63 : 0 | |
|
||
64 -> 127 : 30 |* |
|
||
128 -> 255 : 6 | |
|
||
256 -> 511 : 3 | |
|
||
512 -> 1023 : 5 | |
|
||
1024 -> 2047 : 27 |* |
|
||
2048 -> 4095 : 30 |* |
|
||
4096 -> 8191 : 20 | |
|
||
8192 -> 16383 : 29 |* |".16384 -> 32767 : 809 |****************************************|
|
||
32768 -> 65535 : 64 |*** |
|
||
</code></pre>
|
||
<p>这可以帮助量化在CPU饱和期间等待获取CPU的时间损失。</p>
|
||
<p>更多<a href="https://github.com/iovisor/bcc/tree/master/tools/runqlat_example.txt">示例</a>。</p>
|
||
<h4 id="111-分析"><a class="header" href="#111-分析">1.11. 分析</a></h4>
|
||
<pre><code class="language-sh"># ./profile
|
||
以每秒49次的频率对所有线程进行采样，包括用户和内核栈...按Ctrl-C结束。
|
||
^C
|
||
00007f31d76c3251 [未知]
|
||
47a2c1e752bf47f7 [未知]
|
||
- sign-file (8877)
|
||
1
|
||
|
||
ffffffff813d0af8 __clear_user
|
||
ffffffff813d5277 iov_iter_zero
|
||
ffffffff814ec5f2 read_iter_zero
|
||
ffffffff8120be9d __vfs_read
|
||
ffffffff8120c385 vfs_read
|
||
ffffffff8120d786 sys_read
|
||
ffffffff817cc076 entry_SYSCALL_64_fastpath
|
||
00007fc5652ad9b0 read
|
||
- dd (25036)
|
||
4
|
||
|
||
0000000000400542 func_a
|
||
0000000000400598 main
|
||
00007f12a133e830 __libc_start_main
|
||
083e258d4c544155 [未知]
|
||
- func_ab (13549)
|
||
5
|
||
|
||
[...]
|
||
|
||
ffffffff8105eb66 native_safe_halt
|
||
ffffffff8103659e default_idle
|
||
ffffffff81036d1f arch_cpu_idle
|
||
ffffffff810bba5a default_idle_call
|
||
ffffffff810bbd07 cpu_startup_entry
|
||
ffffffff8104df55 start_secondary
|
||
- swapper/1 (0)
|
||
75
|
||
</code></pre>
|
||
<p>profile是一个CPU分析工具，它在定时间隔内采样堆栈跟踪，并打印唯一堆栈跟踪的摘要及其出现次数。</p>
|
||
<p>使用此工具来了解消耗CPU资源的代码路径。</p>
|
||
<p>更多<a href="https://github.com/iovisor/bcc/tree/master/tools/profile_example.txt">示例</a>。</p>
|
||
<h3 id="2-使用通用工具进行可观察性"><a class="header" href="#2-使用通用工具进行可观察性">2. 使用通用工具进行可观察性</a></h3>
|
||
<p>除了上述用于性能调整的工具外，下面是一个bcc通用工具的清单，首先是一个列表，然后详细说明：</p>
|
||
<ol>
|
||
<li>trace</li>
|
||
<li>argdist</li>
|
||
<li>funccount这些通用工具可能有助于解决您特定问题的可视化。</li>
|
||
</ol>
|
||
<h4 id="21-跟踪"><a class="header" href="#21-跟踪">2.1. 跟踪</a></h4>
|
||
<h5 id="示例-1"><a class="header" href="#示例-1">示例 1</a></h5>
|
||
<p>假设您想要跟踪文件所有权更改。有三个系统调用，<code>chown</code>、<code>fchown</code>和<code>lchown</code>，用户可以使用它们来更改文件所有权。相应的系统调用入口是<code>SyS_[f|l]chown</code>。可以使用以下命令打印系统调用参数和调用进程的用户ID。您可以使用<code>id</code>命令查找特定用户的UID。</p>
|
||
<pre><code class="language-sh">$ trace.py \
|
||
'p::SyS_chown "file = %s, to_uid = %d, to_gid = %d, from_uid = %d", arg1, arg2, arg3, $uid' \
|
||
'p::SyS_fchown "fd = %d, to_uid = %d, to_gid = %d, from_uid = %d", arg1, arg2, arg3, $uid' \
|
||
'p::SyS_lchown "file = %s, to_uid = %d, to_gid = %d, from_uid = %d", arg1, arg2, arg3, $uid'
|
||
PID TID COMM FUNC -
|
||
1269255 1269255 python3.6 SyS_lchown file = /tmp/dotsync-usisgezu/tmp, to_uid = 128203, to_gid = 100, from_uid = 128203
|
||
1269441 1269441 zstd SyS_chown file = /tmp/dotsync-vic7ygj0/dotsync-package.zst, to_uid = 128203, to_gid = 100, from_uid = 128203
|
||
1269255 1269255 python3.6 SyS_lchown file = /tmp/dotsync-a40zd7ev/tmp, to_uid = 128203, to_gid = 100, from_uid = 128203
|
||
1269442 1269442 zstd SyS_chown file = /tmp/dotsync-gzp413o_/dotsync-package.zst, to_uid = 128203, to_gid = 100, from_uid = 128203
|
||
1269255 1269255 python3.6 SyS_lchown file = /tmp/dotsync-whx4fivm/tmp/.bash_profile, to_uid = 128203, to_gid = 100, from_uid = 128203
|
||
</code></pre>
|
||
<h5 id="示例-2"><a class="header" href="#示例-2">示例 2</a></h5>
|
||
<p>假设您想要统计基于bpf的性能监控工具中的非自愿上下文切换（<code>nvcsw</code>），而您不知道正确的方法是什么。<code>/proc/<pid>/status</code>已经告诉您进程的非自愿上下文切换（<code>nonvoluntary_ctxt_switches</code>）的数量，并且您可以使用<code>trace.py</code>进行快速实验以验证您的方法。根据内核源代码，<code>nvcsw</code>在文件<code>linux/kernel/sched/core.c</code>的<code>__schedule</code>函数中计数，并满足以下条件：</p>
|
||
<pre><code class="language-c">.!(!preempt && prev->state) // 即 preempt || !prev->state
|
||
</code></pre>
|
||
<p><code>__schedule</code> 函数被标记为 <code>notrace</code> ，评估上述条件的最佳位置似乎在函数 <code>__schedule</code> 内部的 <code>sched/sched_switch</code> 跟踪点中，并且在 <code>linux/include/trace/events/sched.h</code> 中定义。<code>trace.py</code> 已经将 <code>args</code> 设置为跟踪点 <code>TP_STRUCT__entry</code> 的指针。函数 <code>__schedule</code> 中的上述条件可以表示为</p>
|
||
<pre><code class="language-c">args->prev_state == TASK_STATE_MAX || args->prev_state == 0
|
||
</code></pre>
|
||
<p>可以使用以下命令来计算非自愿上下文切换（每个进程或每个进程ID），并与 <code>/proc/<pid>/status</code> 或 <code>/proc/<pid>/task/<task_id>/status</code> 进行比较，以确保正确性，因为在典型情况下，非自愿上下文切换并不常见。</p>
|
||
<pre><code class="language-sh">$ trace.py -p 1134138 't:sched:sched_switch (args->prev_state == TASK_STATE_MAX || args->prev_state == 0)'
|
||
PID TID COMM FUNC
|
||
1134138 1134140 contention_test sched_switch
|
||
1134138 1134142 contention_test sched_switch
|
||
...
|
||
$ trace.py -L 1134140 't:sched:sched_switch (args->prev_state == TASK_STATE_MAX || args->prev_state == 0)'
|
||
PID TID COMM FUNC
|
||
1134138 1134140 contention_test sched_switch
|
||
1134138 1134140 contention_test sched_switch
|
||
...
|
||
</code></pre>
|
||
<h5 id="示例-3"><a class="header" href="#示例-3">示例 3</a></h5>
|
||
<p>此示例与问题 <a href="https://github.com/iovisor/bcc/issues/1231">1231</a> 和 <a href="https://github.com/iovisor/bcc/issues/1516">1516</a> 相关，其中在某些情况下，uprobes 完全无法工作。首先，你可以执行以下 <code>strace</code></p>
|
||
<pre><code class="language-sh">$ strace trace.py 'r:bash:readline "%s", retval'
|
||
...
|
||
perf_event_open(0x7ffd968212f0, -1, 0, -1, 0x8 /* PERF_FLAG_??? */) = -1 EIO (Input/output error)
|
||
...
|
||
</code></pre>
|
||
<p><code>perf_event_open</code>系统调用返回<code>-EIO</code>。在<code>/kernel/trace</code>和<code>/kernel/events</code>目录中查找与<code>EIO</code>相关的内核uprobe代码，函数<code>uprobe_register</code>最可疑。让我们找出是否调用了这个函数，如果调用了，返回值是什么。在一个终端中使用以下命令打印出<code>uprobe_register</code>的返回值：</p>
|
||
<pre><code class="language-sh">trace.py 'r::uprobe_register "ret = %d", retval'
|
||
</code></pre>
|
||
<p>在另一个终端中运行相同的bash uretprobe跟踪示例，您应该得到：</p>
|
||
<pre><code class="language-sh">$ trace.py 'r::uprobe_register "ret = %d", retval'
|
||
PID TID COMM FUNC -
|
||
1041401 1041401 python2.7 uprobe_register ret = -5
|
||
</code></pre>
|
||
<p>错误代码<code>-5</code>是EIO。这证实了函数<code>uprobe_register</code>中的以下代码是最可疑的罪魁祸首。</p>
|
||
<pre><code class="language-c"> if (!inode->i_mapping->a_ops->readpage && !shmem_mapping(inode->i_mapping))
|
||
return -EIO;
|
||
</code></pre>
|
||
<p><code>shmem_mapping</code>函数定义如下：</p>
|
||
<pre><code class="language-c">bool shmem_mapping(struct address_space *mapping)
|
||
{
|
||
return mapping->a_ops == &shmem_aops;
|
||
}
|
||
</code></pre>
|
||
<p>为了确认这个理论，使用以下命令找出<code>inode->i_mapping->a_ops</code>的值：</p>
|
||
<pre><code class="language-sh">$ trace.py -I 'linux/fs.h' 'p::uprobe_register(struct inode *inode) "a_ops = %llx", inode->i_mapping->a_ops'
|
||
PID TID COMM FUNC -
|
||
814288 814288 python2.7 uprobe_register a_ops = ffffffff81a2adc0
|
||
^C$ grep ffffffff81a2adc0 /proc/kallsyms
|
||
ffffffff81a2adc0 R empty_aops
|
||
</code></pre>
|
||
<p>内核符号<code>empty_aops</code>没有定义<code>readpage</code>，因此上述可疑条件为真。进一步检查内核源代码显示，<code>overlayfs</code>没有提供自己的<code>a_ops</code>，而其他一些文件系统（例如ext4）定义了自己的<code>a_ops</code>（例如<code>ext4_da_aops</code>），并且<code>ext4_da_aops</code>定义了<code>readpage</code>。因此，uprobe对于ext4正常工作，但在overlayfs上不正常工作。</p>
|
||
<p>更多<a href="https://github.com/iovisor/bcc/tree/master/tools/trace_example.txt">示例</a>。</p>
|
||
<h4 id="22-argdist更多示例"><a class="header" href="#22-argdist更多示例">2.2. argdist"。更多<a href="https://github.com/iovisor/bcc/tree/master/tools/argdist_example.txt">示例</a></a></h4>
|
||
<h4 id="23-funccount"><a class="header" href="#23-funccount">2.3. funccount</a></h4>
|
||
<p>更多<a href="https://github.com/iovisor/bcc/tree/master/tools/funccount_example.txt">示例</a>.</p>
|
||
<h2 id="网络"><a class="header" href="#网络">网络</a></h2>
|
||
<p>To do.</p>
|
||
|
||
</main>
|
||
|
||
<nav class="nav-wrapper" aria-label="Page navigation">
|
||
<!-- Mobile navigation buttons -->
|
||
<a rel="prev" href="../bcc-documents/special_filtering.html" class="mobile-nav-chapters previous" title="Previous chapter" aria-label="Previous chapter" aria-keyshortcuts="Left">
|
||
<i class="fa fa-angle-left"></i>
|
||
</a>
|
||
|
||
<a rel="next" href="../bcc-documents/tutorial_bcc_python_developer.html" class="mobile-nav-chapters next" title="Next chapter" aria-label="Next chapter" aria-keyshortcuts="Right">
|
||
<i class="fa fa-angle-right"></i>
|
||
</a>
|
||
|
||
<div style="clear: both"></div>
|
||
</nav>
|
||
</div>
|
||
</div>
|
||
|
||
<nav class="nav-wide-wrapper" aria-label="Page navigation">
|
||
<a rel="prev" href="../bcc-documents/special_filtering.html" class="nav-chapters previous" title="Previous chapter" aria-label="Previous chapter" aria-keyshortcuts="Left">
|
||
<i class="fa fa-angle-left"></i>
|
||
</a>
|
||
|
||
<a rel="next" href="../bcc-documents/tutorial_bcc_python_developer.html" class="nav-chapters next" title="Next chapter" aria-label="Next chapter" aria-keyshortcuts="Right">
|
||
<i class="fa fa-angle-right"></i>
|
||
</a>
|
||
</nav>
|
||
|
||
</div>
|
||
|
||
|
||
|
||
|
||
<script>
|
||
window.playground_copyable = true;
|
||
</script>
|
||
|
||
|
||
<script src="../elasticlunr.min.js"></script>
|
||
<script src="../mark.min.js"></script>
|
||
<script src="../searcher.js"></script>
|
||
|
||
<script src="../clipboard.min.js"></script>
|
||
<script src="../highlight.js"></script>
|
||
<script src="../book.js"></script>
|
||
|
||
<!-- Custom JS scripts -->
|
||
|
||
|
||
</div>
|
||
</body>
|
||
</html>
|