【ebpf】first try2

虚拟机基本环境信息

1
2
3
4
5
6
7
8
9
10
11
12
13
[root@ubuntu-22 ~]# uname -a
Linux ubuntu-22 5.15.0-58-generic #64-Ubuntu SMP Thu Jan 5 11:43:13 UTC 2023 x86_64 x86_64 x86_64 GNU/Linux

[root@ubuntu-22 ~]# uname -r
5.15.0-58-generic

[root@ubuntu-22 ~]# lsb_release -a
No LSB modules are available.
Distributor ID: Ubuntu
Description:    Ubuntu 22.04.1 LTS
Release:        22.04
Codename:       jammy

搭建ubuntu虚拟机可以看到我的另一篇教程：
# 【网络成长记】VMware16.2.2中创建ubuntu22.04.1虚拟机+修改静态ip+换源+mobaxterm远程连接+修改命令提示符颜色

安装 eBPF 开发和运行所需要的开发工具

1
[root@ubuntu-22 ~]# sudo apt-get install -y make clang llvm libelf-dev libbpf-dev bpfcc-tools libbpfcc-dev linux-tools-$(uname -r) linux-headers-$(uname -r)

ebpf的CO-RE特性前提条件

在使用 CO-RE 之前，内核需要开启 CONFIG_DEBUG_INFO_BTF=y 和 CONFIG_DEBUG_INFO=y 这两个编译选项。

1
2
3
4
5
6
7
8
9
10
11
12
13
14
[root@ubuntu-22 ~]# cat /boot/config-5.15.0-58-generic  |grep -i config_debug_info
CONFIG_DEBUG_INFO=y
# CONFIG_DEBUG_INFO_REDUCED is not set
# CONFIG_DEBUG_INFO_COMPRESSED is not set
# CONFIG_DEBUG_INFO_SPLIT is not set
# CONFIG_DEBUG_INFO_DWARF_TOOLCHAIN_DEFAULT is not set
# CONFIG_DEBUG_INFO_DWARF4 is not set
CONFIG_DEBUG_INFO_DWARF5=y
CONFIG_DEBUG_INFO_BTF=y
CONFIG_DEBUG_INFO_BTF_MODULES=y
[root@ubuntu-22 ~]#
[root@ubuntu-22 ~]# ll /sys/kernel/btf/vmlinux
-r--r--r-- 1 root root 5178563  1月 29 16:02 /sys/kernel/btf/vmlinux

第一个ebpf程序：跟踪 openat()（即打开文件）系统调用

第一步：使用 C 开发一个 eBPF 程序

1
2
3
4
5
6
7
8
9
[root@ubuntu-22 geektime]# cat hello.c
int hello_world(void *ctx)
{
    bpf_trace_printk("Hello, World!");  // 最常用的 BPF 辅助函数，它的作用是输出一段字符串。不过，由于 eBPF 运行在内核中，它的输出并不是通常的标准输出（stdout），而是内核调试文件 /sys/kernel/debug/tracing/trace_pipe
    return 0;
}

// eBPF程序并不能随便的调用内核函数，必须通过辅助函数才可完成eBPF程序和其他内核模块的交互，eg bpf_trace_printk()

第二步：使用 Python 和 BCC 库开发一个用户态程序

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
[root@ubuntu-22 geektime]# cat hello.py
#!/usr/bin/env python3
# 1) import bcc library
# 导入了 BCC  库的 BPF 模块，以便接下来调用；
from bcc import BPF

# 2) load BPF program
# 调用 BPF() 加载第一步开发的 BPF 源代码；
b = BPF(src_file="hello.c")

# 3) attach kprobe
# 将 BPF 程序挂载到内核探针（简称 kprobe），其中 do_sys_openat2() 是系统调用 openat() 在内核中的实现；
# 调用了 attach_kprobe 函数，绑定了一个内核跟踪事件
b.attach_kprobe(event="do_sys_openat2", fn_name="hello_world")


# 4) read and print /sys/kernel/debug/tracing/trace_pipe
# 读取内核调试文件 /sys/kernel/debug/tracing/trace_pipe 的内容，并打印到标准输出中。
b.trace_print()


# 内核函数 do_sys_openat2

# 理解：opennat()和opennat2()都是系统调用
# 在计算机中运行程序、读写文件，都会涉及到文件的打开操作，Linux v5.10 与文件打开相关的系统调用有 open() / creat() / openat() / openat2这四类，在使用 glibc v2.32 时，几乎所有的文件打开操作使用的都是 openat2() 这个系统调用。

# 用高级语言开发的 eBPF 程序，需要首先编译为 BPF 字节码，然后借助 bpf 系统调用加载到内核中，最后再通过性能监控等接口与具体的内核事件进行绑定。这样，内核的性能监控模块才会在内核事件发生时，自动执行我们开发的 eBPF 程序。

第三步：执行 eBPF 程序

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
[root@ubuntu-22 geektime]# sudo python3 hello.py
In file included from <built-in>:2:
In file included from /virtual/include/bcc/bpf.h:12:
In file included from include/linux/types.h:6:
In file included from include/uapi/linux/types.h:14:
In file included from include/uapi/linux/posix_types.h:5:
In file included from include/linux/stddef.h:5:
In file included from include/uapi/linux/stddef.h:5:
In file included from include/linux/compiler_types.h:80:
include/linux/compiler-clang.h:41:9: warning: '__HAVE_BUILTIN_BSWAP32__' macro redefined [-Wmacro-redefined]
#define __HAVE_BUILTIN_BSWAP32__
        ^
<command line>:4:9: note: previous definition is here
#define __HAVE_BUILTIN_BSWAP32__ 1
        ^
In file included from <built-in>:2:
In file included from /virtual/include/bcc/bpf.h:12:
In file included from include/linux/types.h:6:
In file included from include/uapi/linux/types.h:14:
In file included from include/uapi/linux/posix_types.h:5:
In file included from include/linux/stddef.h:5:
In file included from include/uapi/linux/stddef.h:5:
In file included from include/linux/compiler_types.h:80:
include/linux/compiler-clang.h:42:9: warning: '__HAVE_BUILTIN_BSWAP64__' macro redefined [-Wmacro-redefined]
#define __HAVE_BUILTIN_BSWAP64__
        ^
<command line>:5:9: note: previous definition is here
#define __HAVE_BUILTIN_BSWAP64__ 1
        ^
In file included from <built-in>:2:
In file included from /virtual/include/bcc/bpf.h:12:
In file included from include/linux/types.h:6:
In file included from include/uapi/linux/types.h:14:
In file included from include/uapi/linux/posix_types.h:5:
In file included from include/linux/stddef.h:5:
In file included from include/uapi/linux/stddef.h:5:
In file included from include/linux/compiler_types.h:80:
include/linux/compiler-clang.h:43:9: warning: '__HAVE_BUILTIN_BSWAP16__' macro redefined [-Wmacro-redefined]
#define __HAVE_BUILTIN_BSWAP16__
        ^
<command line>:3:9: note: previous definition is here
#define __HAVE_BUILTIN_BSWAP16__ 1
        ^
3 warnings generated.
b'    systemd-oomd-636     [001] d...1  7270.693212: bpf_trace_printk: Hello, World!'
b'    systemd-oomd-636     [001] d...1  7270.693407: bpf_trace_printk: Hello, World!'
b'            head-43243   [000] d...1  7271.246610: bpf_trace_printk: Hello, World!'
b'            head-43243   [000] d...1  7271.246653: bpf_trace_printk: Hello, World!'
b'            head-43243   [000] d...1  7271.246911: bpf_trace_printk: Hello, World!'
b'            head-43243   [000] d...1  7271.247487: bpf_trace_printk: Hello, World!'
b'            head-43243   [000] d...1  7271.247529: bpf_trace_printk: Hello, World!'
b'           <...>-43244   [000] d...1  7271.248757: bpf_trace_printk: Hello, World!'
b'           <...>-43244   [000] d...1  7271.248848: bpf_trace_printk: Hello, World!'
b'           <...>-43245   [000] d...1  7271.249767: bpf_trace_printk: Hello, World!'
b'           <...>-43245   [000] d...1  7271.249825: bpf_trace_printk: Hello, World!'
b'             who-43246   [000] d...1  7271.251964: bpf_trace_printk: Hello, World!'
b'             who-43246   [000] d...1  7271.252028: bpf_trace_printk: Hello, World!'
b'           <...>-43247   [000] d...1  7271.253499: bpf_trace_printk: Hello, World!'
b'           <...>-43247   [000] d...1  7271.253554: bpf_trace_printk: Hello, World!'
b'    systemd-oomd-636     [001] d...1  7271.943581: bpf_trace_printk: Hello, World!'
b'    systemd-oomd-636     [001] d...1  7272.943654: bpf_trace_printk: Hello, World!'
b'            head-43255   [000] d...1  7273.267842: bpf_trace_printk: Hello, World!'
b'            head-43255   [000] d...1  7273.268730: bpf_trace_printk: Hello, World!'
b'            tail-43256   [000] d...1  7273.272320: bpf_trace_printk: Hello, World!'
b'            tail-43256   [000] d...1  7273.272921: bpf_trace_printk: Hello, World!'
b'            tail-43256   [000] d...1  7273.273573: bpf_trace_printk: Hello, World!'
b'            tail-43256   [000] d...1  7273.274131: bpf_trace_printk: Hello, World!'


改进

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
[root@ubuntu-22 geektime]# cat trace-open.c
// 包含头文件
#include <uapi/linux/openat2.h>
#include <linux/sched.h>

// 定义数据结构
struct data_t {
  u32 pid;
  u64 ts;
  char comm[TASK_COMM_LEN];
  char fname[NAME_MAX];
};

// 定义性能事件映射  定义一个 Perf 事件类型的 BPF 映射
BPF_PERF_OUTPUT(events);



// 定义kprobe处理函数
int hello_world(struct pt_regs *ctx, int dfd, const char __user * filename, struct open_how *how)
{
  struct data_t data = { };

  // 获取PID和时间
  data.pid = bpf_get_current_pid_tgid();  // bpf_get_current_pid_tgid用于获取进程的 TGID 和 PID。因为这儿定义的 data.pid 数据类型为 u32，所以高 32 位舍弃掉后就是进程的 PID
  data.ts = bpf_ktime_get_ns();  // bpf_ktime_get_ns用于获取系统自启动以来的时间，单位是纳秒

  // 获取进程名
  if (bpf_get_current_comm(&data.comm, sizeof(data.comm)) == 0)  // bpf_get_current_comm用于获取进程名，并把进程名复制到预定义的缓冲区中
  {
    bpf_probe_read(&data.fname, sizeof(data.fname), (void *)filename); // bpf_probe_read 用于从指定指针处读取固定大小的数据，这里则用于读取进程打开的文件名。
  }

  // 提交性能事件 调用 perf_submit() 把数据提交到刚才定义的 BPF 映射
  events.perf_submit(ctx, &data, sizeof(data));
  return 0;
}

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
[root@ubuntu-22 geektime]# cat trace-open.py
from bcc import BPF

# 1) load BPF program
# 加载 eBPF 程序并挂载到内核探针上
b = BPF(src_file="trace-open.c")
b.attach_kprobe(event="do_sys_openat2", fn_name="hello_world")

# 2) print header
# 输出一行 Header 字符串表示数据的格式
print("%-18s %-16s %-6s %-16s" % ("TIME(s)", "COMM", "PID", "FILE"))

# 3) define the callback for perf event
# print_event 定义一个数据处理的回调函数，打印进程的名字、PID 以及它调用 openat 时打开的文件
start = 0
def print_event(cpu, data, size):
    global start
    event = b["events"].event(data)
    if start == 0:
            start = event.ts
    time_s = (float(event.ts - start)) / 1000000000
    print("%-18.9f %-16s %-6d %-16s" % (time_s, event.comm, event.pid, event.fname))

# 4) loop with callback to print_event
# open_perf_buffer 定义了名为 “events” 的 Perf 事件映射，而后通过一个循环调用 perf_buffer_poll 读取映射的内容，并执行回调函数输出进程信息
b["events"].open_perf_buffer(print_event)
while 1:
    try:
        b.perf_buffer_poll()
    except KeyboardInterrupt:
        exit()


# 怎样从用户态读取 BPF 映射内容并输出到标准输出（stdout）呢？
# 在 BCC 中，与 eBPF 程序中 BPF_PERF_OUTPUT  相对应的用户态辅助函数是 open_perf_buffer() 。它需要传入一个回调函数，用于处理从 Perf 事件类型的 BPF 映射中读取到的数据。
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
[root@ubuntu-22 geektime]# sudo python3 trace-open.py
In file included from <built-in>:2:
In file included from /virtual/include/bcc/bpf.h:12:
In file included from include/linux/types.h:6:
In file included from include/uapi/linux/types.h:14:
In file included from include/uapi/linux/posix_types.h:5:
In file included from include/linux/stddef.h:5:
In file included from include/uapi/linux/stddef.h:5:
In file included from include/linux/compiler_types.h:80:
include/linux/compiler-clang.h:41:9: warning: '__HAVE_BUILTIN_BSWAP32__' macro redefined [-Wmacro-redefined]
#define __HAVE_BUILTIN_BSWAP32__
        ^
<command line>:4:9: note: previous definition is here
#define __HAVE_BUILTIN_BSWAP32__ 1
        ^
In file included from <built-in>:2:
In file included from /virtual/include/bcc/bpf.h:12:
In file included from include/linux/types.h:6:
In file included from include/uapi/linux/types.h:14:
In file included from include/uapi/linux/posix_types.h:5:
In file included from include/linux/stddef.h:5:
In file included from include/uapi/linux/stddef.h:5:
In file included from include/linux/compiler_types.h:80:
include/linux/compiler-clang.h:42:9: warning: '__HAVE_BUILTIN_BSWAP64__' macro redefined [-Wmacro-redefined]
#define __HAVE_BUILTIN_BSWAP64__
        ^
<command line>:5:9: note: previous definition is here
#define __HAVE_BUILTIN_BSWAP64__ 1
        ^
In file included from <built-in>:2:
In file included from /virtual/include/bcc/bpf.h:12:
In file included from include/linux/types.h:6:
In file included from include/uapi/linux/types.h:14:
In file included from include/uapi/linux/posix_types.h:5:
In file included from include/linux/stddef.h:5:
In file included from include/uapi/linux/stddef.h:5:
In file included from include/linux/compiler_types.h:80:
include/linux/compiler-clang.h:43:9: warning: '__HAVE_BUILTIN_BSWAP16__' macro redefined [-Wmacro-redefined]
#define __HAVE_BUILTIN_BSWAP16__
        ^
<command line>:3:9: note: previous definition is here
#define __HAVE_BUILTIN_BSWAP16__ 1
        ^
3 warnings generated.
TIME(s)            COMM             PID    FILE
0.000000000        b'systemd-oomd'  636    b'/proc/meminfo'
0.249586834        b'systemd-oomd'  636    b'/sys/fs/cgroup/user.slice/user-0.slice/user@0.service/memory.pressure'
0.249744243        b'systemd-oomd'  636    b'/sys/fs/cgroup/user.slice/user-0.slice/user@0.service/memory.current'
0.249765844        b'systemd-oomd'  636    b'/sys/fs/cgroup/user.slice/user-0.slice/user@0.service/memory.min'
0.249783697        b'systemd-oomd'  636    b'/sys/fs/cgroup/user.slice/user-0.slice/user@0.service/memory.low'
0.249800399        b'systemd-oomd'  636    b'/sys/fs/cgroup/user.slice/user-0.slice/user@0.service/memory.swap.current'
0.249818063        b'systemd-oomd'  636    b'/sys/fs/cgroup/user.slice/user-0.slice/user@0.service/memory.stat'
0.249918042        b'systemd-oomd'  636    b'/sys/fs/cgroup/user.slice/user-1000.slice/user@1000.service/memory.pressure'
0.249993365        b'systemd-oomd'  636    b'/sys/fs/cgroup/user.slice/user-1000.slice/user@1000.service/memory.current'
0.250011319        b'systemd-oomd'  636    b'/sys/fs/cgroup/user.slice/user-1000.slice/user@1000.service/memory.min'
0.250028852        b'systemd-oomd'  636    b'/sys/fs/cgroup/user.slice/user-1000.slice/user@1000.service/memory.low'
0.250045734        b'systemd-oomd'  636    b'/sys/fs/cgroup/user.slice/user-1000.slice/user@1000.service/memory.swap.current'
0.250062576        b'systemd-oomd'  636    b'/sys/fs/cgroup/user.slice/user-1000.slice/user@1000.service/memory.stat'
0.250117370        b'systemd-oomd'  636    b'/proc/meminfo'
0.499947185        b'systemd-oomd'  636    b'/proc/meminfo'
0.537863286        b'head'          53569  b'/etc/ld.so.cache'
0.537898282        b'head'          53569  b'/lib/x86_64-linux-gnu/libc.so.6'
0.538648765        b'head'          53569  b'/usr/lib/locale/locale-archive'
0.538766849        b'head'          53569  b'/usr/share/locale/locale.alias'
0.538832774        b'head'          53569  b'/usr/share/locale/en_US/LC_MESSAGES/coreutils.mo'
0.538838585        b'head'          53569  b'/usr/share/locale/en/LC_MESSAGES/coreutils.mo'
0.538844537        b'head'          53569  b'/usr/share/locale-langpack/en_US/LC_MESSAGES/coreutils.mo'
0.538850207        b'head'          53569  b'/usr/share/locale-langpack/en/LC_MESSAGES/coreutils.mo'
0.538868322        b'head'          53569  b'/proc/meminfo'
0.539986006        b'head'          53570  b'/etc/ld.so.cache'
0.540310641        b'head'          53570  b'/lib/x86_64-linux-gnu/libc.so.6'
0.541208063        b'head'          53570  b'/usr/lib/locale/locale-archive'
0.541523040        b'head'          53570  b'/usr/share/locale/locale.alias'
0.541664007        b'head'          53570  b'/usr/share/locale/en_US/LC_MESSAGES/coreutils.mo'
0.541768956        b'head'          53570  b'/usr/share/locale/en/LC_MESSAGES/coreutils.mo'
0.541841113        b'head'          53570  b'/usr/share/locale-langpack/en_US/LC_MESSAGES/coreutils.mo'
0.541911206        b'head'          53570  b'/usr/share/locale-langpack/en/LC_MESSAGES/coreutils.mo'
0.541993672        b'head'          53570  b'/proc/stat'
0.542297589        b'head'          53570  b'/proc/version'
0.542427826        b'head'          53570  b'/proc/uptime'
0.542513568        b'head'          53570  b'/proc/loadavg'
0.542576598        b'head'          53570  b'/proc/sys/fs/file-nr'
0.542649275        b'head'          53570  b'/proc/sys/kernel/hostname'

ebpf
【ebpf】first try2
http://example.com/2023/01/29/ebpf/【ebpf】first try2/
作者
ningan123
发布于
2023年1月29日
许可协议