From 7b8b4545713865c1a5ca16a1b27a10e2802c2745 Mon Sep 17 00:00:00 2001 From: yunwei37 <1067852565@qq.com> Date: Wed, 14 Dec 2022 14:23:49 +0800 Subject: [PATCH 01/10] add eunomia-bpf as submodule --- .gitmodules | 3 +++ third/eunomia-bpf | 1 + 2 files changed, 4 insertions(+) create mode 100644 .gitmodules create mode 160000 third/eunomia-bpf diff --git a/.gitmodules b/.gitmodules new file mode 100644 index 0000000..54e17ea --- /dev/null +++ b/.gitmodules @@ -0,0 +1,3 @@ +[submodule "third/eunomia-bpf"] + path = third/eunomia-bpf + url = https://github.com/eunomia-bpf/eunomia-bpf diff --git a/third/eunomia-bpf b/third/eunomia-bpf new file mode 160000 index 0000000..ad470dd --- /dev/null +++ b/third/eunomia-bpf @@ -0,0 +1 @@ +Subproject commit ad470dd8f6ad73f05196a160ffb9eb785406bc25 -- Gitee From 11efd9f1cc638a8eb081e7af2b486f64cdc70159 Mon Sep 17 00:00:00 2001 From: yunwei37 <1067852565@qq.com> Date: Wed, 14 Dec 2022 14:27:01 +0800 Subject: [PATCH 02/10] libbpf: Do not use btf_dump__new macro for c++ objects see https://lore.kernel.org/bpf/5a60e93c-26e4-d35a-b849-e09d002c4c40@fb.com/t/#u and https://github.com/libbpf/libbpf/issues/283 Fix this so that we can compile with eunomia-bpf. --- third/libbpf/src/btf.h | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/third/libbpf/src/btf.h b/third/libbpf/src/btf.h index 742a2bf..061839f 100644 --- a/third/libbpf/src/btf.h +++ b/third/libbpf/src/btf.h @@ -313,12 +313,18 @@ LIBBPF_API struct btf_dump *btf_dump__new_deprecated(const struct btf *btf, * * The rest works just like in case of ___libbpf_override() usage with symbol * versioning. + * + * C++ compilers don't support __builtin_types_compatible_p(), so at least + * don't screw up compilation for them and let C++ users pick btf_dump__new + * vs btf_dump__new_deprecated explicitly. */ +#ifndef __cplusplus #define btf_dump__new(a1, a2, a3, a4) __builtin_choose_expr( \ __builtin_types_compatible_p(typeof(a4), btf_dump_printf_fn_t) || \ __builtin_types_compatible_p(typeof(a4), void(void *, const char *, va_list)), \ btf_dump__new_deprecated((void *)a1, (void *)a2, (void *)a3, (void *)a4), \ btf_dump__new((void *)a1, (void *)a2, (void *)a3, (void *)a4)) +#endif LIBBPF_API void btf_dump__free(struct btf_dump *d); -- Gitee From 6ccbca51700ff837de65b06e481d140de5f02c3c Mon Sep 17 00:00:00 2001 From: yunwei37 <1067852565@qq.com> Date: Wed, 14 Dec 2022 16:13:12 +0800 Subject: [PATCH 03/10] add build Makefile for eunomia-lcc --- lcc/eunomia-lcc/Makefile | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) create mode 100644 lcc/eunomia-lcc/Makefile diff --git a/lcc/eunomia-lcc/Makefile b/lcc/eunomia-lcc/Makefile new file mode 100644 index 0000000..591204a --- /dev/null +++ b/lcc/eunomia-lcc/Makefile @@ -0,0 +1,19 @@ +.PHONY: all build +all: build + +EUNOMIA_SOURCE := ../../third/eunomia-bpf +LIBBPF_SRC := $(abspath ../../third/libbpf/src) + +ecli: + cd $(EUNOMIA_SOURCE)/ecli && \ + cmake -Bbuild -DCMAKE_INSTALL_PREFIX=$(INSTALL_LOCATION)\ + -DCMAKE_BUILD_TYPE=Release\ + -DLIBBPF_SOURCE_DIR=$(LIBBPF_SRC) + cd $(EUNOMIA_SOURCE)/ecli && cmake --build build --config Release + cp $(EUNOMIA_SOURCE)/ecli/build/bin/Release/ecli ecli + +ecc: + cd $(EUNOMIA_SOURCE)/eunomia-cc && make + cp $(EUNOMIA_SOURCE)/eunomia-cc/workspace/bin/ecc ecc + +build: ecli ecc -- Gitee From 2f6694349a3e8d9fe56c6bfbe639d671dead0e54 Mon Sep 17 00:00:00 2001 From: yunwei37 <1067852565@qq.com> Date: Wed, 14 Dec 2022 16:14:01 +0800 Subject: [PATCH 04/10] add example sigsnoop for eunomia-lcc --- lcc/eunomia-lcc/.gitignore | 4 + lcc/eunomia-lcc/example/sigsnoop.bpf.c | 145 +++++++++++++++++++++++++ lcc/eunomia-lcc/example/sigsnoop.h | 16 +++ 3 files changed, 165 insertions(+) create mode 100644 lcc/eunomia-lcc/.gitignore create mode 100644 lcc/eunomia-lcc/example/sigsnoop.bpf.c create mode 100644 lcc/eunomia-lcc/example/sigsnoop.h diff --git a/lcc/eunomia-lcc/.gitignore b/lcc/eunomia-lcc/.gitignore new file mode 100644 index 0000000..c211399 --- /dev/null +++ b/lcc/eunomia-lcc/.gitignore @@ -0,0 +1,4 @@ +ecli +ecc +example/*.o +example/*.json diff --git a/lcc/eunomia-lcc/example/sigsnoop.bpf.c b/lcc/eunomia-lcc/example/sigsnoop.bpf.c new file mode 100644 index 0000000..e03981f --- /dev/null +++ b/lcc/eunomia-lcc/example/sigsnoop.bpf.c @@ -0,0 +1,145 @@ +// SPDX-License-Identifier: (LGPL-2.1 OR BSD-2-Clause) +/* Copyright (c) 2021~2022 Hengqi Chen */ +#include +#include +#include "sigsnoop.h" + +#define MAX_ENTRIES 10240 + +const volatile pid_t filtered_pid = 0; +const volatile int target_signal = 0; +const volatile bool failed_only = false; + +struct { + __uint(type, BPF_MAP_TYPE_HASH); + __uint(max_entries, MAX_ENTRIES); + __type(key, __u32); + __type(value, struct event); +} values SEC(".maps"); + +struct { + __uint(type, BPF_MAP_TYPE_PERF_EVENT_ARRAY); + __uint(key_size, sizeof(__u32)); + __uint(value_size, sizeof(__u32)); +} events SEC(".maps"); + +static int probe_entry(pid_t tpid, int sig) +{ + struct event event = {}; + __u64 pid_tgid; + __u32 pid, tid; + + if (target_signal && sig != target_signal) + return 0; + + pid_tgid = bpf_get_current_pid_tgid(); + pid = pid_tgid >> 32; + tid = (__u32)pid_tgid; + if (filtered_pid && pid != filtered_pid) + return 0; + + event.pid = pid; + event.tpid = tpid; + event.sig = sig; + bpf_get_current_comm(event.comm, sizeof(event.comm)); + bpf_map_update_elem(&values, &tid, &event, BPF_ANY); + return 0; +} + +static int probe_exit(void *ctx, int ret) +{ + __u64 pid_tgid = bpf_get_current_pid_tgid(); + __u32 tid = (__u32)pid_tgid; + struct event *eventp; + + eventp = bpf_map_lookup_elem(&values, &tid); + if (!eventp) + return 0; + + if (failed_only && ret >= 0) + goto cleanup; + + eventp->ret = ret; + bpf_perf_event_output(ctx, &events, BPF_F_CURRENT_CPU, eventp, sizeof(*eventp)); + +cleanup: + bpf_map_delete_elem(&values, &tid); + return 0; +} + +SEC("tracepoint/syscalls/sys_enter_kill") +int kill_entry(struct trace_event_raw_sys_enter *ctx) +{ + pid_t tpid = (pid_t)ctx->args[0]; + int sig = (int)ctx->args[1]; + + return probe_entry(tpid, sig); +} + +SEC("tracepoint/syscalls/sys_exit_kill") +int kill_exit(struct trace_event_raw_sys_exit *ctx) +{ + return probe_exit(ctx, ctx->ret); +} + +SEC("tracepoint/syscalls/sys_enter_tkill") +int tkill_entry(struct trace_event_raw_sys_enter *ctx) +{ + pid_t tpid = (pid_t)ctx->args[0]; + int sig = (int)ctx->args[1]; + + return probe_entry(tpid, sig); +} + +SEC("tracepoint/syscalls/sys_exit_tkill") +int tkill_exit(struct trace_event_raw_sys_exit *ctx) +{ + return probe_exit(ctx, ctx->ret); +} + +SEC("tracepoint/syscalls/sys_enter_tgkill") +int tgkill_entry(struct trace_event_raw_sys_enter *ctx) +{ + pid_t tpid = (pid_t)ctx->args[1]; + int sig = (int)ctx->args[2]; + + return probe_entry(tpid, sig); +} + +SEC("tracepoint/syscalls/sys_exit_tgkill") +int tgkill_exit(struct trace_event_raw_sys_exit *ctx) +{ + return probe_exit(ctx, ctx->ret); +} + +SEC("tracepoint/signal/signal_generate") +int sig_trace(struct trace_event_raw_signal_generate *ctx) +{ + struct event event = {}; + pid_t tpid = ctx->pid; + int ret = ctx->errno; + int sig = ctx->sig; + __u64 pid_tgid; + __u32 pid; + + if (failed_only && ret == 0) + return 0; + + if (target_signal && sig != target_signal) + return 0; + + pid_tgid = bpf_get_current_pid_tgid(); + pid = pid_tgid >> 32; + if (filtered_pid && pid != filtered_pid) + return 0; + + event.pid = pid; + event.tpid = tpid; + event.sig = sig; + event.ret = ret; + bpf_get_current_comm(event.comm, sizeof(event.comm)); + bpf_perf_event_output(ctx, &events, BPF_F_CURRENT_CPU, &event, sizeof(event)); + return 0; +} + +char LICENSE[] SEC("license") = "Dual BSD/GPL"; diff --git a/lcc/eunomia-lcc/example/sigsnoop.h b/lcc/eunomia-lcc/example/sigsnoop.h new file mode 100644 index 0000000..a9826d8 --- /dev/null +++ b/lcc/eunomia-lcc/example/sigsnoop.h @@ -0,0 +1,16 @@ +// SPDX-License-Identifier: (LGPL-2.1 OR BSD-2-Clause) +/* Copyright (c) 2021~2022 Hengqi Chen */ +#ifndef __SIGSNOOP_H +#define __SIGSNOOP_H + +#define TASK_COMM_LEN 16 + +struct event { + unsigned int pid; + unsigned int tpid; + int sig; + int ret; + char comm[TASK_COMM_LEN]; +}; + +#endif /* __SIGSNOOP_H */ -- Gitee From ab03cc8390b2846b33a7ec3d163665af52a4a619 Mon Sep 17 00:00:00 2001 From: yunwei37 <1067852565@qq.com> Date: Wed, 14 Dec 2022 16:41:06 +0800 Subject: [PATCH 05/10] add eunomia-lcc README.md --- lcc/eunomia-lcc/README.md | 84 ++++++++++++++++++++++++++ lcc/eunomia-lcc/example/sigsnoop.bpf.c | 4 ++ 2 files changed, 88 insertions(+) create mode 100644 lcc/eunomia-lcc/README.md diff --git a/lcc/eunomia-lcc/README.md b/lcc/eunomia-lcc/README.md new file mode 100644 index 0000000..a7a36d1 --- /dev/null +++ b/lcc/eunomia-lcc/README.md @@ -0,0 +1,84 @@ +# eunomia-lcc + +use eunomia-bpf as a frontend for Coolbpf libbpf and kernel bpf supported. + +使用 [eunomia-bpf](https://gitee.com/anolis/eunomia) 作为 Coolbpf 的用户态开发库,让 Coolbpf 也能: + +- 在编写 eBPF 程序或工具时只编写 libbpf 内核态代码,自动获取内核态导出信息,自动生成命令行参数、直方图输出等; +- 使用 WASM 进行用户态交互程序的开发,在 WASM 虚拟机内部控制整个 eBPF 程序的加载和执行,以及处理相关数据; +- 可以将预编译的 eBPF 程序打包为通用的 JSON 或 WASM 模块,跨架构和内核版本进行分发,无需重新编译即可动态加载运行。 + +同时保留 Coolbpf 的低版本兼容、BTF 自动获取、远程编译等特性,让 eBPF 程序的开发更加简便易行。 + +## example: signsoop + +以编译 example 为例,首先确保 eunomia-bpf 已经作为一个 submodule 加入到项目中: + +```bash +git submodule update --init --recursive +``` + +然后在项目根目录下执行,编译 eunomia-bpf 的编译工具链和运行时,在编译时需要安装 `libclang`, `libelf` and `zlib` 库: + +```bash +cd lcc/eunomia-lcc +make +``` + +编译完成后可以看到编译工具 `ecc` 和运行工具 `ecli`, 可以使用 `ecc` 编译 example 中的 eBPF 程序: + +```console +$ ./ecc example/sigsnoop.bpf.c example/sigsnoop.h +Compiling bpf object... +Generating export types... +Packing ebpf object and config into package.json... +``` + +或者在 x86 上也可以用 docker 编译: + +```bash +cd example +docker run -it -v `pwd`/:/src/ yunwei37/ebpm:latest +``` + +可以使用 `ecli` 运行编译后的程序: + +```console +$ sudo ./ecli example/package.json +TIME PID TPID SIG RET COMM +20:43:44 21276 3054 0 0 cpptools-srv +20:43:44 22407 3054 0 0 cpptools-srv +20:43:44 20222 3054 0 0 cpptools-srv +20:43:44 8933 3054 0 0 cpptools-srv +20:43:44 2915 2803 0 0 node +20:43:44 2943 2803 0 0 node +20:43:44 31453 3054 0 0 cpptools-srv + +$ sudo ./ecli example/package.json -h +Usage: sigsnoop_bpf [--help] [--version] [--verbose] [--filtered_pid VAR] [--target_signal VAR] [--failed_only] + +Trace standard and real-time signals. + +Optional arguments: + -h, --help shows help message and exits + -v, --version prints version information and exits + --verbose prints libbpf debug information + --filtered_pid Process ID to trace. If set to 0, trace all processes. + --target_signal Signal number to trace. If set to 0, trace all signals. + --failed_only Trace only failed signals. If set to false, trace all signals. + +Built with eunomia-bpf framework. + +$ sudo ./ecli example/package.json --filtered_pid 3024 +TIME PID TPID SIG RET COMM +16:38:33 3024 2920 0 0 node +16:38:34 3024 2920 0 0 node +16:38:34 3024 2920 0 0 node +16:38:35 3024 2920 0 0 node +16:38:35 3024 2920 0 0 node +16:38:36 3024 2920 0 0 node +``` + +## 更多信息 + +请参考: diff --git a/lcc/eunomia-lcc/example/sigsnoop.bpf.c b/lcc/eunomia-lcc/example/sigsnoop.bpf.c index e03981f..873f1de 100644 --- a/lcc/eunomia-lcc/example/sigsnoop.bpf.c +++ b/lcc/eunomia-lcc/example/sigsnoop.bpf.c @@ -6,8 +6,11 @@ #define MAX_ENTRIES 10240 +/// "Process ID to trace. If set to 0, trace all processes." const volatile pid_t filtered_pid = 0; +/// "Signal number to trace. If set to 0, trace all signals." const volatile int target_signal = 0; +/// "Trace only failed signals. If set to false, trace all signals." const volatile bool failed_only = false; struct { @@ -142,4 +145,5 @@ int sig_trace(struct trace_event_raw_signal_generate *ctx) return 0; } +/// "Trace standard and real-time signals." char LICENSE[] SEC("license") = "Dual BSD/GPL"; -- Gitee From 830ef1918fa6b293719361de902c421a91d3e14e Mon Sep 17 00:00:00 2001 From: yunwei37 <1067852565@qq.com> Date: Wed, 14 Dec 2022 17:05:08 +0800 Subject: [PATCH 06/10] update eunomia-bpf with gitee submodule --- .gitmodules | 2 +- third/eunomia-bpf | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.gitmodules b/.gitmodules index 54e17ea..9eb0ff1 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,3 +1,3 @@ [submodule "third/eunomia-bpf"] path = third/eunomia-bpf - url = https://github.com/eunomia-bpf/eunomia-bpf + url = https://gitee.com/anolis/eunomia diff --git a/third/eunomia-bpf b/third/eunomia-bpf index ad470dd..d3280e0 160000 --- a/third/eunomia-bpf +++ b/third/eunomia-bpf @@ -1 +1 @@ -Subproject commit ad470dd8f6ad73f05196a160ffb9eb785406bc25 +Subproject commit d3280e06244383b5a4f5f0d5a8e5f3d53165c6d0 -- Gitee From 1dc06a84230651abcd65b6ade0e6462b4129ff88 Mon Sep 17 00:00:00 2001 From: yunwei37 <1067852565@qq.com> Date: Wed, 14 Dec 2022 18:16:53 +0800 Subject: [PATCH 07/10] move sigsnoop to example/sigsnoop dir --- lcc/eunomia-lcc/README.md | 21 +++++++++++-------- .../example/{ => sigsnoop}/sigsnoop.bpf.c | 0 .../example/{ => sigsnoop}/sigsnoop.h | 0 3 files changed, 12 insertions(+), 9 deletions(-) rename lcc/eunomia-lcc/example/{ => sigsnoop}/sigsnoop.bpf.c (100%) rename lcc/eunomia-lcc/example/{ => sigsnoop}/sigsnoop.h (100%) diff --git a/lcc/eunomia-lcc/README.md b/lcc/eunomia-lcc/README.md index a7a36d1..d9fb06c 100644 --- a/lcc/eunomia-lcc/README.md +++ b/lcc/eunomia-lcc/README.md @@ -2,7 +2,7 @@ use eunomia-bpf as a frontend for Coolbpf libbpf and kernel bpf supported. -使用 [eunomia-bpf](https://gitee.com/anolis/eunomia) 作为 Coolbpf 的用户态开发库,让 Coolbpf 也能: +使用开源社区的 [eunomia-bpf](https://gitee.com/anolis/eunomia) 框架作为 Coolbpf 的用户态开发库,让 Coolbpf 也能: - 在编写 eBPF 程序或工具时只编写 libbpf 内核态代码,自动获取内核态导出信息,自动生成命令行参数、直方图输出等; - 使用 WASM 进行用户态交互程序的开发,在 WASM 虚拟机内部控制整个 eBPF 程序的加载和执行,以及处理相关数据; @@ -10,9 +10,11 @@ use eunomia-bpf as a frontend for Coolbpf libbpf and kernel bpf supported. 同时保留 Coolbpf 的低版本兼容、BTF 自动获取、远程编译等特性,让 eBPF 程序的开发更加简便易行。 -## example: signsoop +## example -以编译 example 为例,首先确保 eunomia-bpf 已经作为一个 submodule 加入到项目中: +### 编译运行(以编译 signsoop 为例) + +以编译 example/signsoop 为例,首先确保 eunomia-bpf 已经作为一个 submodule 加入到项目中: ```bash git submodule update --init --recursive @@ -28,23 +30,22 @@ make 编译完成后可以看到编译工具 `ecc` 和运行工具 `ecli`, 可以使用 `ecc` 编译 example 中的 eBPF 程序: ```console -$ ./ecc example/sigsnoop.bpf.c example/sigsnoop.h +$ ./ecc sigsnoop.bpf.c sigsnoop.h Compiling bpf object... Generating export types... Packing ebpf object and config into package.json... ``` -或者在 x86 上也可以用 docker 编译: +或者在 x86 上也可以用 docker 编译,在 sigsnoop 目录下: ```bash -cd example docker run -it -v `pwd`/:/src/ yunwei37/ebpm:latest ``` 可以使用 `ecli` 运行编译后的程序: ```console -$ sudo ./ecli example/package.json +$ sudo ./ecli package.json TIME PID TPID SIG RET COMM 20:43:44 21276 3054 0 0 cpptools-srv 20:43:44 22407 3054 0 0 cpptools-srv @@ -54,7 +55,7 @@ TIME PID TPID SIG RET COMM 20:43:44 2943 2803 0 0 node 20:43:44 31453 3054 0 0 cpptools-srv -$ sudo ./ecli example/package.json -h +$ sudo ./ecli package.json -h Usage: sigsnoop_bpf [--help] [--version] [--verbose] [--filtered_pid VAR] [--target_signal VAR] [--failed_only] Trace standard and real-time signals. @@ -69,7 +70,7 @@ Optional arguments: Built with eunomia-bpf framework. -$ sudo ./ecli example/package.json --filtered_pid 3024 +$ sudo ./ecli package.json --filtered_pid 3024 TIME PID TPID SIG RET COMM 16:38:33 3024 2920 0 0 node 16:38:34 3024 2920 0 0 node @@ -79,6 +80,8 @@ TIME PID TPID SIG RET COMM 16:38:36 3024 2920 0 0 node ``` +### + ## 更多信息 请参考: diff --git a/lcc/eunomia-lcc/example/sigsnoop.bpf.c b/lcc/eunomia-lcc/example/sigsnoop/sigsnoop.bpf.c similarity index 100% rename from lcc/eunomia-lcc/example/sigsnoop.bpf.c rename to lcc/eunomia-lcc/example/sigsnoop/sigsnoop.bpf.c diff --git a/lcc/eunomia-lcc/example/sigsnoop.h b/lcc/eunomia-lcc/example/sigsnoop/sigsnoop.h similarity index 100% rename from lcc/eunomia-lcc/example/sigsnoop.h rename to lcc/eunomia-lcc/example/sigsnoop/sigsnoop.h -- Gitee From 893d40be80706ef981b0351e0cceb340b5bd1449 Mon Sep 17 00:00:00 2001 From: yunwei37 <1067852565@qq.com> Date: Wed, 14 Dec 2022 18:26:30 +0800 Subject: [PATCH 08/10] add runqlat as an example --- lcc/eunomia-lcc/.gitignore | 4 +- lcc/eunomia-lcc/README.md | 48 +++++- lcc/eunomia-lcc/example/runqlat/bits.bpf.h | 31 ++++ .../example/runqlat/core_fixes.bpf.h | 112 +++++++++++++ lcc/eunomia-lcc/example/runqlat/maps.bpf.h | 26 +++ lcc/eunomia-lcc/example/runqlat/runqlat.bpf.c | 158 ++++++++++++++++++ lcc/eunomia-lcc/example/runqlat/runqlat.h | 14 ++ 7 files changed, 390 insertions(+), 3 deletions(-) create mode 100644 lcc/eunomia-lcc/example/runqlat/bits.bpf.h create mode 100644 lcc/eunomia-lcc/example/runqlat/core_fixes.bpf.h create mode 100644 lcc/eunomia-lcc/example/runqlat/maps.bpf.h create mode 100644 lcc/eunomia-lcc/example/runqlat/runqlat.bpf.c create mode 100644 lcc/eunomia-lcc/example/runqlat/runqlat.h diff --git a/lcc/eunomia-lcc/.gitignore b/lcc/eunomia-lcc/.gitignore index c211399..f0966be 100644 --- a/lcc/eunomia-lcc/.gitignore +++ b/lcc/eunomia-lcc/.gitignore @@ -1,4 +1,4 @@ ecli ecc -example/*.o -example/*.json +example/*/*.o +example/*/*.json diff --git a/lcc/eunomia-lcc/README.md b/lcc/eunomia-lcc/README.md index d9fb06c..f1f5099 100644 --- a/lcc/eunomia-lcc/README.md +++ b/lcc/eunomia-lcc/README.md @@ -80,7 +80,53 @@ TIME PID TPID SIG RET COMM 16:38:36 3024 2920 0 0 node ``` -### +### runqlat + +This program summarizes scheduler run queue latency as a histogram, showing +how long tasks spent waiting their turn to run on-CPU. + +```console +$ ./ecc runqlat.bpf.c runqlat.h +Compiling bpf object... +Generating export types... +Packing ebpf object and config into package.json... +$ sudo ./ecli package.json + (unit) : count distribution + 0 -> 1 : 69 |********************************** | + 2 -> 3 : 58 |***************************** | + 4 -> 7 : 42 |********************* | + 8 -> 15 : 69 |********************************** | + 16 -> 31 : 80 |****************************************| + 32 -> 63 : 28 |************** | + 64 -> 127 : 10 |***** | + 128 -> 255 : 6 |*** | + 256 -> 511 : 1 | | + 512 -> 1023 : 1 | | + 1024 -> 2047 : 0 | | + 2048 -> 4095 : 1 | | + 4096 -> 8191 : 0 | | + 8192 -> 16383 : 1 | | + +$ sudo ./ecli package.json -h +Usage: runqlat_bpf [--help] [--version] [--verbose] [--filter_cg] [--targ_per_process] [--targ_per_thread] [--targ_per_pidns] [--targ_ms] [--targ_tgid VAR] + +Summarize run queue (scheduler) latency as a histogram + +Optional arguments: + -h, --help shows help message and exits + -v, --version prints version information and exits + --verbose prints libbpf debug information + --filter_cg set value of bool variable filter_cg + --targ_per_process print a histogram per process ID + --targ_per_thread print a histogram per thread ID + --targ_per_pidns print a histogram per PID namespace + --targ_ms millisecond histogram + --targ_tgid trace this PID only + +Built with eunomia-bpf framework. +``` + + ## 更多信息 diff --git a/lcc/eunomia-lcc/example/runqlat/bits.bpf.h b/lcc/eunomia-lcc/example/runqlat/bits.bpf.h new file mode 100644 index 0000000..a2b7bb9 --- /dev/null +++ b/lcc/eunomia-lcc/example/runqlat/bits.bpf.h @@ -0,0 +1,31 @@ +/* SPDX-License-Identifier: (LGPL-2.1 OR BSD-2-Clause) */ +#ifndef __BITS_BPF_H +#define __BITS_BPF_H + +#define READ_ONCE(x) (*(volatile typeof(x) *)&(x)) +#define WRITE_ONCE(x, val) ((*(volatile typeof(x) *)&(x)) = val) + +static __always_inline u64 log2(u32 v) +{ + u32 shift, r; + + r = (v > 0xFFFF) << 4; v >>= r; + shift = (v > 0xFF) << 3; v >>= shift; r |= shift; + shift = (v > 0xF) << 2; v >>= shift; r |= shift; + shift = (v > 0x3) << 1; v >>= shift; r |= shift; + r |= (v >> 1); + + return r; +} + +static __always_inline u64 log2l(u64 v) +{ + u32 hi = v >> 32; + + if (hi) + return log2(hi) + 32; + else + return log2(v); +} + +#endif /* __BITS_BPF_H */ diff --git a/lcc/eunomia-lcc/example/runqlat/core_fixes.bpf.h b/lcc/eunomia-lcc/example/runqlat/core_fixes.bpf.h new file mode 100644 index 0000000..003163a --- /dev/null +++ b/lcc/eunomia-lcc/example/runqlat/core_fixes.bpf.h @@ -0,0 +1,112 @@ +/* SPDX-License-Identifier: (LGPL-2.1 OR BSD-2-Clause) */ +/* Copyright (c) 2021 Hengqi Chen */ + +#ifndef __CORE_FIXES_BPF_H +#define __CORE_FIXES_BPF_H + +#include +#include + +/** + * commit 2f064a59a1 ("sched: Change task_struct::state") changes + * the name of task_struct::state to task_struct::__state + * see: + * https://github.com/torvalds/linux/commit/2f064a59a1 + */ +struct task_struct___o { + volatile long int state; +} __attribute__((preserve_access_index)); + +struct task_struct___x { + unsigned int __state; +} __attribute__((preserve_access_index)); + +static __always_inline __s64 get_task_state(void *task) +{ + struct task_struct___x *t = task; + + if (bpf_core_field_exists(t->__state)) + return BPF_CORE_READ(t, __state); + return BPF_CORE_READ((struct task_struct___o *)task, state); +} + +/** + * commit 309dca309fc3 ("block: store a block_device pointer in struct bio") + * adds a new member bi_bdev which is a pointer to struct block_device + * see: + * https://github.com/torvalds/linux/commit/309dca309fc3 + */ +struct bio___o { + struct gendisk *bi_disk; +} __attribute__((preserve_access_index)); + +struct bio___x { + struct block_device *bi_bdev; +} __attribute__((preserve_access_index)); + +static __always_inline struct gendisk *get_gendisk(void *bio) +{ + struct bio___x *b = bio; + + if (bpf_core_field_exists(b->bi_bdev)) + return BPF_CORE_READ(b, bi_bdev, bd_disk); + return BPF_CORE_READ((struct bio___o *)bio, bi_disk); +} + +/** + * commit d5869fdc189f ("block: introduce block_rq_error tracepoint") + * adds a new tracepoint block_rq_error and it shares the same arguments + * with tracepoint block_rq_complete. As a result, the kernel BTF now has + * a `struct trace_event_raw_block_rq_completion` instead of + * `struct trace_event_raw_block_rq_complete`. + * see: + * https://github.com/torvalds/linux/commit/d5869fdc189f + */ +struct trace_event_raw_block_rq_complete___x { + dev_t dev; + sector_t sector; + unsigned int nr_sector; +} __attribute__((preserve_access_index)); + +struct trace_event_raw_block_rq_completion___x { + dev_t dev; + sector_t sector; + unsigned int nr_sector; +} __attribute__((preserve_access_index)); + +static __always_inline bool has_block_rq_completion() +{ + if (bpf_core_type_exists(struct trace_event_raw_block_rq_completion___x)) + return true; + return false; +} + +/** + * commit d152c682f03c ("block: add an explicit ->disk backpointer to the + * request_queue") and commit f3fa33acca9f ("block: remove the ->rq_disk + * field in struct request") make some changes to `struct request` and + * `struct request_queue`. Now, to get the `struct gendisk *` field in a CO-RE + * way, we need both `struct request` and `struct request_queue`. + * see: + * https://github.com/torvalds/linux/commit/d152c682f03c + * https://github.com/torvalds/linux/commit/f3fa33acca9f + */ +struct request_queue___x { + struct gendisk *disk; +} __attribute__((preserve_access_index)); + +struct request___x { + struct request_queue___x *q; + struct gendisk *rq_disk; +} __attribute__((preserve_access_index)); + +static __always_inline struct gendisk *get_disk(void *request) +{ + struct request___x *r = request; + + if (bpf_core_field_exists(r->rq_disk)) + return BPF_CORE_READ(r, rq_disk); + return BPF_CORE_READ(r, q, disk); +} + +#endif /* __CORE_FIXES_BPF_H */ diff --git a/lcc/eunomia-lcc/example/runqlat/maps.bpf.h b/lcc/eunomia-lcc/example/runqlat/maps.bpf.h new file mode 100644 index 0000000..51d1012 --- /dev/null +++ b/lcc/eunomia-lcc/example/runqlat/maps.bpf.h @@ -0,0 +1,26 @@ +// SPDX-License-Identifier: (LGPL-2.1 OR BSD-2-Clause) +// Copyright (c) 2020 Anton Protopopov +#ifndef __MAPS_BPF_H +#define __MAPS_BPF_H + +#include +#include + +static __always_inline void * +bpf_map_lookup_or_try_init(void *map, const void *key, const void *init) +{ + void *val; + long err; + + val = bpf_map_lookup_elem(map, key); + if (val) + return val; + + err = bpf_map_update_elem(map, key, init, BPF_NOEXIST); + if (err && err != -EEXIST) + return 0; + + return bpf_map_lookup_elem(map, key); +} + +#endif /* __MAPS_BPF_H */ diff --git a/lcc/eunomia-lcc/example/runqlat/runqlat.bpf.c b/lcc/eunomia-lcc/example/runqlat/runqlat.bpf.c new file mode 100644 index 0000000..1fbb52a --- /dev/null +++ b/lcc/eunomia-lcc/example/runqlat/runqlat.bpf.c @@ -0,0 +1,158 @@ +// SPDX-License-Identifier: GPL-2.0 +// Copyright (c) 2020 Wenbo Zhang +#include +#include +#include +#include +#include "runqlat.h" +#include "bits.bpf.h" +#include "maps.bpf.h" +#include "core_fixes.bpf.h" + +#define MAX_ENTRIES 10240 +#define TASK_RUNNING 0 + +const volatile bool filter_cg = false; +/// "print a histogram per process ID" +const volatile bool targ_per_process = false; +/// "print a histogram per thread ID" +const volatile bool targ_per_thread = false; +/// "print a histogram per PID namespace" +const volatile bool targ_per_pidns = false; +/// "millisecond histogram" +const volatile bool targ_ms = false; +/// "trace this PID only" +const volatile pid_t targ_tgid = 0; + +struct { + __uint(type, BPF_MAP_TYPE_CGROUP_ARRAY); + __type(key, u32); + __type(value, u32); + __uint(max_entries, 1); +} cgroup_map SEC(".maps"); + +struct { + __uint(type, BPF_MAP_TYPE_HASH); + __uint(max_entries, MAX_ENTRIES); + __type(key, u32); + __type(value, u64); +} start SEC(".maps"); + +static struct hist zero; + +/// @sample {"interval": 1000, "type" : "log2_hist"} +struct { + __uint(type, BPF_MAP_TYPE_HASH); + __uint(max_entries, MAX_ENTRIES); + __type(key, u32); + __type(value, struct hist); +} hists SEC(".maps"); + +static int trace_enqueue(u32 tgid, u32 pid) +{ + u64 ts; + + if (!pid) + return 0; + if (targ_tgid && targ_tgid != tgid) + return 0; + + ts = bpf_ktime_get_ns(); + bpf_map_update_elem(&start, &pid, &ts, BPF_ANY); + return 0; +} + +static unsigned int pid_namespace(struct task_struct *task) +{ + struct pid *pid; + unsigned int level; + struct upid upid; + unsigned int inum; + + /* get the pid namespace by following task_active_pid_ns(), + * pid->numbers[pid->level].ns + */ + pid = BPF_CORE_READ(task, thread_pid); + level = BPF_CORE_READ(pid, level); + bpf_core_read(&upid, sizeof(upid), &pid->numbers[level]); + inum = BPF_CORE_READ(upid.ns, ns.inum); + + return inum; +} + +static int handle_switch(bool preempt, struct task_struct *prev, struct task_struct *next) +{ + struct hist *histp; + u64 *tsp, slot; + u32 pid, hkey; + s64 delta; + + if (filter_cg && !bpf_current_task_under_cgroup(&cgroup_map, 0)) + return 0; + + if (get_task_state(prev) == TASK_RUNNING) + trace_enqueue(BPF_CORE_READ(prev, tgid), BPF_CORE_READ(prev, pid)); + + pid = BPF_CORE_READ(next, pid); + + tsp = bpf_map_lookup_elem(&start, &pid); + if (!tsp) + return 0; + delta = bpf_ktime_get_ns() - *tsp; + if (delta < 0) + goto cleanup; + + if (targ_per_process) + hkey = BPF_CORE_READ(next, tgid); + else if (targ_per_thread) + hkey = pid; + else if (targ_per_pidns) + hkey = pid_namespace(next); + else + hkey = -1; + histp = bpf_map_lookup_or_try_init(&hists, &hkey, &zero); + if (!histp) + goto cleanup; + if (!histp->comm[0]) + bpf_probe_read_kernel_str(&histp->comm, sizeof(histp->comm), + next->comm); + if (targ_ms) + delta /= 1000000U; + else + delta /= 1000U; + slot = log2l(delta); + if (slot >= MAX_SLOTS) + slot = MAX_SLOTS - 1; + __sync_fetch_and_add(&histp->slots[slot], 1); + +cleanup: + bpf_map_delete_elem(&start, &pid); + return 0; +} + +SEC("raw_tp/sched_wakeup") +int BPF_PROG(handle_sched_wakeup, struct task_struct *p) +{ + if (filter_cg && !bpf_current_task_under_cgroup(&cgroup_map, 0)) + return 0; + + return trace_enqueue(BPF_CORE_READ(p, tgid), BPF_CORE_READ(p, pid)); +} + +SEC("raw_tp/sched_wakeup_new") +int BPF_PROG(handle_sched_wakeup_new, struct task_struct *p) +{ + if (filter_cg && !bpf_current_task_under_cgroup(&cgroup_map, 0)) + return 0; + + return trace_enqueue(BPF_CORE_READ(p, tgid), BPF_CORE_READ(p, pid)); +} + +SEC("raw_tp/sched_switch") +int BPF_PROG(handle_sched_switch, bool preempt, struct task_struct *prev, struct task_struct *next) +{ + return handle_switch(preempt, prev, next); +} + +/// "Summarize run queue (scheduler) latency as a histogram" +char LICENSE[] SEC("license") = "GPL"; diff --git a/lcc/eunomia-lcc/example/runqlat/runqlat.h b/lcc/eunomia-lcc/example/runqlat/runqlat.h new file mode 100644 index 0000000..b6f0a02 --- /dev/null +++ b/lcc/eunomia-lcc/example/runqlat/runqlat.h @@ -0,0 +1,14 @@ + +/* SPDX-License-Identifier: (LGPL-2.1 OR BSD-2-Clause) */ +#ifndef __RUNQLAT_H +#define __RUNQLAT_H + +#define TASK_COMM_LEN 16 +#define MAX_SLOTS 26 + +struct hist { + __u32 slots[MAX_SLOTS]; + char comm[TASK_COMM_LEN]; +}; + +#endif /* __RUNQLAT_H */ -- Gitee From 6ca4e199aba787c369263e6aeee52c499d7ca4ae Mon Sep 17 00:00:00 2001 From: yunwei37 <1067852565@qq.com> Date: Wed, 14 Dec 2022 18:33:50 +0800 Subject: [PATCH 09/10] add minimal example --- lcc/eunomia-lcc/README.md | 31 +++++++++++++++++-- lcc/eunomia-lcc/example/minimal/minimal.bpf.c | 21 +++++++++++++ 2 files changed, 49 insertions(+), 3 deletions(-) create mode 100644 lcc/eunomia-lcc/example/minimal/minimal.bpf.c diff --git a/lcc/eunomia-lcc/README.md b/lcc/eunomia-lcc/README.md index f1f5099..c3c909d 100644 --- a/lcc/eunomia-lcc/README.md +++ b/lcc/eunomia-lcc/README.md @@ -42,7 +42,7 @@ Packing ebpf object and config into package.json... docker run -it -v `pwd`/:/src/ yunwei37/ebpm:latest ``` -可以使用 `ecli` 运行编译后的程序: +可以使用 `ecli` 运行编译后的程序,可以通过内核态代码中的全局变量指定命令行参数,生成帮助信息等: ```console $ sudo ./ecli package.json @@ -80,6 +80,33 @@ TIME PID TPID SIG RET COMM 16:38:36 3024 2920 0 0 node ``` +### minimal + +`minimal` is just that – a minimal practical BPF application example. It +doesn't use or require BPF CO-RE, so should run on quite old kernels. It +installs a tracepoint handler which is triggered once every second. It uses +`bpf_printk()` BPF helper to communicate with the world. + +```console +$ ./ecc minimal.bpf.c +Compiling bpf object... +Packing ebpf object and config into package.json... +$ sudo ecli package.json +Runing eBPF program... +``` + +To see it's output, +read `/sys/kernel/debug/tracing/trace_pipe` file as a root: + +```shell +$ sudo cat /sys/kernel/debug/tracing/trace_pipe + <...>-3840345 [010] d... 3220701.101143: bpf_trace_printk: BPF triggered from PID 3840345. + <...>-3840345 [010] d... 3220702.101265: bpf_trace_printk: BPF triggered from PID 3840345. +``` + +`minimal` is great as a bare-bones experimental playground to quickly try out +new ideas or BPF features. + ### runqlat This program summarizes scheduler run queue latency as a histogram, showing @@ -126,8 +153,6 @@ Optional arguments: Built with eunomia-bpf framework. ``` - - ## 更多信息 请参考: diff --git a/lcc/eunomia-lcc/example/minimal/minimal.bpf.c b/lcc/eunomia-lcc/example/minimal/minimal.bpf.c new file mode 100644 index 0000000..0c65717 --- /dev/null +++ b/lcc/eunomia-lcc/example/minimal/minimal.bpf.c @@ -0,0 +1,21 @@ +/* SPDX-License-Identifier: (LGPL-2.1 OR BSD-2-Clause) */ +#define BPF_NO_GLOBAL_DATA +#include +#include +#include + +typedef unsigned int u32; +typedef int pid_t; +const pid_t pid_filter = 0; + +char LICENSE[] SEC("license") = "Dual BSD/GPL"; + +SEC("tp/syscalls/sys_enter_write") +int handle_tp(void *ctx) +{ + pid_t pid = bpf_get_current_pid_tgid() >> 32; + if (pid_filter && pid != pid_filter) + return 0; + bpf_printk("BPF triggered from PID %d.\n", pid); + return 0; +} -- Gitee From efbf01288522359bb6502be1e704e3da4a5586c6 Mon Sep 17 00:00:00 2001 From: yunwei37 <1067852565@qq.com> Date: Wed, 14 Dec 2022 18:36:41 +0800 Subject: [PATCH 10/10] add opensnoop example --- lcc/eunomia-lcc/README.md | 43 ++++++ .../example/opensnoop/opensnoop.bpf.c | 140 ++++++++++++++++++ lcc/eunomia-lcc/example/opensnoop/opensnoop.h | 21 +++ 3 files changed, 204 insertions(+) create mode 100644 lcc/eunomia-lcc/example/opensnoop/opensnoop.bpf.c create mode 100644 lcc/eunomia-lcc/example/opensnoop/opensnoop.h diff --git a/lcc/eunomia-lcc/README.md b/lcc/eunomia-lcc/README.md index c3c909d..446150d 100644 --- a/lcc/eunomia-lcc/README.md +++ b/lcc/eunomia-lcc/README.md @@ -153,6 +153,49 @@ Optional arguments: Built with eunomia-bpf framework. ``` +### opensnoop + +Demonstrations of opensnoop, the Linux eBPF/bcc version. + +opensnoop traces the open() syscall system-wide, and prints various details. +Example output: + +```console +$ sudo ecli package.json -h +Usage: opensnoop_bpf [--help] [--version] [--verbose] [--pid_target VAR] [--tgid_target VAR] [--uid_target VAR] [--failed] + +Trace open family syscalls. + +Optional arguments: + -h, --help shows help message and exits + -v, --version prints version information and exits + --verbose prints libbpf debug information + --pid_target Process ID to trace + --tgid_target Thread ID to trace + --uid_target User ID to trace + -f, --failed trace only failed events + +Built with eunomia-bpf framework. + +$ sudo ecli examples/bpftools/opensnoop/package.json +TIME TS PID UID RET FLAGS COMM FNAME +20:31:50 0 1 0 51 524288 systemd /proc/614/cgroup +20:31:50 0 33182 0 25 524288 ecli /etc/localtime +20:31:53 0 754 0 6 0 irqbalance /proc/interrupts +20:31:53 0 754 0 6 0 irqbalance /proc/stat +20:32:03 0 754 0 6 0 irqbalance /proc/interrupts +20:32:03 0 754 0 6 0 irqbalance /proc/stat +20:32:03 0 632 0 7 524288 vmtoolsd /etc/mtab +20:32:03 0 632 0 9 0 vmtoolsd /proc/devices + +$ sudo ecli examples/bpftools/opensnoop/package.json --pid_target 754 +TIME TS PID UID RET FLAGS COMM FNAME +20:34:13 0 754 0 6 0 irqbalance /proc/interrupts +20:34:13 0 754 0 6 0 irqbalance /proc/stat +20:34:23 0 754 0 6 0 irqbalance /proc/interrupts +20:34:23 0 754 0 6 0 irqbalance /proc/stat +``` + ## 更多信息 请参考: diff --git a/lcc/eunomia-lcc/example/opensnoop/opensnoop.bpf.c b/lcc/eunomia-lcc/example/opensnoop/opensnoop.bpf.c new file mode 100644 index 0000000..597c760 --- /dev/null +++ b/lcc/eunomia-lcc/example/opensnoop/opensnoop.bpf.c @@ -0,0 +1,140 @@ +// SPDX-License-Identifier: GPL-2.0 +// Copyright (c) 2019 Facebook +// Copyright (c) 2020 Netflix +#include +#include +#include "opensnoop.h" + +struct args_t { + const char *fname; + int flags; +}; + +/// Process ID to trace +const volatile int pid_target = 0; +/// Thread ID to trace +const volatile int tgid_target = 0; +/// @description User ID to trace +const volatile int uid_target = 0; +/// @cmdarg {"default": false, "short": "f", "long": "failed"} +/// @description trace only failed events +const volatile bool targ_failed = false; + +struct { + __uint(type, BPF_MAP_TYPE_HASH); + __uint(max_entries, 10240); + __type(key, u32); + __type(value, struct args_t); +} start SEC(".maps"); + +struct { + __uint(type, BPF_MAP_TYPE_PERF_EVENT_ARRAY); + __uint(key_size, sizeof(u32)); + __uint(value_size, sizeof(u32)); +} events SEC(".maps"); + +static __always_inline bool valid_uid(uid_t uid) { + return uid != INVALID_UID; +} + +static __always_inline +bool trace_allowed(u32 tgid, u32 pid) +{ + u32 uid; + + /* filters */ + if (tgid_target && tgid_target != tgid) + return false; + if (pid_target && pid_target != pid) + return false; + if (valid_uid(uid_target)) { + uid = (u32)bpf_get_current_uid_gid(); + if (uid_target != uid) { + return false; + } + } + return true; +} + +SEC("tracepoint/syscalls/sys_enter_open") +int tracepoint__syscalls__sys_enter_open(struct trace_event_raw_sys_enter* ctx) +{ + u64 id = bpf_get_current_pid_tgid(); + /* use kernel terminology here for tgid/pid: */ + u32 tgid = id >> 32; + u32 pid = id; + + /* store arg info for later lookup */ + if (trace_allowed(tgid, pid)) { + struct args_t args = {}; + args.fname = (const char *)ctx->args[0]; + args.flags = (int)ctx->args[1]; + bpf_map_update_elem(&start, &pid, &args, 0); + } + return 0; +} + +SEC("tracepoint/syscalls/sys_enter_openat") +int tracepoint__syscalls__sys_enter_openat(struct trace_event_raw_sys_enter* ctx) +{ + u64 id = bpf_get_current_pid_tgid(); + /* use kernel terminology here for tgid/pid: */ + u32 tgid = id >> 32; + u32 pid = id; + + /* store arg info for later lookup */ + if (trace_allowed(tgid, pid)) { + struct args_t args = {}; + args.fname = (const char *)ctx->args[1]; + args.flags = (int)ctx->args[2]; + bpf_map_update_elem(&start, &pid, &args, 0); + } + return 0; +} + +static __always_inline +int trace_exit(struct trace_event_raw_sys_exit* ctx) +{ + struct event event = {}; + struct args_t *ap; + int ret; + u32 pid = bpf_get_current_pid_tgid(); + + ap = bpf_map_lookup_elem(&start, &pid); + if (!ap) + return 0; /* missed entry */ + ret = ctx->ret; + if (targ_failed && ret >= 0) + goto cleanup; /* want failed only */ + + /* event data */ + event.pid = bpf_get_current_pid_tgid() >> 32; + event.uid = bpf_get_current_uid_gid(); + bpf_get_current_comm(&event.comm, sizeof(event.comm)); + bpf_probe_read_user_str(&event.fname, sizeof(event.fname), ap->fname); + event.flags = ap->flags; + event.ret = ret; + + /* emit event */ + bpf_perf_event_output(ctx, &events, BPF_F_CURRENT_CPU, + &event, sizeof(event)); + +cleanup: + bpf_map_delete_elem(&start, &pid); + return 0; +} + +SEC("tracepoint/syscalls/sys_exit_open") +int tracepoint__syscalls__sys_exit_open(struct trace_event_raw_sys_exit* ctx) +{ + return trace_exit(ctx); +} + +SEC("tracepoint/syscalls/sys_exit_openat") +int tracepoint__syscalls__sys_exit_openat(struct trace_event_raw_sys_exit* ctx) +{ + return trace_exit(ctx); +} + +/// Trace open family syscalls. +char LICENSE[] SEC("license") = "GPL"; diff --git a/lcc/eunomia-lcc/example/opensnoop/opensnoop.h b/lcc/eunomia-lcc/example/opensnoop/opensnoop.h new file mode 100644 index 0000000..a5aa43f --- /dev/null +++ b/lcc/eunomia-lcc/example/opensnoop/opensnoop.h @@ -0,0 +1,21 @@ +/* SPDX-License-Identifier: (LGPL-2.1 OR BSD-2-Clause) */ +#ifndef __OPENSNOOP_H +#define __OPENSNOOP_H + +#define TASK_COMM_LEN 16 +#define NAME_MAX 255 +#define INVALID_UID ((uid_t)-1) + +// used for export event +struct event { + /* user terminology for pid: */ + unsigned long long ts; + int pid; + int uid; + int ret; + int flags; + char comm[TASK_COMM_LEN]; + char fname[NAME_MAX]; +}; + +#endif /* __OPENSNOOP_H */ \ No newline at end of file -- Gitee