diff --git a/config/collector.conf b/config/collector.conf new file mode 100644 index 0000000000000000000000000000000000000000..b2567d813913d7006c821bda721c6db6efacc462 --- /dev/null +++ b/config/collector.conf @@ -0,0 +1,10 @@ +[common] +modules=io + +[io] +period_time=1 +max_save=10 +disk=default + +[log] +level=info \ No newline at end of file diff --git a/config/inspect.conf b/config/inspect.conf index 071cca1169a15c7ff6cc807e6a1013d684520ad8..f451d9e4b07294f7b5697bdb572eedc5357c129d 100644 --- a/config/inspect.conf +++ b/config/inspect.conf @@ -1,2 +1,5 @@ [inspect] -Interval=3 \ No newline at end of file +Interval=3 + +[log] +level=info diff --git a/config/logrotate b/config/logrotate index f54e7b3c44c50602a659d6b6107bcc3cb2fd4a8e..3dc77f511671c7ac3081d839ad1ad69567a09f89 100644 --- a/config/logrotate +++ b/config/logrotate @@ -1,8 +1,9 @@ /var/log/sysSentry/*.log { - nocompress + compress missingok notifempty copytruncate rotate 2 size +4096k + hourly } diff --git a/config/plugins/ai_block_io.ini b/config/plugins/ai_block_io.ini new file mode 100644 index 0000000000000000000000000000000000000000..69f44ba07c12c4ad91f4f11fbe0331975a2df08f --- /dev/null +++ b/config/plugins/ai_block_io.ini @@ -0,0 +1,39 @@ +[log] +level=info + +[common] +period_time=1 +disk=default +stage=default +iotype=read,write + +[algorithm] +train_data_duration=24 +train_update_duration=2 +algorithm_type=boxplot +boxplot_parameter=1.5 +win_type=not_continuous +win_size=30 +win_threshold=6 + +[latency_sata_ssd] +read_avg_lim=10000 +write_avg_lim=10000 +read_tot_lim=50000 +write_tot_lim=50000 + +[latency_nvme_ssd] +read_avg_lim=10000 +write_avg_lim=10000 +read_tot_lim=50000 +write_tot_lim=50000 + +[latency_sata_hdd] +read_avg_lim=15000 +write_avg_lim=15000 +read_tot_lim=50000 +write_tot_lim=50000 + +[iodump] +read_iodump_lim=0 +write_iodump_lim=0 \ No newline at end of file diff --git a/config/plugins/avg_block_io.ini b/config/plugins/avg_block_io.ini new file mode 100644 index 0000000000000000000000000000000000000000..3b4ee330664233bbd9e6340a9cec327855ba9687 --- /dev/null +++ b/config/plugins/avg_block_io.ini @@ -0,0 +1,40 @@ +[log] +level=info + +[common] +disk=default +stage=default +iotype=read,write +period_time=1 + +[algorithm] +win_size=30 +win_threshold=6 + +[latency_nvme_ssd] +read_avg_lim=10000 +write_avg_lim=10000 +read_avg_time=3 +write_avg_time=3 +read_tot_lim=50000 +write_tot_lim=50000 + +[latency_sata_ssd] +read_avg_lim=10000 +write_avg_lim=10000 +read_avg_time=3 +write_avg_time=3 +read_tot_lim=50000 +write_tot_lim=50000 + +[latency_sata_hdd] +read_avg_lim=15000 +write_avg_lim=15000 +read_avg_time=3 +write_avg_time=3 +read_tot_lim=50000 +write_tot_lim=50000 + +[iodump] +read_iodump_lim=0 +write_iodump_lim=0 diff --git a/config/tasks/ai_block_io.mod b/config/tasks/ai_block_io.mod new file mode 100644 index 0000000000000000000000000000000000000000..82f4f0bd2a3c67d8181393537e3c1e2f24d13d86 --- /dev/null +++ b/config/tasks/ai_block_io.mod @@ -0,0 +1,7 @@ +[common] +enabled=yes +task_start=/usr/bin/python3 /usr/bin/ai_block_io +task_stop=pkill -f /usr/bin/ai_block_io +type=oneshot +alarm_id=1002 +alarm_clear_time=5 \ No newline at end of file diff --git a/config/tasks/avg_block_io.mod b/config/tasks/avg_block_io.mod new file mode 100644 index 0000000000000000000000000000000000000000..bcd063b0e8f063875eca905b80937048bd196f95 --- /dev/null +++ b/config/tasks/avg_block_io.mod @@ -0,0 +1,7 @@ +[common] +enabled=yes +task_start=/usr/bin/python3 /usr/bin/avg_block_io +task_stop=pkill -f /usr/bin/avg_block_io +type=oneshot +alarm_id=1002 +alarm_clear_time=5 diff --git a/config/xalarm.conf b/config/xalarm.conf index 14c6d3902afbb2623f0fde8639a5f8ff7e8d5792..323d2dd53837a79af9e7f4c849e71defd6ee7985 100644 --- a/config/xalarm.conf +++ b/config/xalarm.conf @@ -1,2 +1,5 @@ [filter] id_mask = 1001-1128 + +[log] +level=info diff --git a/selftest/test/test_ai_block_io.py b/selftest/test/test_ai_block_io.py new file mode 100644 index 0000000000000000000000000000000000000000..c36fef54488a2e5f5a7b6d1b3b74c0a3a72f7264 --- /dev/null +++ b/selftest/test/test_ai_block_io.py @@ -0,0 +1,165 @@ +# coding: utf-8 +# Copyright (c) 2024 Huawei Technologies Co., Ltd. +# sysSentry is licensed under the Mulan PSL v2. +# You can use this software according to the terms and conditions of the Mulan PSL v2. +# You may obtain a copy of Mulan PSL v2 at: +# http://license.coscl.org.cn/MulanPSL2 +# THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR +# PURPOSE. +# See the Mulan PSL v2 for more details. + +import unittest +import numpy as np + +from sentryPlugins.ai_threshold_slow_io_detection.threshold import AbsoluteThreshold, BoxplotThreshold, NSigmaThreshold +from sentryPlugins.ai_threshold_slow_io_detection.sliding_window import (NotContinuousSlidingWindow, + ContinuousSlidingWindow, MedianSlidingWindow) + + +def _get_boxplot_threshold(data_list: list, parameter): + q1 = np.percentile(data_list, 25) + q3 = np.percentile(data_list, 75) + iqr = q3 - q1 + return q3 + parameter * iqr + + +def _get_n_sigma_threshold(data_list: list, parameter): + mean = np.mean(data_list) + std = np.std(data_list) + return mean + parameter * std + + +class Test(unittest.TestCase): + @classmethod + def setUpClass(cls): + print("UnitTest Begin...") + + @classmethod + def tearDownClass(cls): + print("UnitTest End...") + + def setUp(self): + print("Begin...") + + def tearDown(self): + print("End...") + + def test_absolute_threshold(self): + absolute = AbsoluteThreshold() + self.assertEqual(None, absolute.get_threshold()) + self.assertFalse(absolute.is_abnormal(5000)) + absolute.set_threshold(40) + self.assertEqual(40, absolute.get_threshold()) + self.assertTrue(absolute.is_abnormal(50)) + + def test_boxplot_threshold(self): + boxplot = BoxplotThreshold(1.5, 5, 1) + # 阶段1：尚未初始化 + self.assertEqual(None, boxplot.get_threshold()) + self.assertFalse(boxplot.is_abnormal(5000)) + # 往boxplot中插入5个元素后，会生成阈值 + data_list = [20, 20, 20, 30, 10] + for data in data_list: + boxplot.push_latest_data_to_queue(data) + # 阶段2：初始化 + boxplot_threshold = boxplot.get_threshold() + self.assertEqual(_get_boxplot_threshold(data_list, 1.5), boxplot_threshold) + self.assertTrue(boxplot.is_abnormal(5000)) + data_list.pop(0) + data_list.append(100) + boxplot.push_latest_data_to_queue(100) + # 阶段3：更新阈值 + boxplot_threshold = boxplot.get_threshold() + self.assertEqual(_get_boxplot_threshold(data_list, 1.5), boxplot_threshold) + + def test_n_sigma_threshold(self): + n_sigma = NSigmaThreshold(3, 5, 1) + self.assertEqual(None, n_sigma.get_threshold()) + self.assertFalse(n_sigma.is_abnormal(5000)) + data_list = [20, 20, 20, 30, 10] + for data in data_list: + n_sigma.push_latest_data_to_queue(data) + n_sigma_threshold = n_sigma.get_threshold() + self.assertEqual(_get_n_sigma_threshold(data_list, 3), n_sigma_threshold) + self.assertTrue(n_sigma.is_abnormal(5000)) + data_list.pop(0) + data_list.append(100) + n_sigma.push_latest_data_to_queue(100) + # 阶段3：更新阈值 + n_sigma_threshold = n_sigma.get_threshold() + self.assertEqual(_get_n_sigma_threshold(data_list, 3), n_sigma_threshold) + + def test_not_continuous_sliding_window(self): + not_continuous = NotContinuousSlidingWindow(5, 3) + boxplot_threshold = BoxplotThreshold(1.5, 10, 8) + boxplot_threshold.attach_observer(not_continuous) + data_list1 = [19, 20, 20, 20, 20, 20, 22, 24, 23, 20] + for data in data_list1: + boxplot_threshold.push_latest_data_to_queue(data) + result = not_continuous.is_slow_io_event(data) + self.assertFalse(result[0]) + self.assertEqual(23.75, boxplot_threshold.get_threshold()) + boxplot_threshold.push_latest_data_to_queue(24) + result = not_continuous.is_slow_io_event(24) + self.assertFalse(result[0]) + boxplot_threshold.push_latest_data_to_queue(25) + result = not_continuous.is_slow_io_event(25) + self.assertTrue(result[0]) + data_list2 = [20, 20, 20, 20, 20, 20] + for data in data_list2: + boxplot_threshold.push_latest_data_to_queue(data) + result = not_continuous.is_slow_io_event(data) + self.assertFalse(result[0]) + self.assertEqual(25.625, boxplot_threshold.get_threshold()) + + def test_continuous_sliding_window(self): + continuous = ContinuousSlidingWindow(5, 3) + boxplot_threshold = BoxplotThreshold(1.5, 10, 8) + boxplot_threshold.attach_observer(continuous) + data_list = [19, 20, 20, 20, 20, 20, 22, 24, 23, 20] + for data in data_list: + boxplot_threshold.push_latest_data_to_queue(data) + result = continuous.is_slow_io_event(data) + self.assertFalse(result[0]) + self.assertEqual(23.75, boxplot_threshold.get_threshold()) + # 没有三个异常点 + self.assertFalse(continuous.is_slow_io_event(25)[0]) + # 不连续的三个异常点 + self.assertFalse(continuous.is_slow_io_event(25)[0]) + # 连续的三个异常点 + self.assertTrue(continuous.is_slow_io_event(25)[0]) + + def test_median_sliding_window(self): + median = MedianSlidingWindow(5, 3) + absolute_threshold = AbsoluteThreshold(10, 8) + absolute_threshold.attach_observer(median) + absolute_threshold.set_threshold(24.5) + data_list = [24, 24, 24, 25, 25] + for data in data_list: + self.assertFalse(median.is_slow_io_event(data)[0]) + self.assertTrue(median.is_slow_io_event(25)[0]) + + def test_parse_collect_data(self): + collect = { + "read": [1.0, 2.0, 3.0, 4.0], + "write": [5.0, 6.0, 7.0, 8.0], + "flush": [9.0, 10.0, 11.0, 12.0], + "discard": [13.0, 14.0, 15.0, 16.0], + } + from io_data import BaseData + from data_access import _get_io_stage_data + + io_data = _get_io_stage_data(collect) + self.assertEqual( + io_data.read, BaseData(latency=1.0, io_dump=2.0, io_length=3.0, iops=4.0) + ) + self.assertEqual( + io_data.write, BaseData(latency=5.0, io_dump=6.0, io_length=7.0, iops=8.0) + ) + self.assertEqual( + io_data.flush, BaseData(latency=9.0, io_dump=10.0, io_length=11.0, iops=12.0) + ) + self.assertEqual( + io_data.discard, BaseData(latency=13.0, io_dump=14.0, io_length=15.0, iops=16.0) + ) diff --git a/service/sentryCollector.service b/service/sentryCollector.service new file mode 100644 index 0000000000000000000000000000000000000000..e09ddb3edcac2c303678117f1c4213371876b2ec --- /dev/null +++ b/service/sentryCollector.service @@ -0,0 +1,12 @@ +[Unit] +Description = Collection module added for sysSentry + +[Service] +ExecStart=/usr/bin/python3 /usr/bin/sentryCollector +ExecStop=/bin/kill $MAINPID +KillMode=process +Restart=on-failure +RestartSec=10s + +[Install] +WantedBy = multi-user.target diff --git a/service/sysSentry.service b/service/sysSentry.service index 4d85a6c2af31ac4f14b86259bc9e4525491a5cc1..1d8338f037a2f390b50d868a10685d4000cf8009 100644 --- a/service/sysSentry.service +++ b/service/sysSentry.service @@ -2,7 +2,7 @@ Description=EulerOS System Inspection Frame [Service] -ExecStart=/usr/bin/syssentry +ExecStart=/usr/bin/python3 /usr/bin/syssentry ExecStop=/bin/kill $MAINPID KillMode=process Restart=on-failure diff --git a/src/c/catcli/catlib/cli_param_checker.c b/src/c/catcli/catlib/cli_param_checker.c index a1aa636f220d86f7896a85eee786359c7aa1bc32..71edf17df88dbc1bfdef26540c0e68b02778759c 100644 --- a/src/c/catcli/catlib/cli_param_checker.c +++ b/src/c/catcli/catlib/cli_param_checker.c @@ -2,6 +2,7 @@ #include #include #include +#include #include #include #include "cli_common.h" @@ -13,11 +14,13 @@ void checkset_cpu_usage_percentage(char *getopt_optarg, catcli_request_body *p_request_body, struct option_errs *errs) { long cpu_utility = strtol(getopt_optarg, NULL, DECIMAL); - if (cpu_utility <= 0 || cpu_utility > CPU_USAGE_PERCENTAGE_MAX) { + if (cpu_utility <= 0 || cpu_utility > CPU_USAGE_PERCENTAGE_MAX || strchr(getopt_optarg, '.') != NULL) { strncpy(errs->patrol_module_err, "\"cpu_utility \" must be an integer greater in the range (0,100],correct \"-u, --cpu_utility\"\n", MAX_ERR_LEN); + p_request_body->cpu_utility = 0; + } else { + p_request_body->cpu_utility = (int)cpu_utility; } - p_request_body->cpu_utility = (int)cpu_utility; } void checkset_cpulist(char *getopt_optarg, catcli_request_body *p_request_body, struct option_errs *errs) @@ -68,12 +71,13 @@ void checkset_cpulist(char *getopt_optarg, catcli_request_body *p_request_body, void checkset_patrol_time(char *getopt_optarg, catcli_request_body *p_request_body, struct option_errs *errs) { long second = strtol(getopt_optarg, NULL, DECIMAL); - if (second <= 0 || second > INT_MAX) { + if (second <= 0 || second > INT_MAX || strchr(getopt_optarg, '.') != NULL) { strncpy(errs->patrol_time_err, "\"patrol_second\" must be a number in the range of (0,INT_MAX] ,correct \"-t, --patrol_second\"\n", MAX_ERR_LEN); + } else { + p_request_body->patrol_second = (int)second; } - p_request_body->patrol_second = (int)second; } void checkset_patrol_type(char *getopt_optarg, catcli_request_body *p_request_body, struct option_errs *errs) diff --git a/src/c/ebpf_collector/Makefile b/src/c/ebpf_collector/Makefile new file mode 100644 index 0000000000000000000000000000000000000000..20a9b725816e81b81acb2e333e8e146c6f8abfa7 --- /dev/null +++ b/src/c/ebpf_collector/Makefile @@ -0,0 +1,101 @@ +# Copyright (c) Huawei Technologies Co., Ltd. 2024. All rights reserved. +# Description: ebpf collector program +ARCH ?= $(shell uname -m | sed 's/x86_64/x86/' \ + | sed 's/arm.*/arm/' \ + | sed 's/aarch64/arm64/' \ + | sed 's/ppc64le/powerpc/' \ + | sed 's/mips.*/mips/' \ + | sed 's/riscv64/riscv/' \ + | sed 's/loongarch64/loongarch/') + +KERNEL_VERSION ?= $(shell rpm -qa | grep "kernel-source-4.19" | cut -d' ' -f1 | sed 's/kernel-source-//') +KERNEL_SRC := /usr/src/kernels/$(KERNEL_VERSION) +KERNEL_PATH := /usr/src/linux-$(KERNEL_VERSION) +GCC_ARCH ?= $(shell gcc -dumpmachine) +GCC_VERSION ?= $(shell gcc -dumpversion) + +LINUX_INCLUDE := -I$(KERNEL_SRC)/include/ +LINUX_INCLUDE += -I$(KERNEL_SRC)/arch/$(ARCH)/include/ +LINUX_INCLUDE += -I$(KERNEL_SRC)/arch/$(ARCH)/include/generated +LINUX_INCLUDE += -I$(KERNEL_SRC)/arch/$(ARCH)/include/uapi +LINUX_INCLUDE += -I$(KERNEL_SRC)/arch/$(ARCH)/include/uapi/linux +LINUX_INCLUDE += -I$(KERNEL_SRC)/arch/$(ARCH)/include/generated/uapi +LINUX_INCLUDE += -I$(KERNEL_SRC)/include/uapi +LINUX_INCLUDE += -I$(KERNEL_SRC)/include/generated/uapi +LINUX_INCLUDE += -include $(KERNEL_SRC)/include/linux/kconfig.h +LINUX_INCLUDE += -I$(KERNEL_PATH)/samples/bpf +LINUX_INCLUDE += -I$(KERNEL_SRC)/tools/lib/ +LINUX_INCLUDE += -I/usr/src/kernels/$(KERNEL_VERSION)/samples/bpf +LINUX_INCLUDE += -I$(KERNEL_SRC)/tools/perf/include/bpf +LINUX_INCLUDE += -I/usr/include/libbpf/src/bpf +LINUX_INCLUDE += -I/usr/src/kernels/$(KERNEL_VERSION)/include/uapi/linux/ +LINUX_INCLUDE += -I/usr/include/bpf/ +LINUX_INCLUDE += -I/usr/include/ +BPF_LOAD_INCLUDE := -I/usr/include +BPF_LOAD_INCLUDE += -I$(KERNEL_SRC)/include/ +BPF_LOAD_INCLUDE += -I/usr/src/kernels/$(KERNEL_VERSION)/include/ +KBUILD_HOSTCFLAGS := -I$(KERNEL_PATH)/include/ +KBUILD_HOSTCFLAGS += -I$(KERNEL_PATH)/tools/lib/ -I$(KERNEL_PATH)/tools/include +KBUILD_HOSTCFLAGS += -I$(KERNEL_PATH)/tools/perf +NOSTDINC_FLAGS := -nostdinc +EXTRA_CFLAGS := -isystem /usr/lib/gcc/$(GCC_ARCH)/$(GCC_VERSION)/include +CFLAGS := -g -Wall -w + +CLANG_BPF_SYS_INCLUDES ?= $(shell $(CLANG) -v -E - &1 \ + | sed -n '/<...> search starts here:/,/End of search list./{ s| \(/.*\)|-idirafter \1|p }') + +APPS = ebpf_collector + +CC = gcc +LLC ?= llc +CLANG ?= clang + +USER_CFLAGS = -I. -I/usr/src/kernels/$(KERNEL_VERSION)/include/uapi/linux/ -I/usr/src/kernel/include -Wall +KERNEL_CFLAGS = -I. -I/usr/src/kernels/$(KERNEL_VERSION)/include/uapi/linux/ -Wall +LOADER_CFLAGS = -I. -I/usr/src/kernels/$(KERNEL_VERSION)/include/uapi/linux/ -I/usr/src/kernel/include +CLANG_FLAGS = -O2 -emit-llvm -c +LLC_FLAGS = -march=bpf -filetype=obj + +OUTPUT := output + +.PHONY: all +all: $(APPS) + +.PHONY: clean +clean: + $(call msg,CLEAN) + $(Q)rm -rf $(OUTPUT) $(APPS) + +$(OUTPUT): + $(call msg,MKDIR,$@) + $(Q)mkdir -p $@ + +$(OUTPUT)/%.bpf.o: %.bpf.c + $(call msg,BPF,$@) + $(CLANG) $(NOSTDINC_FLAGS) $(EXTRA_CFLAGS) $(LINUX_INCLUDE) $(KBUILD_HOSTCFLAGS) \ + -D__KERNEL__ -D__BPF_TRACING__ -Wno-unused-value -Wno-pointer-sign \ + -D__TARGET_ARCH_$(ARCH) -Wno-compare-distinct-pointer-types \ + -Wno-gnu-variable-sized-type-not-at-end \ + -Wno-address-of-packed-member -Wno-tautological-compare \ + -Wno-unknown-warning-option $(CLANG_ARCH_ARGS) \ + -O2 -emit-llvm -c $< -o -| $(LLC) $(LLC_FLAGS) -o $@ + +$(patsubst %,$(OUTPUT)/%.o,$(APPS)): %.o: %.bpf.o + +$(OUTPUT)/bpf_load.o: bpf_load.c | $(OUTPUT) + $(call msg,CC,$@) + $(CC) $(NOSTDINC_FLAGS) $(EXTRA_CFLAGS) $(CFLAGS) -I$(KERNEL_PATH)/samples/bpf -I$(KERNEL_PATH)/tools/perf $(BPF_LOAD_INCLUDE) \ + -I$(KERNEL_PATH)/tools/lib/ -I$(KERNEL_PATH)/tools/include \ + -c $(filter %.c,$^) -o $@ + +$(OUTPUT)/%.o: %.c | $(OUTPUT) + $(call msg,CC,$@) + $(CC) $(CFLAGS) $(INCLUDES) -I$(KERNEL_PATH)/samples/bpf -c $(filter %.c,$^) -o $@ + +$(APPS): %: $(OUTPUT)/%.o $(OUTPUT)/bpf_load.o | $(OUTPUT) + $(call msg,BINARY,$@) + $(Q)$(CC) $(CFLAGS) $^ $(ALL_LDFLAGS) -I$(KERNEL_PATH)/samples/bpf -lelf -lbpf -lz -o $@ + +.DELETE_ON_ERROR: + +.SECONDARY: diff --git a/src/c/ebpf_collector/bpf_helpers.h b/src/c/ebpf_collector/bpf_helpers.h new file mode 100644 index 0000000000000000000000000000000000000000..352965ab8df24899884631e81dcb97d43d8a8971 --- /dev/null +++ b/src/c/ebpf_collector/bpf_helpers.h @@ -0,0 +1,535 @@ +// SPDX-License-Identifier: GPL-2.0 +#ifndef __BPF_HELPERS__ +#define __BPF_HELPERS__ + +#define __uint(name, val) int (*name)[val] +#define __type(name, val) val *name + +/* helper macro to print out debug messages */ +#define bpf_printk(fmt, ...) \ +({ \ + char ____fmt[] = fmt; \ + bpf_trace_printk(____fmt, sizeof(____fmt), \ + ##__VA_ARGS__); \ +}) + +#ifdef __clang__ + +/* helper macro to place programs, maps, license in + * different sections in elf_bpf file. Section names + * are interpreted by elf_bpf loader + */ +#define SEC(NAME) __attribute__((section(NAME), used)) + +/* helper functions called from eBPF programs written in C */ +static void *(*bpf_map_lookup_elem)(void *map, const void *key) = + (void *) BPF_FUNC_map_lookup_elem; +static int (*bpf_map_update_elem)(void *map, const void *key, const void *value, + unsigned long long flags) = + (void *) BPF_FUNC_map_update_elem; +static int (*bpf_map_delete_elem)(void *map, const void *key) = + (void *) BPF_FUNC_map_delete_elem; +static int (*bpf_map_push_elem)(void *map, const void *value, + unsigned long long flags) = + (void *) BPF_FUNC_map_push_elem; +static int (*bpf_map_pop_elem)(void *map, void *value) = + (void *) BPF_FUNC_map_pop_elem; +static int (*bpf_map_peek_elem)(void *map, void *value) = + (void *) BPF_FUNC_map_peek_elem; +static int (*bpf_probe_read)(void *dst, int size, const void *unsafe_ptr) = + (void *) BPF_FUNC_probe_read; +static unsigned long long (*bpf_ktime_get_ns)(void) = + (void *) BPF_FUNC_ktime_get_ns; +static int (*bpf_trace_printk)(const char *fmt, int fmt_size, ...) = + (void *) BPF_FUNC_trace_printk; +static void (*bpf_tail_call)(void *ctx, void *map, int index) = + (void *) BPF_FUNC_tail_call; +static unsigned long long (*bpf_get_smp_processor_id)(void) = + (void *) BPF_FUNC_get_smp_processor_id; +static unsigned long long (*bpf_get_current_pid_tgid)(void) = + (void *) BPF_FUNC_get_current_pid_tgid; +static unsigned long long (*bpf_get_current_uid_gid)(void) = + (void *) BPF_FUNC_get_current_uid_gid; +static int (*bpf_get_current_comm)(void *buf, int buf_size) = + (void *) BPF_FUNC_get_current_comm; +static unsigned long long (*bpf_perf_event_read)(void *map, + unsigned long long flags) = + (void *) BPF_FUNC_perf_event_read; +static int (*bpf_clone_redirect)(void *ctx, int ifindex, int flags) = + (void *) BPF_FUNC_clone_redirect; +static int (*bpf_redirect)(int ifindex, int flags) = + (void *) BPF_FUNC_redirect; +static int (*bpf_redirect_map)(void *map, int key, int flags) = + (void *) BPF_FUNC_redirect_map; +static int (*bpf_perf_event_output)(void *ctx, void *map, + unsigned long long flags, void *data, + int size) = + (void *) BPF_FUNC_perf_event_output; +static int (*bpf_get_stackid)(void *ctx, void *map, int flags) = + (void *) BPF_FUNC_get_stackid; +static int (*bpf_probe_write_user)(void *dst, const void *src, int size) = + (void *) BPF_FUNC_probe_write_user; +static int (*bpf_current_task_under_cgroup)(void *map, int index) = + (void *) BPF_FUNC_current_task_under_cgroup; +static int (*bpf_skb_get_tunnel_key)(void *ctx, void *key, int size, int flags) = + (void *) BPF_FUNC_skb_get_tunnel_key; +static int (*bpf_skb_set_tunnel_key)(void *ctx, void *key, int size, int flags) = + (void *) BPF_FUNC_skb_set_tunnel_key; +static int (*bpf_skb_get_tunnel_opt)(void *ctx, void *md, int size) = + (void *) BPF_FUNC_skb_get_tunnel_opt; +static int (*bpf_skb_set_tunnel_opt)(void *ctx, void *md, int size) = + (void *) BPF_FUNC_skb_set_tunnel_opt; +static unsigned long long (*bpf_get_prandom_u32)(void) = + (void *) BPF_FUNC_get_prandom_u32; +static int (*bpf_xdp_adjust_head)(void *ctx, int offset) = + (void *) BPF_FUNC_xdp_adjust_head; +static int (*bpf_xdp_adjust_meta)(void *ctx, int offset) = + (void *) BPF_FUNC_xdp_adjust_meta; +static int (*bpf_get_socket_cookie)(void *ctx) = + (void *) BPF_FUNC_get_socket_cookie; +static int (*bpf_setsockopt)(void *ctx, int level, int optname, void *optval, + int optlen) = + (void *) BPF_FUNC_setsockopt; +static int (*bpf_getsockopt)(void *ctx, int level, int optname, void *optval, + int optlen) = + (void *) BPF_FUNC_getsockopt; +static int (*bpf_sock_ops_cb_flags_set)(void *ctx, int flags) = + (void *) BPF_FUNC_sock_ops_cb_flags_set; +static int (*bpf_sk_redirect_map)(void *ctx, void *map, int key, int flags) = + (void *) BPF_FUNC_sk_redirect_map; +static int (*bpf_sk_redirect_hash)(void *ctx, void *map, void *key, int flags) = + (void *) BPF_FUNC_sk_redirect_hash; +static int (*bpf_sock_map_update)(void *map, void *key, void *value, + unsigned long long flags) = + (void *) BPF_FUNC_sock_map_update; +static int (*bpf_sock_hash_update)(void *map, void *key, void *value, + unsigned long long flags) = + (void *) BPF_FUNC_sock_hash_update; +static int (*bpf_perf_event_read_value)(void *map, unsigned long long flags, + void *buf, unsigned int buf_size) = + (void *) BPF_FUNC_perf_event_read_value; +static int (*bpf_perf_prog_read_value)(void *ctx, void *buf, + unsigned int buf_size) = + (void *) BPF_FUNC_perf_prog_read_value; +static int (*bpf_override_return)(void *ctx, unsigned long rc) = + (void *) BPF_FUNC_override_return; +static int (*bpf_msg_redirect_map)(void *ctx, void *map, int key, int flags) = + (void *) BPF_FUNC_msg_redirect_map; +static int (*bpf_msg_redirect_hash)(void *ctx, + void *map, void *key, int flags) = + (void *) BPF_FUNC_msg_redirect_hash; +static int (*bpf_msg_apply_bytes)(void *ctx, int len) = + (void *) BPF_FUNC_msg_apply_bytes; +static int (*bpf_msg_cork_bytes)(void *ctx, int len) = + (void *) BPF_FUNC_msg_cork_bytes; +static int (*bpf_msg_pull_data)(void *ctx, int start, int end, int flags) = + (void *) BPF_FUNC_msg_pull_data; +static int (*bpf_msg_push_data)(void *ctx, int start, int end, int flags) = + (void *) BPF_FUNC_msg_push_data; +static int (*bpf_msg_pop_data)(void *ctx, int start, int cut, int flags) = + (void *) BPF_FUNC_msg_pop_data; +static int (*bpf_bind)(void *ctx, void *addr, int addr_len) = + (void *) BPF_FUNC_bind; +static int (*bpf_xdp_adjust_tail)(void *ctx, int offset) = + (void *) BPF_FUNC_xdp_adjust_tail; +static int (*bpf_skb_get_xfrm_state)(void *ctx, int index, void *state, + int size, int flags) = + (void *) BPF_FUNC_skb_get_xfrm_state; +static int (*bpf_sk_select_reuseport)(void *ctx, void *map, void *key, __u32 flags) = + (void *) BPF_FUNC_sk_select_reuseport; +static int (*bpf_get_stack)(void *ctx, void *buf, int size, int flags) = + (void *) BPF_FUNC_get_stack; +static int (*bpf_fib_lookup)(void *ctx, struct bpf_fib_lookup *params, + int plen, __u32 flags) = + (void *) BPF_FUNC_fib_lookup; +static int (*bpf_lwt_push_encap)(void *ctx, unsigned int type, void *hdr, + unsigned int len) = + (void *) BPF_FUNC_lwt_push_encap; +static int (*bpf_lwt_seg6_store_bytes)(void *ctx, unsigned int offset, + void *from, unsigned int len) = + (void *) BPF_FUNC_lwt_seg6_store_bytes; +static int (*bpf_lwt_seg6_action)(void *ctx, unsigned int action, void *param, + unsigned int param_len) = + (void *) BPF_FUNC_lwt_seg6_action; +static int (*bpf_lwt_seg6_adjust_srh)(void *ctx, unsigned int offset, + unsigned int len) = + (void *) BPF_FUNC_lwt_seg6_adjust_srh; +static int (*bpf_rc_repeat)(void *ctx) = + (void *) BPF_FUNC_rc_repeat; +static int (*bpf_rc_keydown)(void *ctx, unsigned int protocol, + unsigned long long scancode, unsigned int toggle) = + (void *) BPF_FUNC_rc_keydown; +static unsigned long long (*bpf_get_current_cgroup_id)(void) = + (void *) BPF_FUNC_get_current_cgroup_id; +static void *(*bpf_get_local_storage)(void *map, unsigned long long flags) = + (void *) BPF_FUNC_get_local_storage; +static unsigned long long (*bpf_skb_cgroup_id)(void *ctx) = + (void *) BPF_FUNC_skb_cgroup_id; +static unsigned long long (*bpf_skb_ancestor_cgroup_id)(void *ctx, int level) = + (void *) BPF_FUNC_skb_ancestor_cgroup_id; +static struct bpf_sock *(*bpf_sk_lookup_tcp)(void *ctx, + struct bpf_sock_tuple *tuple, + int size, unsigned long long netns_id, + unsigned long long flags) = + (void *) BPF_FUNC_sk_lookup_tcp; +static struct bpf_sock *(*bpf_skc_lookup_tcp)(void *ctx, + struct bpf_sock_tuple *tuple, + int size, unsigned long long netns_id, + unsigned long long flags) = + (void *) BPF_FUNC_skc_lookup_tcp; +static struct bpf_sock *(*bpf_sk_lookup_udp)(void *ctx, + struct bpf_sock_tuple *tuple, + int size, unsigned long long netns_id, + unsigned long long flags) = + (void *) BPF_FUNC_sk_lookup_udp; +static int (*bpf_sk_release)(struct bpf_sock *sk) = + (void *) BPF_FUNC_sk_release; +static int (*bpf_skb_vlan_push)(void *ctx, __be16 vlan_proto, __u16 vlan_tci) = + (void *) BPF_FUNC_skb_vlan_push; +static int (*bpf_skb_vlan_pop)(void *ctx) = + (void *) BPF_FUNC_skb_vlan_pop; +static int (*bpf_rc_pointer_rel)(void *ctx, int rel_x, int rel_y) = + (void *) BPF_FUNC_rc_pointer_rel; +static void (*bpf_spin_lock)(struct bpf_spin_lock *lock) = + (void *) BPF_FUNC_spin_lock; +static void (*bpf_spin_unlock)(struct bpf_spin_lock *lock) = + (void *) BPF_FUNC_spin_unlock; +static struct bpf_sock *(*bpf_sk_fullsock)(struct bpf_sock *sk) = + (void *) BPF_FUNC_sk_fullsock; +static struct bpf_tcp_sock *(*bpf_tcp_sock)(struct bpf_sock *sk) = + (void *) BPF_FUNC_tcp_sock; +static struct bpf_sock *(*bpf_get_listener_sock)(struct bpf_sock *sk) = + (void *) BPF_FUNC_get_listener_sock; +static int (*bpf_skb_ecn_set_ce)(void *ctx) = + (void *) BPF_FUNC_skb_ecn_set_ce; +static int (*bpf_tcp_check_syncookie)(struct bpf_sock *sk, + void *ip, int ip_len, void *tcp, int tcp_len) = + (void *) BPF_FUNC_tcp_check_syncookie; +static int (*bpf_sysctl_get_name)(void *ctx, char *buf, + unsigned long long buf_len, + unsigned long long flags) = + (void *) BPF_FUNC_sysctl_get_name; +static int (*bpf_sysctl_get_current_value)(void *ctx, char *buf, + unsigned long long buf_len) = + (void *) BPF_FUNC_sysctl_get_current_value; +static int (*bpf_sysctl_get_new_value)(void *ctx, char *buf, + unsigned long long buf_len) = + (void *) BPF_FUNC_sysctl_get_new_value; +static int (*bpf_sysctl_set_new_value)(void *ctx, const char *buf, + unsigned long long buf_len) = + (void *) BPF_FUNC_sysctl_set_new_value; +static int (*bpf_strtol)(const char *buf, unsigned long long buf_len, + unsigned long long flags, long *res) = + (void *) BPF_FUNC_strtol; +static int (*bpf_strtoul)(const char *buf, unsigned long long buf_len, + unsigned long long flags, unsigned long *res) = + (void *) BPF_FUNC_strtoul; +static void *(*bpf_sk_storage_get)(void *map, struct bpf_sock *sk, + void *value, __u64 flags) = + (void *) BPF_FUNC_sk_storage_get; +static int (*bpf_sk_storage_delete)(void *map, struct bpf_sock *sk) = + (void *)BPF_FUNC_sk_storage_delete; +static int (*bpf_send_signal)(unsigned sig) = (void *)BPF_FUNC_send_signal; +static long long (*bpf_tcp_gen_syncookie)(struct bpf_sock *sk, void *ip, + int ip_len, void *tcp, int tcp_len) = + (void *) BPF_FUNC_tcp_gen_syncookie; + +/* llvm builtin functions that eBPF C program may use to + * emit BPF_LD_ABS and BPF_LD_IND instructions + */ +struct sk_buff; +unsigned long long load_byte(void *skb, + unsigned long long off) asm("llvm.bpf.load.byte"); +unsigned long long load_half(void *skb, + unsigned long long off) asm("llvm.bpf.load.half"); +unsigned long long load_word(void *skb, + unsigned long long off) asm("llvm.bpf.load.word"); + +/* a helper structure used by eBPF C program + * to describe map attributes to elf_bpf loader + */ +struct bpf_map_def { + unsigned int type; + unsigned int key_size; + unsigned int value_size; + unsigned int max_entries; + unsigned int map_flags; + unsigned int inner_map_idx; + unsigned int numa_node; +}; + +#else + +#include + +#endif + +#define BPF_ANNOTATE_KV_PAIR(name, type_key, type_val) \ + struct ____btf_map_##name { \ + type_key key; \ + type_val value; \ + }; \ + struct ____btf_map_##name \ + __attribute__ ((section(".maps." #name), used)) \ + ____btf_map_##name = { } + +static int (*bpf_skb_load_bytes)(void *ctx, int off, void *to, int len) = + (void *) BPF_FUNC_skb_load_bytes; +static int (*bpf_skb_load_bytes_relative)(void *ctx, int off, void *to, int len, __u32 start_header) = + (void *) BPF_FUNC_skb_load_bytes_relative; +static int (*bpf_skb_store_bytes)(void *ctx, int off, void *from, int len, int flags) = + (void *) BPF_FUNC_skb_store_bytes; +static int (*bpf_l3_csum_replace)(void *ctx, int off, int from, int to, int flags) = + (void *) BPF_FUNC_l3_csum_replace; +static int (*bpf_l4_csum_replace)(void *ctx, int off, int from, int to, int flags) = + (void *) BPF_FUNC_l4_csum_replace; +static int (*bpf_csum_diff)(void *from, int from_size, void *to, int to_size, int seed) = + (void *) BPF_FUNC_csum_diff; +static int (*bpf_skb_under_cgroup)(void *ctx, void *map, int index) = + (void *) BPF_FUNC_skb_under_cgroup; +static int (*bpf_skb_change_head)(void *, int len, int flags) = + (void *) BPF_FUNC_skb_change_head; +static int (*bpf_skb_pull_data)(void *, int len) = + (void *) BPF_FUNC_skb_pull_data; +static unsigned int (*bpf_get_cgroup_classid)(void *ctx) = + (void *) BPF_FUNC_get_cgroup_classid; +static unsigned int (*bpf_get_route_realm)(void *ctx) = + (void *) BPF_FUNC_get_route_realm; +static int (*bpf_skb_change_proto)(void *ctx, __be16 proto, __u64 flags) = + (void *) BPF_FUNC_skb_change_proto; +static int (*bpf_skb_change_type)(void *ctx, __u32 type) = + (void *) BPF_FUNC_skb_change_type; +static unsigned int (*bpf_get_hash_recalc)(void *ctx) = + (void *) BPF_FUNC_get_hash_recalc; +static unsigned long long (*bpf_get_current_task)(void) = + (void *) BPF_FUNC_get_current_task; +static int (*bpf_skb_change_tail)(void *ctx, __u32 len, __u64 flags) = + (void *) BPF_FUNC_skb_change_tail; +static long long (*bpf_csum_update)(void *ctx, __u32 csum) = + (void *) BPF_FUNC_csum_update; +static void (*bpf_set_hash_invalid)(void *ctx) = + (void *) BPF_FUNC_set_hash_invalid; +static int (*bpf_get_numa_node_id)(void) = + (void *) BPF_FUNC_get_numa_node_id; +static int (*bpf_probe_read_str)(void *ctx, __u32 size, + const void *unsafe_ptr) = + (void *) BPF_FUNC_probe_read_str; +static unsigned int (*bpf_get_socket_uid)(void *ctx) = + (void *) BPF_FUNC_get_socket_uid; +static unsigned int (*bpf_set_hash)(void *ctx, __u32 hash) = + (void *) BPF_FUNC_set_hash; +static int (*bpf_skb_adjust_room)(void *ctx, __s32 len_diff, __u32 mode, + unsigned long long flags) = + (void *) BPF_FUNC_skb_adjust_room; + +/* Scan the ARCH passed in from ARCH env variable (see Makefile) */ +#if defined(__TARGET_ARCH_x86) + #define bpf_target_x86 + #define bpf_target_defined +#elif defined(__TARGET_ARCH_s390) + #define bpf_target_s390 + #define bpf_target_defined +#elif defined(__TARGET_ARCH_arm) + #define bpf_target_arm + #define bpf_target_defined +#elif defined(__TARGET_ARCH_arm64) + #define bpf_target_arm64 + #define bpf_target_defined +#elif defined(__TARGET_ARCH_mips) + #define bpf_target_mips + #define bpf_target_defined +#elif defined(__TARGET_ARCH_powerpc) + #define bpf_target_powerpc + #define bpf_target_defined +#elif defined(__TARGET_ARCH_sparc) + #define bpf_target_sparc + #define bpf_target_defined +#else + #undef bpf_target_defined +#endif + +/* Fall back to what the compiler says */ +#ifndef bpf_target_defined +#if defined(__x86_64__) + #define bpf_target_x86 +#elif defined(__s390__) + #define bpf_target_s390 +#elif defined(__arm__) + #define bpf_target_arm +#elif defined(__aarch64__) + #define bpf_target_arm64 +#elif defined(__mips__) + #define bpf_target_mips +#elif defined(__powerpc__) + #define bpf_target_powerpc +#elif defined(__sparc__) + #define bpf_target_sparc +#endif +#endif + +#if defined(bpf_target_x86) + +#ifdef __KERNEL__ +#define PT_REGS_PARM1(x) ((x)->di) +#define PT_REGS_PARM2(x) ((x)->si) +#define PT_REGS_PARM3(x) ((x)->dx) +#define PT_REGS_PARM4(x) ((x)->cx) +#define PT_REGS_PARM5(x) ((x)->r8) +#define PT_REGS_RET(x) ((x)->sp) +#define PT_REGS_FP(x) ((x)->bp) +#define PT_REGS_RC(x) ((x)->ax) +#define PT_REGS_SP(x) ((x)->sp) +#define PT_REGS_IP(x) ((x)->ip) +#else +#ifdef __i386__ +/* i386 kernel is built with -mregparm=3 */ +#define PT_REGS_PARM1(x) ((x)->eax) +#define PT_REGS_PARM2(x) ((x)->edx) +#define PT_REGS_PARM3(x) ((x)->ecx) +#define PT_REGS_PARM4(x) 0 +#define PT_REGS_PARM5(x) 0 +#define PT_REGS_RET(x) ((x)->esp) +#define PT_REGS_FP(x) ((x)->ebp) +#define PT_REGS_RC(x) ((x)->eax) +#define PT_REGS_SP(x) ((x)->esp) +#define PT_REGS_IP(x) ((x)->eip) +#else +#define PT_REGS_PARM1(x) ((x)->rdi) +#define PT_REGS_PARM2(x) ((x)->rsi) +#define PT_REGS_PARM3(x) ((x)->rdx) +#define PT_REGS_PARM4(x) ((x)->rcx) +#define PT_REGS_PARM5(x) ((x)->r8) +#define PT_REGS_RET(x) ((x)->rsp) +#define PT_REGS_FP(x) ((x)->rbp) +#define PT_REGS_RC(x) ((x)->rax) +#define PT_REGS_SP(x) ((x)->rsp) +#define PT_REGS_IP(x) ((x)->rip) +#endif +#endif + +#elif defined(bpf_target_s390) + +/* s390 provides user_pt_regs instead of struct pt_regs to userspace */ +struct pt_regs; +#define PT_REGS_S390 const volatile user_pt_regs +#define PT_REGS_PARM1(x) (((PT_REGS_S390 *)(x))->gprs[2]) +#define PT_REGS_PARM2(x) (((PT_REGS_S390 *)(x))->gprs[3]) +#define PT_REGS_PARM3(x) (((PT_REGS_S390 *)(x))->gprs[4]) +#define PT_REGS_PARM4(x) (((PT_REGS_S390 *)(x))->gprs[5]) +#define PT_REGS_PARM5(x) (((PT_REGS_S390 *)(x))->gprs[6]) +#define PT_REGS_RET(x) (((PT_REGS_S390 *)(x))->gprs[14]) +/* Works only with CONFIG_FRAME_POINTER */ +#define PT_REGS_FP(x) (((PT_REGS_S390 *)(x))->gprs[11]) +#define PT_REGS_RC(x) (((PT_REGS_S390 *)(x))->gprs[2]) +#define PT_REGS_SP(x) (((PT_REGS_S390 *)(x))->gprs[15]) +#define PT_REGS_IP(x) (((PT_REGS_S390 *)(x))->psw.addr) + +#elif defined(bpf_target_arm) + +#define PT_REGS_PARM1(x) ((x)->uregs[0]) +#define PT_REGS_PARM2(x) ((x)->uregs[1]) +#define PT_REGS_PARM3(x) ((x)->uregs[2]) +#define PT_REGS_PARM4(x) ((x)->uregs[3]) +#define PT_REGS_PARM5(x) ((x)->uregs[4]) +#define PT_REGS_RET(x) ((x)->uregs[14]) +#define PT_REGS_FP(x) ((x)->uregs[11]) /* Works only with CONFIG_FRAME_POINTER */ +#define PT_REGS_RC(x) ((x)->uregs[0]) +#define PT_REGS_SP(x) ((x)->uregs[13]) +#define PT_REGS_IP(x) ((x)->uregs[12]) + +#elif defined(bpf_target_arm64) + +/* arm64 provides struct user_pt_regs instead of struct pt_regs to userspace */ +struct pt_regs; +#define PT_REGS_ARM64 const volatile struct user_pt_regs +#define PT_REGS_PARM1(x) (((PT_REGS_ARM64 *)(x))->regs[0]) +#define PT_REGS_PARM2(x) (((PT_REGS_ARM64 *)(x))->regs[1]) +#define PT_REGS_PARM3(x) (((PT_REGS_ARM64 *)(x))->regs[2]) +#define PT_REGS_PARM4(x) (((PT_REGS_ARM64 *)(x))->regs[3]) +#define PT_REGS_PARM5(x) (((PT_REGS_ARM64 *)(x))->regs[4]) +#define PT_REGS_RET(x) (((PT_REGS_ARM64 *)(x))->regs[30]) +/* Works only with CONFIG_FRAME_POINTER */ +#define PT_REGS_FP(x) (((PT_REGS_ARM64 *)(x))->regs[29]) +#define PT_REGS_RC(x) (((PT_REGS_ARM64 *)(x))->regs[0]) +#define PT_REGS_SP(x) (((PT_REGS_ARM64 *)(x))->sp) +#define PT_REGS_IP(x) (((PT_REGS_ARM64 *)(x))->pc) + +#elif defined(bpf_target_mips) + +#define PT_REGS_PARM1(x) ((x)->regs[4]) +#define PT_REGS_PARM2(x) ((x)->regs[5]) +#define PT_REGS_PARM3(x) ((x)->regs[6]) +#define PT_REGS_PARM4(x) ((x)->regs[7]) +#define PT_REGS_PARM5(x) ((x)->regs[8]) +#define PT_REGS_RET(x) ((x)->regs[31]) +#define PT_REGS_FP(x) ((x)->regs[30]) /* Works only with CONFIG_FRAME_POINTER */ +#define PT_REGS_RC(x) ((x)->regs[1]) +#define PT_REGS_SP(x) ((x)->regs[29]) +#define PT_REGS_IP(x) ((x)->cp0_epc) + +#elif defined(bpf_target_powerpc) + +#define PT_REGS_PARM1(x) ((x)->gpr[3]) +#define PT_REGS_PARM2(x) ((x)->gpr[4]) +#define PT_REGS_PARM3(x) ((x)->gpr[5]) +#define PT_REGS_PARM4(x) ((x)->gpr[6]) +#define PT_REGS_PARM5(x) ((x)->gpr[7]) +#define PT_REGS_RC(x) ((x)->gpr[3]) +#define PT_REGS_SP(x) ((x)->sp) +#define PT_REGS_IP(x) ((x)->nip) + +#elif defined(bpf_target_sparc) + +#define PT_REGS_PARM1(x) ((x)->u_regs[UREG_I0]) +#define PT_REGS_PARM2(x) ((x)->u_regs[UREG_I1]) +#define PT_REGS_PARM3(x) ((x)->u_regs[UREG_I2]) +#define PT_REGS_PARM4(x) ((x)->u_regs[UREG_I3]) +#define PT_REGS_PARM5(x) ((x)->u_regs[UREG_I4]) +#define PT_REGS_RET(x) ((x)->u_regs[UREG_I7]) +#define PT_REGS_RC(x) ((x)->u_regs[UREG_I0]) +#define PT_REGS_SP(x) ((x)->u_regs[UREG_FP]) + +/* Should this also be a bpf_target check for the sparc case? */ +#if defined(__arch64__) +#define PT_REGS_IP(x) ((x)->tpc) +#else +#define PT_REGS_IP(x) ((x)->pc) +#endif + +#endif + +#if defined(bpf_target_powerpc) +#define BPF_KPROBE_READ_RET_IP(ip, ctx) ({ (ip) = (ctx)->link; }) +#define BPF_KRETPROBE_READ_RET_IP BPF_KPROBE_READ_RET_IP +#elif defined(bpf_target_sparc) +#define BPF_KPROBE_READ_RET_IP(ip, ctx) ({ (ip) = PT_REGS_RET(ctx); }) +#define BPF_KRETPROBE_READ_RET_IP BPF_KPROBE_READ_RET_IP +#else +#define BPF_KPROBE_READ_RET_IP(ip, ctx) ({ \ + bpf_probe_read(&(ip), sizeof(ip), (void *)PT_REGS_RET(ctx)); }) +#define BPF_KRETPROBE_READ_RET_IP(ip, ctx) ({ \ + bpf_probe_read(&(ip), sizeof(ip), \ + (void *)(PT_REGS_FP(ctx) + sizeof(ip))); }) +#endif + +/* + * BPF_CORE_READ abstracts away bpf_probe_read() call and captures offset + * relocation for source address using __builtin_preserve_access_index() + * built-in, provided by Clang. + * + * __builtin_preserve_access_index() takes as an argument an expression of + * taking an address of a field within struct/union. It makes compiler emit + * a relocation, which records BTF type ID describing root struct/union and an + * accessor string which describes exact embedded field that was used to take + * an address. See detailed description of this relocation format and + * semantics in comments to struct bpf_offset_reloc in libbpf_internal.h. + * + * This relocation allows libbpf to adjust BPF instruction to use correct + * actual field offset, based on target kernel BTF type that matches original + * (local) BTF, used to record relocation. + */ +#define BPF_CORE_READ(dst, src) \ + bpf_probe_read((dst), sizeof(*(src)), \ + __builtin_preserve_access_index(src)) + +#endif diff --git a/src/c/ebpf_collector/bpf_load.c b/src/c/ebpf_collector/bpf_load.c new file mode 100644 index 0000000000000000000000000000000000000000..db33eb12b323dacdabe411134443053ac41de062 --- /dev/null +++ b/src/c/ebpf_collector/bpf_load.c @@ -0,0 +1,709 @@ +// SPDX-License-Identifier: GPL-2.0 +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#define DEBUGFS "/sys/kernel/debug/tracing/" + +static char license[128]; +static int kern_version; +static bool processed_sec[128]; +char bpf_log_buf[BPF_LOG_BUF_SIZE]; +int map_fd[MAX_MAPS]; +int prog_fd[MAX_PROGS]; +int event_fd[MAX_PROGS]; +int prog_cnt; +int prog_array_fd = -1; + +struct bpf_map_data map_data[MAX_MAPS]; +int map_data_count = 0; + +static int populate_prog_array(const char *event, int prog_fd) +{ + int ind = atoi(event), err; + + err = bpf_map_update_elem(prog_array_fd, &ind, &prog_fd, BPF_ANY); + if (err < 0) { + printf("failed to store prog_fd in prog_array\n"); + return -1; + } + return 0; +} + +static int write_kprobe_events(const char *val) +{ + int fd, ret, flags; + + if (val == NULL) + return -1; + else if (val[0] == '\0') + flags = O_WRONLY | O_TRUNC; + else + flags = O_WRONLY | O_APPEND; + + fd = open("/sys/kernel/debug/tracing/kprobe_events", flags); + + ret = write(fd, val, strlen(val)); + close(fd); + + return ret; +} + +static int load_and_attach(const char *event, struct bpf_insn *prog, int size) +{ + bool is_socket = strncmp(event, "socket", 6) == 0; + bool is_kprobe = strncmp(event, "kprobe/", 7) == 0; + bool is_kretprobe = strncmp(event, "kretprobe/", 10) == 0; + bool is_tracepoint = strncmp(event, "tracepoint/", 11) == 0; + bool is_raw_tracepoint = strncmp(event, "raw_tracepoint/", 15) == 0; + bool is_xdp = strncmp(event, "xdp", 3) == 0; + bool is_perf_event = strncmp(event, "perf_event", 10) == 0; + bool is_cgroup_skb = strncmp(event, "cgroup/skb", 10) == 0; + bool is_cgroup_sk = strncmp(event, "cgroup/sock", 11) == 0; + bool is_sockops = strncmp(event, "sockops", 7) == 0; + bool is_sk_skb = strncmp(event, "sk_skb", 6) == 0; + bool is_sk_msg = strncmp(event, "sk_msg", 6) == 0; + size_t insns_cnt = size / sizeof(struct bpf_insn); + enum bpf_prog_type prog_type; + char buf[256]; + int fd, efd, err, id; + struct perf_event_attr attr = {}; + + attr.type = PERF_TYPE_TRACEPOINT; + attr.sample_type = PERF_SAMPLE_RAW; + attr.sample_period = 1; + attr.wakeup_events = 1; + + if (is_socket) { + prog_type = BPF_PROG_TYPE_SOCKET_FILTER; + } else if (is_kprobe || is_kretprobe) { + prog_type = BPF_PROG_TYPE_KPROBE; + } else if (is_tracepoint) { + prog_type = BPF_PROG_TYPE_TRACEPOINT; + } else if (is_raw_tracepoint) { + prog_type = BPF_PROG_TYPE_RAW_TRACEPOINT; + } else if (is_xdp) { + prog_type = BPF_PROG_TYPE_XDP; + } else if (is_perf_event) { + prog_type = BPF_PROG_TYPE_PERF_EVENT; + } else if (is_cgroup_skb) { + prog_type = BPF_PROG_TYPE_CGROUP_SKB; + } else if (is_cgroup_sk) { + prog_type = BPF_PROG_TYPE_CGROUP_SOCK; + } else if (is_sockops) { + prog_type = BPF_PROG_TYPE_SOCK_OPS; + } else if (is_sk_skb) { + prog_type = BPF_PROG_TYPE_SK_SKB; + } else if (is_sk_msg) { + prog_type = BPF_PROG_TYPE_SK_MSG; + } else { + printf("Unknown event '%s'\n", event); + return -1; + } + + if (prog_cnt == MAX_PROGS) + return -1; + + fd = bpf_load_program(prog_type, prog, insns_cnt, license, kern_version, + bpf_log_buf, BPF_LOG_BUF_SIZE); + if (fd < 0) { + printf("bpf_load_program() err=%d\n%s", errno, bpf_log_buf); + return -1; + } + + prog_fd[prog_cnt++] = fd; + + if (is_xdp || is_perf_event || is_cgroup_skb || is_cgroup_sk) + return 0; + + if (is_socket || is_sockops || is_sk_skb || is_sk_msg) { + if (is_socket) + event += 6; + else + event += 7; + if (*event != '/') + return 0; + event++; + if (!isdigit(*event)) { + printf("invalid prog number\n"); + return -1; + } + return populate_prog_array(event, fd); + } + + if (is_raw_tracepoint) { + efd = bpf_raw_tracepoint_open(event + 15, fd); + if (efd < 0) { + printf("tracepoint %s %s\n", event + 15, strerror(errno)); + return -1; + } + event_fd[prog_cnt - 1] = efd; + return 0; + } + + if (is_kprobe || is_kretprobe) { + bool need_normal_check = true; + const char *event_prefix = ""; + + if (is_kprobe) + event += 7; + else + event += 10; + + if (*event == 0) { + printf("event name cannot be empty\n"); + return -1; + } + + if (isdigit(*event)) + return populate_prog_array(event, fd); + +#ifdef __x86_64__ + if (strncmp(event, "sys_", 4) == 0) { + snprintf(buf, sizeof(buf), "%c:__x64_%s __x64_%s", + is_kprobe ? 'p' : 'r', event, event); + err = write_kprobe_events(buf); + if (err >= 0) { + need_normal_check = false; + event_prefix = "__x64_"; + } + } +#endif + if (need_normal_check) { + if (strcmp("wbt_wait", event) == 0 || strcmp("blk_mq_get_tag", event) == 0) { + if (is_kprobe) { + snprintf(buf, sizeof(buf), "%c:%s_1 %s", + is_kprobe ? 'p' : 'r', event, event); + } + else { + snprintf(buf, sizeof(buf), "%c:%s_2 %s", + is_kprobe ? 'p' : 'r', event, event); + } + } + else { + snprintf(buf, sizeof(buf), "%c:%s %s", + is_kprobe ? 'p' : 'r', event, event); + } + err = write_kprobe_events(buf); + if (err < 0) { + printf("failed to create kprobe '%s' error '%s'\n", + event, strerror(errno)); + return -1; + } + } + + strcpy(buf, DEBUGFS); + strcat(buf, "events/kprobes/"); + strcat(buf, event_prefix); + strcat(buf, event); + + if (strcmp("wbt_wait", event) == 0 || strcmp("blk_mq_get_tag", event) == 0) { + if (is_kprobe) { + strcat(buf, "_1"); + } + else { + strcat(buf, "_2"); + } + } + strcat(buf, "/id"); + } else if (is_tracepoint) { + event += 11; + + if (*event == 0) { + printf("event name cannot be empty\n"); + return -1; + } + strcpy(buf, DEBUGFS); + strcat(buf, "events/"); + strcat(buf, event); + strcat(buf, "/id"); + } + + efd = open(buf, O_RDONLY, 0); + if (efd < 0) { + printf("failed to open event %s\n", event); + return -1; + } + + err = read(efd, buf, sizeof(buf)); + if (err < 0 || err >= sizeof(buf)) { + printf("read from '%s' failed '%s'\n", event, strerror(errno)); + return -1; + } + + close(efd); + + buf[err] = 0; + id = atoi(buf); + attr.config = id; + + efd = sys_perf_event_open(&attr, -1/*pid*/, 0/*cpu*/, -1/*group_fd*/, 0); + if (efd < 0) { + printf("event %d fd %d err %s\n", id, efd, strerror(errno)); + return -1; + } + event_fd[prog_cnt - 1] = efd; + err = ioctl(efd, PERF_EVENT_IOC_ENABLE, 0); + if (err < 0) { + printf("ioctl PERF_EVENT_IOC_ENABLE failed err %s\n", + strerror(errno)); + return -1; + } + err = ioctl(efd, PERF_EVENT_IOC_SET_BPF, fd); + if (err < 0) { + printf("ioctl PERF_EVENT_IOC_SET_BPF failed err %s\n", + strerror(errno)); + return -1; + } + + return 0; +} + +static int load_maps(struct bpf_map_data *maps, int nr_maps, + fixup_map_cb fixup_map) +{ + int i, numa_node; + + for (i = 0; i < nr_maps; i++) { + if (fixup_map) { + fixup_map(&maps[i], i); + /* Allow userspace to assign map FD prior to creation */ + if (maps[i].fd != -1) { + map_fd[i] = maps[i].fd; + continue; + } + } + + numa_node = maps[i].def.map_flags & BPF_F_NUMA_NODE ? + maps[i].def.numa_node : -1; + + if (maps[i].def.type == BPF_MAP_TYPE_ARRAY_OF_MAPS || + maps[i].def.type == BPF_MAP_TYPE_HASH_OF_MAPS) { + int inner_map_fd = map_fd[maps[i].def.inner_map_idx]; + + map_fd[i] = bpf_create_map_in_map_node(maps[i].def.type, + maps[i].name, + maps[i].def.key_size, + inner_map_fd, + maps[i].def.max_entries, + maps[i].def.map_flags, + numa_node); + } else { + map_fd[i] = bpf_create_map_node(maps[i].def.type, + maps[i].name, + maps[i].def.key_size, + maps[i].def.value_size, + maps[i].def.max_entries, + maps[i].def.map_flags, + numa_node); + } + if (map_fd[i] < 0) { + printf("failed to create a map: %d %s\n", + errno, strerror(errno)); + return 1; + } + maps[i].fd = map_fd[i]; + + if (maps[i].def.type == BPF_MAP_TYPE_PROG_ARRAY) + prog_array_fd = map_fd[i]; + } + return 0; +} + +static int get_sec(Elf *elf, int i, GElf_Ehdr *ehdr, char **shname, + GElf_Shdr *shdr, Elf_Data **data) +{ + Elf_Scn *scn; + + scn = elf_getscn(elf, i); + if (!scn) + return 1; + + if (gelf_getshdr(scn, shdr) != shdr) + return 2; + + *shname = elf_strptr(elf, ehdr->e_shstrndx, shdr->sh_name); + if (!*shname || !shdr->sh_size) + return 3; + + *data = elf_getdata(scn, 0); + if (!*data || elf_getdata(scn, *data) != NULL) + return 4; + + return 0; +} + +static int parse_relo_and_apply(Elf_Data *data, Elf_Data *symbols, + GElf_Shdr *shdr, struct bpf_insn *insn, + struct bpf_map_data *maps, int nr_maps) +{ + int i, nrels; + + nrels = shdr->sh_size / shdr->sh_entsize; + + for (i = 0; i < nrels; i++) { + GElf_Sym sym; + GElf_Rel rel; + unsigned int insn_idx; + bool match = false; + int j, map_idx; + + gelf_getrel(data, i, &rel); + + insn_idx = rel.r_offset / sizeof(struct bpf_insn); + + gelf_getsym(symbols, GELF_R_SYM(rel.r_info), &sym); + + if (insn[insn_idx].code != (BPF_LD | BPF_IMM | BPF_DW)) { + printf("invalid relo for insn[%d].code 0x%x\n", + insn_idx, insn[insn_idx].code); + return 1; + } + insn[insn_idx].src_reg = BPF_PSEUDO_MAP_FD; + + /* Match FD relocation against recorded map_data[] offset */ + for (map_idx = 0; map_idx < nr_maps; map_idx++) { + if (maps[map_idx].elf_offset == sym.st_value) { + match = true; + break; + } + } + if (match) { + insn[insn_idx].imm = maps[map_idx].fd; + } else { + printf("invalid relo for insn[%d] no map_data match\n", + insn_idx); + return 1; + } + } + + return 0; +} + +static int cmp_symbols(const void *l, const void *r) +{ + const GElf_Sym *lsym = (const GElf_Sym *)l; + const GElf_Sym *rsym = (const GElf_Sym *)r; + + if (lsym->st_value < rsym->st_value) + return -1; + else if (lsym->st_value > rsym->st_value) + return 1; + else + return 0; +} + +static int load_elf_maps_section(struct bpf_map_data *maps, int maps_shndx, + Elf *elf, Elf_Data *symbols, int strtabidx) +{ + int map_sz_elf, map_sz_copy; + bool validate_zero = false; + Elf_Data *data_maps; + int i, nr_maps; + GElf_Sym *sym; + Elf_Scn *scn; + int copy_sz; + + if (maps_shndx < 0) + return -EINVAL; + if (!symbols) + return -EINVAL; + + /* Get data for maps section via elf index */ + scn = elf_getscn(elf, maps_shndx); + if (scn) + data_maps = elf_getdata(scn, NULL); + if (!scn || !data_maps) { + printf("Failed to get Elf_Data from maps section %d\n", + maps_shndx); + return -EINVAL; + } + + /* For each map get corrosponding symbol table entry */ + sym = calloc(MAX_MAPS+1, sizeof(GElf_Sym)); + for (i = 0, nr_maps = 0; i < symbols->d_size / sizeof(GElf_Sym); i++) { + assert(nr_maps < MAX_MAPS+1); + if (!gelf_getsym(symbols, i, &sym[nr_maps])) + continue; + if (sym[nr_maps].st_shndx != maps_shndx) + continue; + /* Only increment iif maps section */ + nr_maps++; + } + + /* Align to map_fd[] order, via sort on offset in sym.st_value */ + qsort(sym, nr_maps, sizeof(GElf_Sym), cmp_symbols); + + /* Keeping compatible with ELF maps section changes + * ------------------------------------------------ + * The program size of struct bpf_load_map_def is known by loader + * code, but struct stored in ELF file can be different. + * + * Unfortunately sym[i].st_size is zero. To calculate the + * struct size stored in the ELF file, assume all struct have + * the same size, and simply divide with number of map + * symbols. + */ + map_sz_elf = data_maps->d_size / nr_maps; + map_sz_copy = sizeof(struct bpf_load_map_def); + if (map_sz_elf < map_sz_copy) { + /* + * Backward compat, loading older ELF file with + * smaller struct, keeping remaining bytes zero. + */ + map_sz_copy = map_sz_elf; + } else if (map_sz_elf > map_sz_copy) { + /* + * Forward compat, loading newer ELF file with larger + * struct with unknown features. Assume zero means + * feature not used. Thus, validate rest of struct + * data is zero. + */ + validate_zero = true; + } + + /* Memcpy relevant part of ELF maps data to loader maps */ + for (i = 0; i < nr_maps; i++) { + struct bpf_load_map_def *def; + unsigned char *addr, *end; + const char *map_name; + size_t offset; + + map_name = elf_strptr(elf, strtabidx, sym[i].st_name); + maps[i].name = strdup(map_name); + if (!maps[i].name) { + printf("strdup(%s): %s(%d)\n", map_name, + strerror(errno), errno); + free(sym); + return -errno; + } + + /* Symbol value is offset into ELF maps section data area */ + offset = sym[i].st_value; + def = (struct bpf_load_map_def *)(data_maps->d_buf + offset); + maps[i].elf_offset = offset; + memset(&maps[i].def, 0, sizeof(struct bpf_load_map_def)); + memcpy(&maps[i].def, def, map_sz_copy); + + /* Verify no newer features were requested */ + if (validate_zero) { + addr = (unsigned char*) def + map_sz_copy; + end = (unsigned char*) def + map_sz_elf; + for (; addr < end; addr++) { + if (*addr != 0) { + free(sym); + return -EFBIG; + } + } + } + } + + free(sym); + return nr_maps; +} + +static int do_load_bpf_file(const char *path, fixup_map_cb fixup_map) +{ + int fd, i, ret, maps_shndx = -1, strtabidx = -1; + Elf *elf; + GElf_Ehdr ehdr; + GElf_Shdr shdr, shdr_prog; + Elf_Data *data, *data_prog, *data_maps = NULL, *symbols = NULL; + char *shname, *shname_prog; + int nr_maps = 0; + + /* reset global variables */ + kern_version = 0; + memset(license, 0, sizeof(license)); + memset(processed_sec, 0, sizeof(processed_sec)); + + if (elf_version(EV_CURRENT) == EV_NONE) + return 1; + + fd = open(path, O_RDONLY, 0); + if (fd < 0) + return 1; + + elf = elf_begin(fd, ELF_C_READ, NULL); + + if (!elf) + return 1; + + if (gelf_getehdr(elf, &ehdr) != &ehdr) + return 1; + + /* clear all kprobes */ + i = write_kprobe_events(""); + + /* scan over all elf sections to get license and map info */ + for (i = 1; i < ehdr.e_shnum; i++) { + + if (get_sec(elf, i, &ehdr, &shname, &shdr, &data)) + continue; + + if (0) /* helpful for llvm debugging */ + printf("section %d:%s data %p size %zd link %d flags %d\n", + i, shname, data->d_buf, data->d_size, + shdr.sh_link, (int) shdr.sh_flags); + + if (strcmp(shname, "license") == 0) { + processed_sec[i] = true; + memcpy(license, data->d_buf, data->d_size); + } else if (strcmp(shname, "version") == 0) { + processed_sec[i] = true; + if (data->d_size != sizeof(int)) { + printf("invalid size of version section %zd\n", + data->d_size); + return 1; + } + memcpy(&kern_version, data->d_buf, sizeof(int)); + } else if (strcmp(shname, "maps") == 0) { + int j; + + maps_shndx = i; + data_maps = data; + for (j = 0; j < MAX_MAPS; j++) + map_data[j].fd = -1; + } else if (shdr.sh_type == SHT_SYMTAB) { + strtabidx = shdr.sh_link; + symbols = data; + } + } + + ret = 1; + + if (!symbols) { + printf("missing SHT_SYMTAB section\n"); + goto done; + } + + if (data_maps) { + nr_maps = load_elf_maps_section(map_data, maps_shndx, + elf, symbols, strtabidx); + if (nr_maps < 0) { + printf("Error: Failed loading ELF maps (errno:%d):%s\n", + nr_maps, strerror(-nr_maps)); + goto done; + } + if (load_maps(map_data, nr_maps, fixup_map)) + goto done; + map_data_count = nr_maps; + + processed_sec[maps_shndx] = true; + } + + /* process all relo sections, and rewrite bpf insns for maps */ + for (i = 1; i < ehdr.e_shnum; i++) { + if (processed_sec[i]) + continue; + + if (get_sec(elf, i, &ehdr, &shname, &shdr, &data)) + continue; + + if (shdr.sh_type == SHT_REL) { + struct bpf_insn *insns; + + /* locate prog sec that need map fixup (relocations) */ + if (get_sec(elf, shdr.sh_info, &ehdr, &shname_prog, + &shdr_prog, &data_prog)) + continue; + + if (shdr_prog.sh_type != SHT_PROGBITS || + !(shdr_prog.sh_flags & SHF_EXECINSTR)) + continue; + + insns = (struct bpf_insn *) data_prog->d_buf; + processed_sec[i] = true; /* relo section */ + + if (parse_relo_and_apply(data, symbols, &shdr, insns, + map_data, nr_maps)) + continue; + } + } + + /* load programs */ + for (i = 1; i < ehdr.e_shnum; i++) { + + if (processed_sec[i]) + continue; + + if (get_sec(elf, i, &ehdr, &shname, &shdr, &data)) + continue; + + if (memcmp(shname, "kprobe/", 7) == 0 || + memcmp(shname, "kretprobe/", 10) == 0 || + memcmp(shname, "tracepoint/", 11) == 0 || + memcmp(shname, "raw_tracepoint/", 15) == 0 || + memcmp(shname, "xdp", 3) == 0 || + memcmp(shname, "perf_event", 10) == 0 || + memcmp(shname, "socket", 6) == 0 || + memcmp(shname, "cgroup/", 7) == 0 || + memcmp(shname, "sockops", 7) == 0 || + memcmp(shname, "sk_skb", 6) == 0 || + memcmp(shname, "sk_msg", 6) == 0) { + ret = load_and_attach(shname, data->d_buf, + data->d_size); + if (ret != 0) + goto done; + } + } + +done: + close(fd); + return ret; +} + +int load_bpf_file(char *path) +{ + return do_load_bpf_file(path, NULL); +} + +int load_bpf_file_fixup_map(const char *path, fixup_map_cb fixup_map) +{ + return do_load_bpf_file(path, fixup_map); +} + +void read_trace_pipe(void) +{ + int trace_fd; + + trace_fd = open(DEBUGFS "trace_pipe", O_RDONLY, 0); + if (trace_fd < 0) + return; + + while (1) { + static char buf[4096]; + ssize_t sz; + + sz = read(trace_fd, buf, sizeof(buf) - 1); + if (sz > 0) { + buf[sz] = 0; + puts(buf); + } + } +} diff --git a/src/c/ebpf_collector/ebpf_collector.bpf.c b/src/c/ebpf_collector/ebpf_collector.bpf.c new file mode 100644 index 0000000000000000000000000000000000000000..a7ea51b91123cd3334ff2c5f0229ad20801439a4 --- /dev/null +++ b/src/c/ebpf_collector/ebpf_collector.bpf.c @@ -0,0 +1,1545 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2024. All rights reserved. + * Description: ebpf collector program + * Author: Zhang Nan + * Create: 2024-09-27 + */ +#define KBUILD_MODNAME "foo" + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "bpf_helpers.h" +#include "ebpf_collector.h" + +#define _(P) ({typeof(P) val; bpf_probe_read(&val, sizeof(val), &P); val;}) + +struct bpf_map_def SEC("maps") blk_map = { + .type = BPF_MAP_TYPE_HASH, + .key_size = sizeof(u32), + .value_size = sizeof(struct io_counter), + .max_entries = 10000, +}; + +struct bpf_map_def SEC("maps") blk_res = { + .type = BPF_MAP_TYPE_ARRAY, + .key_size = sizeof(u32), + .value_size = sizeof(struct stage_data), + .max_entries = 128, +}; + +struct bpf_map_def SEC("maps") bio_map = { + .type = BPF_MAP_TYPE_HASH, + .key_size = sizeof(u32), + .value_size = sizeof(struct io_counter), + .max_entries = 10000, +}; + +struct bpf_map_def SEC("maps") bio_res = { + .type = BPF_MAP_TYPE_ARRAY, + .key_size = sizeof(u32), + .value_size = sizeof(struct stage_data), + .max_entries = 128, +}; + +struct bpf_map_def SEC("maps") wbt_map = { + .type = BPF_MAP_TYPE_HASH, + .key_size = sizeof(u32), + .value_size = sizeof(struct io_counter), + .max_entries = 10000, +}; + +struct bpf_map_def SEC("maps") wbt_res = { + .type = BPF_MAP_TYPE_ARRAY, + .key_size = sizeof(u32), + .value_size = sizeof(struct stage_data), + .max_entries = 128, +}; + +struct bpf_map_def SEC("maps") wbt_args = { + .type = BPF_MAP_TYPE_HASH, + .key_size = sizeof(u32), + .value_size = sizeof(u64), + .max_entries = 1000, +}; + +struct bpf_map_def SEC("maps") tag_map = { + .type = BPF_MAP_TYPE_HASH, + .key_size = sizeof(u32), + .value_size = sizeof(struct io_counter), + .max_entries = 10000, +}; + +struct bpf_map_def SEC("maps") tag_res = { + .type = BPF_MAP_TYPE_ARRAY, + .key_size = sizeof(u32), + .value_size = sizeof(struct stage_data), + .max_entries = 128, +}; + +struct bpf_map_def SEC("maps") tag_args = { + .type = BPF_MAP_TYPE_HASH, + .key_size = sizeof(u32), + .value_size = sizeof(u64), + .max_entries = 1000, +}; + +struct bpf_map_def SEC("maps") blk_res_2 = { + .type = BPF_MAP_TYPE_HASH, + .key_size = sizeof(u64), + .value_size = sizeof(struct time_range_io_count), + .max_entries = MAX_IO_TIME, +}; + +struct bpf_map_def SEC("maps") bio_res_2 = { + .type = BPF_MAP_TYPE_HASH, + .key_size = sizeof(u64), + .value_size = sizeof(struct time_range_io_count), + .max_entries = MAX_IO_TIME, +}; + +struct bpf_map_def SEC("maps") wbt_res_2 = { + .type = BPF_MAP_TYPE_HASH, + .key_size = sizeof(u64), + .value_size = sizeof(struct time_range_io_count), + .max_entries = MAX_IO_TIME, +}; + +struct bpf_map_def SEC("maps") tag_res_2 = { + .type = BPF_MAP_TYPE_HASH, + .key_size = sizeof(u64), + .value_size = sizeof(struct time_range_io_count), + .max_entries = MAX_IO_TIME, +}; + +struct blk_mq_alloc_data { + /* input parameter */ + struct request_queue *q; + blk_mq_req_flags_t flags; + unsigned int shallow_depth; + + /* input & output parameter */ + struct blk_mq_ctx *ctx; + struct blk_mq_hw_ctx *hctx; +}; + +struct request_kylin { + struct request_queue *q; + struct blk_mq_ctx *mq_ctx; + + int cpu; + unsigned int cmd_flags; /* op and common flags */ + req_flags_t rq_flags; + + int internal_tag; + + /* the following two fields are internal, NEVER access directly */ + unsigned int __data_len; /* total data len */ + int tag; + sector_t __sector; /* sector cursor */ + + struct bio *bio; + struct bio *biotail; + + struct list_head queuelist; + + /* + * The hash is used inside the scheduler, and killed once the + * request reaches the dispatch list. The ipi_list is only used + * to queue the request for softirq completion, which is long + * after the request has been unhashed (and even removed from + * the dispatch list). + */ + union { + struct hlist_node hash; /* merge hash */ + struct list_head ipi_list; + }; + + struct hlist_node front_hash; /* front merge hash */ + + /* + * The rb_node is only used inside the io scheduler, requests + * are pruned when moved to the dispatch queue. So let the + * completion_data share space with the rb_node. + */ + union { + struct rb_node rb_node; /* sort/lookup */ + struct bio_vec special_vec; + void *completion_data; + int error_count; /* for legacy drivers, don't use */ + }; + + /* + * Three pointers are available for the IO schedulers, if they need + * more they have to dynamically allocate it. Flush requests are + * never put on the IO scheduler. So let the flush fields share + * space with the elevator data. + */ + union { + struct { + struct io_cq *icq; + void *priv[2]; + } elv; + + struct { + unsigned int seq; + struct list_head list; + rq_end_io_fn *saved_end_io; + } flush; + }; + + struct gendisk *rq_disk; + struct hd_struct *part; + /* Time that I/O was submitted to the kernel. */ + u64 start_time_ns; + /* Time that I/O was submitted to the device. */ + u64 io_start_time_ns; + +#ifdef CONFIG_BLK_WBT + unsigned short wbt_flags; +#endif +#ifdef CONFIG_BLK_DEV_THROTTLING_LOW + unsigned short throtl_size; +#endif + + /* + * Number of scatter-gather DMA addr+len pairs after + * physical address coalescing is performed. + */ + unsigned short nr_phys_segments; + +#if defined(CONFIG_BLK_DEV_INTEGRITY) + unsigned short nr_integrity_segments; +#endif + + unsigned short write_hint; + unsigned short ioprio; + + void *special; /* opaque pointer available for LLD use */ + + unsigned int extra_len; /* length of alignment and padding */ + + enum mq_rq_state state; + refcount_t ref; + + unsigned int timeout; + + /* access through blk_rq_set_deadline, blk_rq_deadline */ + unsigned long __deadline; + + struct list_head timeout_list; + + union { + struct __call_single_data csd; + u64 fifo_time; + }; + + /* + * completion callback. + */ + rq_end_io_fn *end_io; + void *end_io_data; + + /* for bidi */ + struct request_kylin *next_rq; + +#ifdef CONFIG_BLK_CGROUP + struct request_list *rl; /* rl this rq is alloced from */ +#endif + KABI_RESERVE(1); + KABI_RESERVE(2); +}; + +static __always_inline void blk_fill_rwbs(char *rwbs, unsigned int op) +{ + switch (op & REQ_OP_MASK) { + case REQ_OP_WRITE: + case REQ_OP_WRITE_SAME: + rwbs[0] = 'W'; + break; + case REQ_OP_DISCARD: + rwbs[0] = 'D'; + break; + case REQ_OP_SECURE_ERASE: + rwbs[0] = 'E'; + break; + case REQ_OP_FLUSH: + rwbs[0] = 'F'; + break; + case REQ_OP_READ: + rwbs[0] = 'R'; + break; + default: + rwbs[0] = 'N'; + } + + if (op & REQ_FUA) { + rwbs[1] = 'F'; + } else { + rwbs[1] = '#'; + } + if (op & REQ_RAHEAD) { + rwbs[2] = 'A'; + } else { + rwbs[2] = '#'; + } + if (op & REQ_SYNC) { + rwbs[3] = 'S'; + } else { + rwbs[3] = '#'; + } + if (op & REQ_META) { + rwbs[4] = 'M'; + } else { + rwbs[4] = '#'; + } +} + +void update_curr_data_in_start(struct stage_data *curr_data, struct update_params *params) { + if (curr_data && params) { + curr_data->start_count += 1; + curr_data->major = params->major; + curr_data->first_minor = params->first_minor; + blk_fill_rwbs(curr_data->io_type, params->cmd_flags); + } +} + +void update_curr_data_in_finish(struct stage_data *curr_data, struct update_params *params, u64 duration) { + if (curr_data && params) { + curr_data->finish_count += 1; + curr_data->major = params->major; + curr_data->first_minor = params->first_minor; + blk_fill_rwbs(curr_data->io_type, params->cmd_flags); + if (duration > DURATION_THRESHOLD) { + curr_data->finish_over_time += 1; + } + } +} + +static void init_io_counter(struct io_counter *counterp, int major, int first_minor) { + if (counterp) { + counterp->start_time = bpf_ktime_get_ns(); + counterp->major = major; + counterp->first_minor = first_minor; + } +} + +u32 find_matching_tag_1_keys(int major, int minor) { + u32 key = 0; + struct stage_data *curr_data = bpf_map_lookup_elem(&tag_res, &key); + + if (curr_data != NULL && curr_data->major == major && curr_data->first_minor == minor) { + return key; + } + + u32 key_2 = 1; + struct stage_data *curr_data_2 = bpf_map_lookup_elem(&tag_res, &key_2); + + if (curr_data_2 != NULL && curr_data_2->major == major && curr_data_2->first_minor == minor) { + return key_2; + } + + u32 key_3 = 2; + struct stage_data *curr_data_3 = bpf_map_lookup_elem(&tag_res, &key_3); + + if (curr_data_3 != NULL && curr_data_3->major == major && curr_data_3->first_minor == minor) { + return key_3; + } + + return MAP_SIZE + 1; +} + +u32 find_matching_tag_2_keys(int major, int minor) { + u32 key = 3; + struct stage_data *curr_data = bpf_map_lookup_elem(&tag_res, &key); + + if (curr_data != NULL && curr_data->major == major && curr_data->first_minor == minor) { + return key; + } + + u32 key_2 = 4; + struct stage_data *curr_data_2 = bpf_map_lookup_elem(&tag_res, &key_2); + + if (curr_data_2 != NULL && curr_data_2->major == major && curr_data_2->first_minor == minor) { + return key_2; + } + + u32 key_3 = 5; + struct stage_data *curr_data_3 = bpf_map_lookup_elem(&tag_res, &key_3); + + if (curr_data_3 != NULL && curr_data_3->major == major && curr_data_3->first_minor == minor) { + return key_3; + } + + return MAP_SIZE + 1; +} + +u32 find_matching_tag_3_keys(int major, int minor) { + u32 key = 6; + struct stage_data *curr_data = bpf_map_lookup_elem(&tag_res, &key); + + if (curr_data != NULL && curr_data->major == major && curr_data->first_minor == minor) { + return key; + } + + u32 key_2 = 7; + struct stage_data *curr_data_2 = bpf_map_lookup_elem(&tag_res, &key_2); + + if (curr_data_2 != NULL && curr_data_2->major == major && curr_data_2->first_minor == minor) { + return key_2; + } + + u32 key_3 = 8; + struct stage_data *curr_data_3 = bpf_map_lookup_elem(&tag_res, &key_3); + + if (curr_data_3 != NULL && curr_data_3->major == major && curr_data_3->first_minor == minor) { + return key_3; + } + + return MAP_SIZE + 1; +} + +u32 find_matching_tag_4_keys(int major, int minor) { + u32 key = 9; + struct stage_data *curr_data = bpf_map_lookup_elem(&tag_res, &key); + + if (curr_data != NULL && curr_data->major == major && curr_data->first_minor == minor) { + return key; + } + + u32 key_2 = 10; + struct stage_data *curr_data_2 = bpf_map_lookup_elem(&tag_res, &key_2); + + if (curr_data_2 != NULL && curr_data_2->major == major && curr_data_2->first_minor == minor) { + return key_2; + } + + u32 key_3 = 11; + struct stage_data *curr_data_3 = bpf_map_lookup_elem(&tag_res, &key_3); + + if (curr_data_3 != NULL && curr_data_3->major == major && curr_data_3->first_minor == minor) { + return key_3; + } + + return MAP_SIZE + 1; +} + +u32 find_matching_tag_5_keys(int major, int minor) { + u32 key = 12; + struct stage_data *curr_data = bpf_map_lookup_elem(&tag_res, &key); + + if (curr_data != NULL && curr_data->major == major && curr_data->first_minor == minor) { + return key; + } + + u32 key_2 = 13; + struct stage_data *curr_data_2 = bpf_map_lookup_elem(&tag_res, &key_2); + + if (curr_data_2 != NULL && curr_data_2->major == major && curr_data_2->first_minor == minor) { + return key_2; + } + + u32 key_3 = 14; + struct stage_data *curr_data_3 = bpf_map_lookup_elem(&tag_res, &key_3); + + if (curr_data_3 != NULL && curr_data_3->major == major && curr_data_3->first_minor == minor) { + return key_3; + } + + return MAP_SIZE + 1; +} + +u32 find_matching_blk_1_keys(int major, int minor) { + u32 key = 0; + struct stage_data *curr_data = bpf_map_lookup_elem(&blk_res, &key); + + if (curr_data != NULL && curr_data->major == major && curr_data->first_minor == minor) { + return key; + } + + u32 key_2 = 1; + struct stage_data *curr_data_2 = bpf_map_lookup_elem(&blk_res, &key_2); + + if (curr_data_2 != NULL && curr_data_2->major == major && curr_data_2->first_minor == minor) { + return key_2; + } + + u32 key_3 = 2; + struct stage_data *curr_data_3 = bpf_map_lookup_elem(&blk_res, &key_3); + + if (curr_data_3 != NULL && curr_data_3->major == major && curr_data_3->first_minor == minor) { + return key_3; + } + + return MAP_SIZE + 1; +} + +u32 find_matching_blk_2_keys(int major, int minor) { + u32 key = 3; + struct stage_data *curr_data = bpf_map_lookup_elem(&blk_res, &key); + + if (curr_data != NULL && curr_data->major == major && curr_data->first_minor == minor) { + return key; + } + + u32 key_2 = 4; + struct stage_data *curr_data_2 = bpf_map_lookup_elem(&blk_res, &key_2); + + if (curr_data_2 != NULL && curr_data_2->major == major && curr_data_2->first_minor == minor) { + return key_2; + } + + u32 key_3 = 5; + struct stage_data *curr_data_3 = bpf_map_lookup_elem(&blk_res, &key_3); + + if (curr_data_3 != NULL && curr_data_3->major == major && curr_data_3->first_minor == minor) { + return key_3; + } + + return MAP_SIZE + 1; +} + +u32 find_matching_blk_3_keys(int major, int minor) { + u32 key = 6; + struct stage_data *curr_data = bpf_map_lookup_elem(&blk_res, &key); + + if (curr_data != NULL && curr_data->major == major && curr_data->first_minor == minor) { + return key; + } + + u32 key_2 = 7; + struct stage_data *curr_data_2 = bpf_map_lookup_elem(&blk_res, &key_2); + + if (curr_data_2 != NULL && curr_data_2->major == major && curr_data_2->first_minor == minor) { + return key_2; + } + + u32 key_3 = 8; + struct stage_data *curr_data_3 = bpf_map_lookup_elem(&blk_res, &key_3); + + if (curr_data_3 != NULL && curr_data_3->major == major && curr_data_3->first_minor == minor) { + return key_3; + } + + return MAP_SIZE + 1; +} + +u32 find_matching_blk_4_keys(int major, int minor) { + u32 key = 9; + struct stage_data *curr_data = bpf_map_lookup_elem(&blk_res, &key); + + if (curr_data != NULL && curr_data->major == major && curr_data->first_minor == minor) { + return key; + } + + u32 key_2 = 10; + struct stage_data *curr_data_2 = bpf_map_lookup_elem(&blk_res, &key_2); + + if (curr_data_2 != NULL && curr_data_2->major == major && curr_data_2->first_minor == minor) { + return key_2; + } + + u32 key_3 = 11; + struct stage_data *curr_data_3 = bpf_map_lookup_elem(&blk_res, &key_3); + + if (curr_data_3 != NULL && curr_data_3->major == major && curr_data_3->first_minor == minor) { + return key_3; + } + + return MAP_SIZE + 1; +} + +u32 find_matching_blk_5_keys(int major, int minor) { + u32 key = 12; + struct stage_data *curr_data = bpf_map_lookup_elem(&blk_res, &key); + + if (curr_data != NULL && curr_data->major == major && curr_data->first_minor == minor) { + return key; + } + + u32 key_2 = 13; + struct stage_data *curr_data_2 = bpf_map_lookup_elem(&blk_res, &key_2); + + if (curr_data_2 != NULL && curr_data_2->major == major && curr_data_2->first_minor == minor) { + return key_2; + } + + u32 key_3 = 14; + struct stage_data *curr_data_3 = bpf_map_lookup_elem(&blk_res, &key_3); + + if (curr_data_3 != NULL && curr_data_3->major == major && curr_data_3->first_minor == minor) { + return key_3; + } + + return MAP_SIZE + 1; +} + +u32 find_matching_bio_1_keys(int major, int minor) { + u32 key = 0; + struct stage_data *curr_data = bpf_map_lookup_elem(&bio_res, &key); + + if (curr_data != NULL && curr_data->major == major && curr_data->first_minor == minor) { + return key; + } + + u32 key_2 = 1; + struct stage_data *curr_data_2 = bpf_map_lookup_elem(&bio_res, &key_2); + + if (curr_data_2 != NULL && curr_data_2->major == major && curr_data_2->first_minor == minor) { + return key_2; + } + + u32 key_3 = 2; + struct stage_data *curr_data_3 = bpf_map_lookup_elem(&bio_res, &key_3); + + if (curr_data_3 != NULL && curr_data_3->major == major && curr_data_3->first_minor == minor) { + return key_3; + } + + return MAP_SIZE + 1; +} + +u32 find_matching_bio_2_keys(int major, int minor) { + u32 key = 3; + struct stage_data *curr_data = bpf_map_lookup_elem(&bio_res, &key); + + if (curr_data != NULL && curr_data->major == major && curr_data->first_minor == minor) { + return key; + } + + u32 key_2 = 4; + struct stage_data *curr_data_2 = bpf_map_lookup_elem(&bio_res, &key_2); + + if (curr_data_2 != NULL && curr_data_2->major == major && curr_data_2->first_minor == minor) { + return key_2; + } + + u32 key_3 = 5; + struct stage_data *curr_data_3 = bpf_map_lookup_elem(&bio_res, &key_3); + + if (curr_data_3 != NULL && curr_data_3->major == major && curr_data_3->first_minor == minor) { + return key_3; + } + + return MAP_SIZE + 1; +} + +u32 find_matching_bio_3_keys(int major, int minor) { + u32 key = 6; + struct stage_data *curr_data = bpf_map_lookup_elem(&bio_res, &key); + + if (curr_data != NULL && curr_data->major == major && curr_data->first_minor == minor) { + return key; + } + + u32 key_2 = 7; + struct stage_data *curr_data_2 = bpf_map_lookup_elem(&bio_res, &key_2); + + if (curr_data_2 != NULL && curr_data_2->major == major && curr_data_2->first_minor == minor) { + return key_2; + } + + u32 key_3 = 8; + struct stage_data *curr_data_3 = bpf_map_lookup_elem(&bio_res, &key_3); + + if (curr_data_3 != NULL && curr_data_3->major == major && curr_data_3->first_minor == minor) { + return key_3; + } + + return MAP_SIZE + 1; +} + +u32 find_matching_bio_4_keys(int major, int minor) { + u32 key = 9; + struct stage_data *curr_data = bpf_map_lookup_elem(&bio_res, &key); + + if (curr_data != NULL && curr_data->major == major && curr_data->first_minor == minor) { + return key; + } + + u32 key_2 = 10; + struct stage_data *curr_data_2 = bpf_map_lookup_elem(&bio_res, &key_2); + + if (curr_data_2 != NULL && curr_data_2->major == major && curr_data_2->first_minor == minor) { + return key_2; + } + + u32 key_3 = 11; + struct stage_data *curr_data_3 = bpf_map_lookup_elem(&bio_res, &key_3); + + if (curr_data_3 != NULL && curr_data_3->major == major && curr_data_3->first_minor == minor) { + return key_3; + } + + return MAP_SIZE + 1; +} + +u32 find_matching_bio_5_keys(int major, int minor) { + u32 key = 12; + struct stage_data *curr_data = bpf_map_lookup_elem(&bio_res, &key); + + if (curr_data != NULL && curr_data->major == major && curr_data->first_minor == minor) { + return key; + } + + u32 key_2 = 13; + struct stage_data *curr_data_2 = bpf_map_lookup_elem(&bio_res, &key_2); + + if (curr_data_2 != NULL && curr_data_2->major == major && curr_data_2->first_minor == minor) { + return key_2; + } + + u32 key_3 = 14; + struct stage_data *curr_data_3 = bpf_map_lookup_elem(&bio_res, &key_3); + + if (curr_data_3 != NULL && curr_data_3->major == major && curr_data_3->first_minor == minor) { + return key_3; + } + + return MAP_SIZE + 1; +} + +u32 find_matching_wbt_1_keys(int major, int minor) { + u32 key = 0; + struct stage_data *curr_data = bpf_map_lookup_elem(&wbt_res, &key); + + if (curr_data != NULL && curr_data->major == major && curr_data->first_minor == minor) { + return key; + } + + u32 key_2 = 1; + struct stage_data *curr_data_2 = bpf_map_lookup_elem(&wbt_res, &key_2); + + if (curr_data_2 != NULL && curr_data_2->major == major && curr_data_2->first_minor == minor) { + return key_2; + } + + u32 key_3 = 2; + struct stage_data *curr_data_3 = bpf_map_lookup_elem(&wbt_res, &key_3); + + if (curr_data_3 != NULL && curr_data_3->major == major && curr_data_3->first_minor == minor) { + return key_3; + } + + return MAP_SIZE + 1; +} + +u32 find_matching_wbt_2_keys(int major, int minor) { + u32 key = 3; + struct stage_data *curr_data = bpf_map_lookup_elem(&wbt_res, &key); + + if (curr_data != NULL && curr_data->major == major && curr_data->first_minor == minor) { + return key; + } + + u32 key_2 = 4; + struct stage_data *curr_data_2 = bpf_map_lookup_elem(&wbt_res, &key_2); + + if (curr_data_2 != NULL && curr_data_2->major == major && curr_data_2->first_minor == minor) { + return key_2; + } + + u32 key_3 = 5; + struct stage_data *curr_data_3 = bpf_map_lookup_elem(&wbt_res, &key_3); + + if (curr_data_3 != NULL && curr_data_3->major == major && curr_data_3->first_minor == minor) { + return key_3; + } + + return MAP_SIZE + 1; +} + +u32 find_matching_wbt_3_keys(int major, int minor) { + u32 key = 6; + struct stage_data *curr_data = bpf_map_lookup_elem(&wbt_res, &key); + + if (curr_data != NULL && curr_data->major == major && curr_data->first_minor == minor) { + return key; + } + + u32 key_2 = 7; + struct stage_data *curr_data_2 = bpf_map_lookup_elem(&wbt_res, &key_2); + + if (curr_data_2 != NULL && curr_data_2->major == major && curr_data_2->first_minor == minor) { + return key_2; + } + + u32 key_3 = 8; + struct stage_data *curr_data_3 = bpf_map_lookup_elem(&wbt_res, &key_3); + + if (curr_data_3 != NULL && curr_data_3->major == major && curr_data_3->first_minor == minor) { + return key_3; + } + + return MAP_SIZE + 1; +} + +u32 find_matching_wbt_4_keys(int major, int minor) { + u32 key = 9; + struct stage_data *curr_data = bpf_map_lookup_elem(&wbt_res, &key); + + if (curr_data != NULL && curr_data->major == major && curr_data->first_minor == minor) { + return key; + } + + u32 key_2 = 10; + struct stage_data *curr_data_2 = bpf_map_lookup_elem(&wbt_res, &key_2); + + if (curr_data_2 != NULL && curr_data_2->major == major && curr_data_2->first_minor == minor) { + return key_2; + } + + u32 key_3 = 11; + struct stage_data *curr_data_3 = bpf_map_lookup_elem(&wbt_res, &key_3); + + if (curr_data_3 != NULL && curr_data_3->major == major && curr_data_3->first_minor == minor) { + return key_3; + } + + return MAP_SIZE + 1; +} + +u32 find_matching_wbt_5_keys(int major, int minor) { + u32 key = 12; + struct stage_data *curr_data = bpf_map_lookup_elem(&wbt_res, &key); + + if (curr_data != NULL && curr_data->major == major && curr_data->first_minor == minor) { + return key; + } + + u32 key_2 = 13; + struct stage_data *curr_data_2 = bpf_map_lookup_elem(&wbt_res, &key_2); + + if (curr_data_2 != NULL && curr_data_2->major == major && curr_data_2->first_minor == minor) { + return key_2; + } + + u32 key_3 = 14; + struct stage_data *curr_data_3 = bpf_map_lookup_elem(&wbt_res, &key_3); + + if (curr_data_3 != NULL && curr_data_3->major == major && curr_data_3->first_minor == minor) { + return key_3; + } + + return MAP_SIZE + 1; +} + +// start rq_driver +SEC("kprobe/blk_mq_start_request") +int kprobe_blk_mq_start_request(struct pt_regs *regs) +{ + struct request_kylin *rq = (struct request_kylin *)PT_REGS_PARM1(regs); + struct gendisk *curr_rq_disk = _(rq->rq_disk); + int major = _(curr_rq_disk->major); + int first_minor = _(curr_rq_disk->first_minor); + unsigned int cmd_flags = _(rq->cmd_flags); + + struct io_counter *counterp, zero = {}; + + u32 key = find_matching_blk_1_keys(major, first_minor); + if (key >= MAP_SIZE){ + key = find_matching_blk_2_keys(major, first_minor); + if (key >= MAP_SIZE){ + key = find_matching_blk_3_keys(major, first_minor); + if (key >= MAP_SIZE){ + key = find_matching_blk_4_keys(major, first_minor); + if (key >= MAP_SIZE){ + key = find_matching_blk_5_keys(major, first_minor); + if (key >= MAP_SIZE){ + return 0; + } + } + } + } + } + init_io_counter(&zero, major, first_minor); + + counterp = bpf_map_lookup_elem(&blk_map, &rq); + if (counterp || major == 0) { + return 0; + } + long err = bpf_map_update_elem(&blk_map, &rq, &zero, BPF_NOEXIST); + if (err) { + return 0; + } + + u64 curr_start_range = zero.start_time / THRESHOLD; + + struct update_params params = { + .major = major, + .first_minor = first_minor, + .cmd_flags = cmd_flags, + .curr_start_range = curr_start_range, + }; + + struct stage_data *curr_data; + curr_data = bpf_map_lookup_elem(&blk_res, &key); + if (!curr_data) { + struct stage_data new_data = { + .start_count = 1, + .finish_count = 0, + .finish_over_time = 0, + .duration = 0, + .major = major, + .first_minor = first_minor, + .io_type = "", + }; + blk_fill_rwbs(new_data.io_type, cmd_flags); + bpf_map_update_elem(&blk_res, &key, &new_data, 0); + } else { + update_curr_data_in_start(curr_data, ¶ms); + } + + struct time_range_io_count *curr_data_time_range; + curr_data_time_range = bpf_map_lookup_elem(&blk_res_2, &curr_start_range); + if (curr_data_time_range == NULL) { + struct time_range_io_count new_data = { .count = {0} }; + bpf_map_update_elem(&blk_res_2, &curr_start_range, &new_data, 0); + } else { + if (key < MAP_SIZE) { + __sync_fetch_and_add(&curr_data_time_range->count[key], 1); + } + } + + return 0; +} + +// finish rq_driver +SEC("kprobe/blk_mq_free_request") +int kprobe_blk_mq_free_request(struct pt_regs *regs) +{ + struct request_kylin *rq = (struct request_kylin *)PT_REGS_PARM1(regs); + struct gendisk *curr_rq_disk = _(rq->rq_disk); + int major = _(curr_rq_disk->major); + int first_minor = _(curr_rq_disk->first_minor); + unsigned int cmd_flags = _(rq->cmd_flags); + + struct io_counter *counterp; + u32 key = find_matching_blk_1_keys(major, first_minor); + if (key >= MAP_SIZE){ + key = find_matching_blk_2_keys(major, first_minor); + if (key >= MAP_SIZE){ + key = find_matching_blk_3_keys(major, first_minor); + if (key >= MAP_SIZE){ + key = find_matching_blk_4_keys(major, first_minor); + if (key >= MAP_SIZE){ + key = find_matching_blk_5_keys(major, first_minor); + if (key >= MAP_SIZE){ + return 0; + } + } + } + } + } + + counterp = bpf_map_lookup_elem(&blk_map, &rq); + + if (!counterp) { + return 0; + } + + u64 duration = bpf_ktime_get_ns() - counterp->start_time; + u64 curr_start_range = counterp->start_time / THRESHOLD; + + struct update_params params = { + .major = major, + .first_minor = first_minor, + .cmd_flags = cmd_flags, + .curr_start_range = curr_start_range, + }; + + struct stage_data *curr_data; + curr_data = bpf_map_lookup_elem(&blk_res, &key); + if (curr_data == NULL && duration > DURATION_THRESHOLD) { + struct stage_data new_data = { + .start_count = 1, + .finish_count = 1, + .finish_over_time = 1, + .duration = 0, + .major = major, + .first_minor = first_minor, + .io_type = "", + }; + blk_fill_rwbs(new_data.io_type, cmd_flags); + bpf_map_update_elem(&blk_res, &key, &new_data, 0); + } else if (curr_data == NULL) { + struct stage_data new_data = { + .start_count = 1, + .finish_count = 1, + .finish_over_time = 0, + .duration = 0, + .major = major, + .first_minor = first_minor, + .io_type = "", + }; + blk_fill_rwbs(new_data.io_type, cmd_flags); + bpf_map_update_elem(&blk_res, &key, &new_data, 0); + } else { + curr_data->duration += duration; + update_curr_data_in_finish(curr_data, ¶ms, &duration); + } + + struct time_range_io_count *curr_data_time_range; + curr_data_time_range = bpf_map_lookup_elem(&blk_res_2, &curr_start_range); + if (curr_data_time_range == NULL) { + struct time_range_io_count new_data = { .count = {0} }; + bpf_map_update_elem(&blk_res_2, &curr_start_range, &new_data, 0); + } else { + if (key < MAP_SIZE && curr_data_time_range->count[key] > 0) { + __sync_fetch_and_add(&curr_data_time_range->count[key], -1); + } + } + + bpf_map_delete_elem(&blk_map, &rq); + return 0; +} + +// start bio +SEC("kprobe/blk_mq_make_request") +int kprobe_blk_mq_make_request(struct pt_regs *regs) +{ + struct bio *bio = (struct bio *)PT_REGS_PARM2(regs); + struct gendisk *curr_rq_disk = _(bio->bi_disk); + int major = _(curr_rq_disk->major); + int first_minor = _(curr_rq_disk->first_minor); + unsigned int cmd_flags = _(bio->bi_opf); + + struct io_counter *counterp, zero = {}; + u32 key = find_matching_bio_1_keys(major, first_minor); + if (key >= MAP_SIZE){ + key = find_matching_bio_2_keys(major, first_minor); + if (key >= MAP_SIZE){ + key = find_matching_bio_3_keys(major, first_minor); + if (key >= MAP_SIZE){ + key = find_matching_bio_4_keys(major, first_minor); + if (key >= MAP_SIZE){ + key = find_matching_bio_5_keys(major, first_minor); + if (key >= MAP_SIZE){ + return 0; + } + } + } + } + } + + init_io_counter(&zero, major, first_minor); + + counterp = bpf_map_lookup_elem(&bio_map, &bio); + if (counterp || major == 0) + return 0; + + long err = bpf_map_update_elem(&bio_map, &bio, &zero, BPF_NOEXIST); + if (err && err != -EEXIST) + return 0; + + u64 curr_start_range = zero.start_time / THRESHOLD; + + struct update_params params = { + .major = major, + .first_minor = first_minor, + .cmd_flags = cmd_flags, + .curr_start_range = curr_start_range, + }; + + struct stage_data *curr_data; + curr_data = bpf_map_lookup_elem(&bio_res, &key); + if (curr_data == NULL) { + struct stage_data new_data = { + .start_count = 1, + .finish_count = 0, + .finish_over_time = 0, + .duration = 0, + .major = major, + .first_minor = first_minor, + .io_type = "", + }; + blk_fill_rwbs(new_data.io_type, cmd_flags); + bpf_map_update_elem(&bio_res, &key, &new_data, 0); + } else { + update_curr_data_in_start(curr_data, ¶ms); + } + + struct time_range_io_count *curr_data_time_range; + curr_data_time_range = bpf_map_lookup_elem(&bio_res_2, &curr_start_range); + if (curr_data_time_range == NULL) { + struct time_range_io_count new_data = { .count = {0} }; + bpf_map_update_elem(&bio_res_2, &curr_start_range, &new_data, 0); + } else { + if (key < MAP_SIZE) { + __sync_fetch_and_add(&curr_data_time_range->count[key], 1); + } + } + + return 0; +} + +// finish bio +SEC("kprobe/bio_endio") +int kprobe_bio_endio(struct pt_regs *regs) +{ + struct bio *bio = (struct bio *)PT_REGS_PARM1(regs); + struct gendisk *curr_rq_disk = _(bio->bi_disk); + int major = _(curr_rq_disk->major); + int first_minor = _(curr_rq_disk->first_minor); + unsigned int cmd_flags = _(bio->bi_opf); + + struct io_counter *counterp; + void *delete_map = NULL; + u32 key = find_matching_bio_1_keys(major, first_minor); + if (key >= MAP_SIZE){ + key = find_matching_bio_2_keys(major, first_minor); + if (key >= MAP_SIZE){ + key = find_matching_bio_3_keys(major, first_minor); + if (key >= MAP_SIZE){ + key = find_matching_bio_4_keys(major, first_minor); + if (key >= MAP_SIZE){ + key = find_matching_bio_5_keys(major, first_minor); + if (key >= MAP_SIZE){ + return 0; + } + } + } + } + } + + counterp = bpf_map_lookup_elem(&bio_map, &bio); + + if (!counterp) { + return 0; + } + + delete_map = &bio_map; + + u64 duration = bpf_ktime_get_ns() - counterp->start_time; + u64 curr_start_range = counterp->start_time / THRESHOLD; + + struct update_params params = { + .major = major, + .first_minor = first_minor, + .cmd_flags = cmd_flags, + .curr_start_range = curr_start_range, + }; + + struct stage_data *curr_data; + curr_data = bpf_map_lookup_elem(&bio_res, &key); + if (curr_data == NULL && duration > DURATION_THRESHOLD) { + struct stage_data new_data = { + .start_count = 1, + .finish_count = 1, + .finish_over_time = 1, + .duration = 0, + .major = major, + .first_minor = first_minor, + .io_type = "", + }; + blk_fill_rwbs(new_data.io_type, cmd_flags); + bpf_map_update_elem(&bio_res, &key, &new_data, 0); + } else if (curr_data == NULL) { + struct stage_data new_data = { + .start_count = 1, + .finish_count = 1, + .finish_over_time = 0, + .duration = 0, + .major = major, + .first_minor = first_minor, + .io_type = "", + }; + blk_fill_rwbs(new_data.io_type, cmd_flags); + bpf_map_update_elem(&bio_res, &key, &new_data, 0); + } else { + curr_data->duration += duration; + update_curr_data_in_finish(curr_data, ¶ms, &duration); + } + + struct time_range_io_count *curr_data_time_range; + curr_data_time_range = bpf_map_lookup_elem(&bio_res_2, &curr_start_range); + if (curr_data_time_range == NULL) { + struct time_range_io_count new_data = { .count = {0} }; + bpf_map_update_elem(&bio_res_2, &curr_start_range, &new_data, 0); + } else { + if (key < MAP_SIZE && curr_data_time_range->count[key] > 0) { + __sync_fetch_and_add(&curr_data_time_range->count[key], -1); + } + } + + bpf_map_delete_elem(delete_map, &bio); + return 0; +} + +// start wbt +SEC("kprobe/wbt_wait") +int kprobe_wbt_wait(struct pt_regs *regs) +{ + u64 wbtkey = bpf_get_current_task(); + u64 value = (u64)PT_REGS_PARM2(regs); + (void)bpf_map_update_elem(&wbt_args, &wbtkey, &value, BPF_ANY); + struct bio *bio = (struct bio *)value; + struct gendisk *curr_rq_disk = _(bio->bi_disk); + int major = _(curr_rq_disk->major); + int first_minor = _(curr_rq_disk->first_minor); + unsigned int cmd_flags = _(bio->bi_opf); + + struct io_counter *counterp, zero = {}; + u32 key = find_matching_wbt_1_keys(major, first_minor); + if (key >= MAP_SIZE){ + key = find_matching_wbt_2_keys(major, first_minor); + if (key >= MAP_SIZE){ + key = find_matching_wbt_3_keys(major, first_minor); + if (key >= MAP_SIZE){ + key = find_matching_wbt_4_keys(major, first_minor); + if (key >= MAP_SIZE){ + key = find_matching_wbt_5_keys(major, first_minor); + if (key >= MAP_SIZE){ + return 0; + } + } + } + } + } + + init_io_counter(&zero, major, first_minor); + + counterp = bpf_map_lookup_elem(&wbt_map, &wbtkey); + + if (counterp || major == 0) + return 0; + long err = bpf_map_update_elem(&wbt_map, &wbtkey, &zero, BPF_NOEXIST); + if (err) + return 0; + + u64 curr_start_range = zero.start_time / THRESHOLD; + + struct update_params params = { + .major = major, + .first_minor = first_minor, + .cmd_flags = cmd_flags, + .curr_start_range = curr_start_range, + }; + + struct stage_data *curr_data; + curr_data = bpf_map_lookup_elem(&wbt_res, &key); + if (!curr_data) { + struct stage_data new_data = { + .start_count = 1, + .finish_count = 0, + .finish_over_time = 0, + .duration = 0, + .major = major, + .first_minor = first_minor, + .io_type = "", + }; + blk_fill_rwbs(new_data.io_type, cmd_flags); + bpf_map_update_elem(&wbt_res, &key, &new_data, 0); + } else { + update_curr_data_in_start(curr_data, ¶ms); + } + + struct time_range_io_count *curr_data_time_range; + curr_data_time_range = bpf_map_lookup_elem(&wbt_res_2, &curr_start_range); + if (curr_data_time_range == NULL) { + struct time_range_io_count new_data = { .count = {0} }; + bpf_map_update_elem(&wbt_res_2, &curr_start_range, &new_data, 0); + } else { + if (key < MAP_SIZE) { + __sync_fetch_and_add(&curr_data_time_range->count[key], 1); + } + } + + return 0; +} + +// finish wbt +SEC("kretprobe/wbt_wait") +int kretprobe_wbt_wait(struct pt_regs *regs) +{ + struct bio *bio = NULL; + u64 *wbtargs = NULL; + u64 wbtkey = bpf_get_current_task(); + wbtargs = (u64 *)bpf_map_lookup_elem(&wbt_args, &wbtkey); + if (wbtargs == NULL) { + bpf_map_delete_elem(&wbt_args, &wbtkey); + return 0; + } + bio = (struct bio *)(*wbtargs); + struct gendisk *curr_rq_disk = _(bio->bi_disk); + int major = _(curr_rq_disk->major); + int first_minor = _(curr_rq_disk->first_minor); + unsigned int cmd_flags = _(bio->bi_opf); + + struct io_counter *counterp; + u32 key = find_matching_wbt_1_keys(major, first_minor); + if (key >= MAP_SIZE){ + key = find_matching_wbt_2_keys(major, first_minor); + if (key >= MAP_SIZE){ + key = find_matching_wbt_3_keys(major, first_minor); + if (key >= MAP_SIZE){ + key = find_matching_wbt_4_keys(major, first_minor); + if (key >= MAP_SIZE){ + key = find_matching_wbt_5_keys(major, first_minor); + if (key >= MAP_SIZE){ + return 0; + } + } + } + } + } + + counterp = bpf_map_lookup_elem(&wbt_map, &wbtkey); + + if (!counterp) + return 0; + + u64 duration = bpf_ktime_get_ns() - counterp->start_time; + u64 curr_start_range = counterp->start_time / THRESHOLD; + + struct update_params params = { + .major = major, + .first_minor = first_minor, + .cmd_flags = cmd_flags, + .curr_start_range = curr_start_range, + }; + + struct stage_data *curr_data; + curr_data = bpf_map_lookup_elem(&wbt_res, &key); + if (curr_data == NULL && duration > DURATION_THRESHOLD) { + struct stage_data new_data = { + .start_count = 1, + .finish_count = 1, + .finish_over_time = 1, + .duration = 0, + .major = major, + .first_minor = first_minor, + .io_type = "", + }; + blk_fill_rwbs(new_data.io_type, cmd_flags); + bpf_map_update_elem(&wbt_res, &key, &new_data, 0); + } else if (curr_data == NULL) { + struct stage_data new_data = { + .start_count = 1, + .finish_count = 1, + .finish_over_time = 0, + .duration = 0, + .io_type = "", + .major = major, + .first_minor = first_minor, + }; + blk_fill_rwbs(new_data.io_type, cmd_flags); + bpf_map_update_elem(&wbt_res, &key, &new_data, 0); + } else { + curr_data->duration += duration; + update_curr_data_in_finish(curr_data, ¶ms, &duration); + } + + struct time_range_io_count *curr_data_time_range; + curr_data_time_range = bpf_map_lookup_elem(&wbt_res_2, &curr_start_range); + if (curr_data_time_range == NULL) { + struct time_range_io_count new_data = { .count = {0} }; + bpf_map_update_elem(&wbt_res_2, &curr_start_range, &new_data, 0); + } else { + if (key < MAP_SIZE && curr_data_time_range->count[key] > 0) { + __sync_fetch_and_add(&curr_data_time_range->count[key], -1); + } + } + + bpf_map_delete_elem(&wbt_map, &wbtkey); + bpf_map_delete_elem(&wbt_args, &wbtkey); + return 0; +} + +// start get_tag +SEC("kprobe/blk_mq_get_tag") +int kprobe_blk_mq_get_tag(struct pt_regs *regs) +{ + u64 tagkey = bpf_get_current_task(); + u64 value = (u64)PT_REGS_PARM1(regs); + (void)bpf_map_update_elem(&tag_args, &tagkey, &value, BPF_ANY); + struct blk_mq_alloc_data *bd= (struct blk_mq_alloc_data *)value; + struct request_queue *q = _(bd->q); + struct backing_dev_info *backing_dev_info = _(q->backing_dev_info); + struct device *owner = _(backing_dev_info->owner); + dev_t devt = _(owner->devt); + int major = MAJOR(devt); + int first_minor = MINOR(devt); + unsigned int cmd_flags = 0; + + struct io_counter *counterp, zero = {}; + u32 key = find_matching_tag_1_keys(major, first_minor); + if (key >= MAP_SIZE){ + key = find_matching_tag_2_keys(major, first_minor); + if (key >= MAP_SIZE){ + key = find_matching_tag_3_keys(major, first_minor); + if (key >= MAP_SIZE){ + key = find_matching_tag_4_keys(major, first_minor); + if (key >= MAP_SIZE){ + key = find_matching_tag_5_keys(major, first_minor); + if (key >= MAP_SIZE){ + return 0; + } + } + } + } + } + + init_io_counter(&zero, major, first_minor); + + counterp = bpf_map_lookup_elem(&tag_map, &tagkey); + if (counterp || major == 0) + return 0; + long err = bpf_map_update_elem(&tag_map, &tagkey, &zero, BPF_NOEXIST); + if (err) + return 0; + + u64 curr_start_range = zero.start_time / THRESHOLD; + + struct update_params params = { + .major = major, + .first_minor = first_minor, + .cmd_flags = cmd_flags, + .curr_start_range = curr_start_range, + }; + + struct stage_data *curr_data; + curr_data = bpf_map_lookup_elem(&tag_res, &key); + if (!curr_data) { + struct stage_data new_data = { + .start_count = 1, + .finish_count = 0, + .finish_over_time = 0, + .duration = 0, + .major = major, + .first_minor = first_minor, + .io_type = "", + }; + blk_fill_rwbs(new_data.io_type, cmd_flags); + bpf_map_update_elem(&tag_res, &key, &new_data, 0); + } else { + update_curr_data_in_start(curr_data, ¶ms); + } + + struct time_range_io_count *curr_data_time_range; + curr_data_time_range = bpf_map_lookup_elem(&tag_res_2, &curr_start_range); + if (curr_data_time_range == NULL) { + struct time_range_io_count new_data = { .count = {0} }; + bpf_map_update_elem(&tag_res_2, &curr_start_range, &new_data, 0); + } else { + if (key < MAP_SIZE) { + __sync_fetch_and_add(&curr_data_time_range->count[key], 1); + } + } + + return 0; +} + +// finish get_tag +SEC("kretprobe/blk_mq_get_tag") +int kretprobe_blk_mq_get_tag(struct pt_regs *regs) +{ + u64 tagkey = bpf_get_current_task(); + u64 *tagargs = NULL; + struct blk_mq_alloc_data *bd = NULL; + + tagargs = (u64 *)bpf_map_lookup_elem(&tag_args, &tagkey); + if (tagargs == NULL) { + bpf_map_delete_elem(&tag_args, &tagkey); + return 0; + } + bd = (struct blk_mq_alloc_data *)(*tagargs); + struct request_queue *q = _(bd->q); + struct backing_dev_info *backing_dev_info = _(q->backing_dev_info); + struct device *owner = _(backing_dev_info->owner); + dev_t devt = _(owner->devt); + int major = MAJOR(devt); + int first_minor = MINOR(devt); + unsigned int cmd_flags = 0; + + struct io_counter *counterp; + u32 key = find_matching_tag_1_keys(major, first_minor); + if (key >= MAP_SIZE){ + key = find_matching_tag_2_keys(major, first_minor); + if (key >= MAP_SIZE){ + key = find_matching_tag_3_keys(major, first_minor); + if (key >= MAP_SIZE){ + key = find_matching_tag_4_keys(major, first_minor); + if (key >= MAP_SIZE){ + key = find_matching_tag_5_keys(major, first_minor); + if (key >= MAP_SIZE){ + return 0; + } + } + } + } + } + + counterp = bpf_map_lookup_elem(&tag_map, &tagkey); + + if (!counterp) + return 0; + + u64 duration = bpf_ktime_get_ns() - counterp->start_time; + u64 curr_start_range = counterp->start_time / THRESHOLD; + + struct update_params params = { + .major = major, + .first_minor = first_minor, + .cmd_flags = cmd_flags, + .curr_start_range = curr_start_range, + }; + + struct stage_data *curr_data; + curr_data = bpf_map_lookup_elem(&tag_res, &key); + if (curr_data == NULL && duration > DURATION_THRESHOLD) { + struct stage_data new_data = { + .start_count = 1, + .finish_count = 1, + .finish_over_time = 1, + .duration = 0, + .major = major, + .first_minor = first_minor, + .io_type = "", + }; + blk_fill_rwbs(new_data.io_type, cmd_flags); + bpf_map_update_elem(&tag_res, &key, &new_data, 0); + } else if (curr_data == NULL) { + struct stage_data new_data = { + .start_count = 1, + .finish_count = 1, + .finish_over_time = 0, + .duration = 0, + .major = major, + .first_minor = first_minor, + .io_type = "", + }; + blk_fill_rwbs(new_data.io_type, cmd_flags); + bpf_map_update_elem(&tag_res, &key, &new_data, 0); + } else { + curr_data->duration += duration; + update_curr_data_in_finish(curr_data, ¶ms, &duration); + } + + struct time_range_io_count *curr_data_time_range; + curr_data_time_range = bpf_map_lookup_elem(&tag_res_2, &curr_start_range); + if (curr_data_time_range == NULL) { + struct time_range_io_count new_data = { .count = {0} }; + bpf_map_update_elem(&tag_res_2, &curr_start_range, &new_data, 0); + } else { + if (key < MAP_SIZE && curr_data_time_range->count[key] > 0) { + __sync_fetch_and_add(&curr_data_time_range->count[key], -1); + } + } + + bpf_map_delete_elem(&tag_map, &tagkey); + bpf_map_delete_elem(&tag_args, &tagkey); + return 0; +} + +char LICENSE[] SEC("license") = "Dual BSD/GPL"; +u32 _version SEC("version") = LINUX_VERSION_CODE; + diff --git a/src/c/ebpf_collector/ebpf_collector.c b/src/c/ebpf_collector/ebpf_collector.c new file mode 100644 index 0000000000000000000000000000000000000000..54a3a9eebf4f92d6ff232bde047c61d6b27693be --- /dev/null +++ b/src/c/ebpf_collector/ebpf_collector.c @@ -0,0 +1,461 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2024. All rights reserved. + * Description: ebpf collector program + * Author: Zhang Nan + * Create: 2024-09-27 + */ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "ebpf_collector.h" + +#define BLK_MAP (map_fd[0]) +#define BLK_RES (map_fd[1]) +#define BIO_MAP (map_fd[2]) +#define BIO_RES (map_fd[3]) +#define WBT_MAP (map_fd[4]) +#define WBT_RES (map_fd[5]) +#define TAG_MAP (map_fd[7]) +#define TAG_RES (map_fd[8]) +#define BLK_RES_2 (map_fd[10]) +#define BIO_RES_2 (map_fd[11]) +#define WBT_RES_2 (map_fd[12]) +#define TAG_RES_2 (map_fd[13]) +#define BPF_FILE "/usr/lib/ebpf_collector.bpf.o" + +#define MAX_LINE_LENGTH 1024 +#define MAX_SECTION_NAME_LENGTH 256 +#define CONFIG_FILE "/etc/sysSentry/collector.conf" + +typedef struct { + int major; + int minor; +} DeviceInfo; + +typedef enum { + LOG_LEVEL_NONE, + LOG_LEVEL_DEBUG, + LOG_LEVEL_INFO, + LOG_LEVEL_WARNING, + LOG_LEVEL_ERROR +} LogLevel; + +LogLevel currentLogLevel = LOG_LEVEL_INFO; + +static volatile bool exiting; + +const char argp_program_doc[] = +"Show block device I/O pattern.\n" +"\n" +"USAGE: ebpf_collector [--help]\n" +"\n" +"EXAMPLES:\n" +" ebpf_collector # show block I/O pattern\n"; + +static const struct argp_option opts[] = { + { NULL, 'h', NULL, OPTION_HIDDEN, "Show the full help" }, + {}, +}; + +static error_t parse_arg(int key, char *arg, struct argp_state *state) { + static int pos_args; + + switch (key) { + case 'h': + argp_state_help(state, stderr, ARGP_HELP_STD_HELP); + break; + default: + return ARGP_ERR_UNKNOWN; + } + return 0; +} + +static void sig_handler(int sig) +{ + exiting = true; +} + +char* extract_device_name(const char *path) { + const char *dev_dir = "/dev/"; + char *name = strrchr(path, '/') + 1; + if (strncmp(dev_dir, path, strlen(dev_dir)) == 0) { + return strdup(name); + } + return NULL; +} + +char* find_device_name(dev_t dev) { + DIR *dir; + struct dirent *entry; + struct stat sb; + char *device_name = NULL; + char path[1024]; + + dir = opendir("/dev"); + if (dir == NULL) { + perror("Failed to open /dev"); + return NULL; + } + + while ((entry = readdir(dir)) != NULL) { + snprintf(path, sizeof(path), "/dev/%s", entry->d_name); + + if (entry->d_type == DT_DIR || entry->d_type == DT_LNK) { + continue; + } + + if (stat(path, &sb) == -1) { + continue; + } + + if (major(sb.st_rdev) == major(dev) && minor(sb.st_rdev) == minor(dev)) { + device_name = extract_device_name(path); + break; + } + } + + closedir(dir); + return device_name; +} + +static void update_io_dump(struct bpf_map *map_res, int *io_dump, int *map_size, char *stage) { + struct time_range_io_count time_count; + u32 io_dump_key = 0; + struct sysinfo info; + sysinfo(&info); + int count_time = 150; + u32 curr_time = info.uptime; + while (count_time >= 0) { + io_dump_key = curr_time - count_time; + int err = bpf_map_lookup_elem(map_res, &io_dump_key, &time_count); + if (err < 0) { + count_time -= 1; + continue; + } + if ((curr_time - io_dump_key) >= 2) { + int isempty = 1; + for (int key = 0; key < map_size; key++) { + if (time_count.count[key] > 0) { + io_dump[key] += time_count.count[key]; + isempty = 0; + } + } + if (isempty || (curr_time - io_dump_key) > IO_DUMP_THRESHOLD) { + bpf_map_delete_elem(map_res, &io_dump_key); + } + } + count_time -= 1; + } +} + +static int print_map_res(struct bpf_map *map_res, char *stage, int *map_size, int *io_dump) +{ + struct stage_data counter; + int key = 0; + + logMessage(LOG_LEVEL_DEBUG, "print_map_res map_size: %d\n", map_size); + for (key = 0; key < map_size; key++) { + int err = bpf_map_lookup_elem(map_res, &key, &counter); + if (err < 0) { + logMessage(LOG_LEVEL_ERROR, "failed to lookup %s map_res: %d\n", stage, err); + return -1; + } + + size_t length = strlen(counter.io_type); + char io_type; + if (length > 0) { + logMessage(LOG_LEVEL_DEBUG, "io_type have value.\n"); + io_type = counter.io_type[0]; + } else { + logMessage(LOG_LEVEL_DEBUG, "io_type not value.\n"); + io_type = NULL; + } + int major = counter.major; + int first_minor = counter.first_minor; + dev_t dev = makedev(major, first_minor); + char *device_name = find_device_name(dev); + logMessage(LOG_LEVEL_DEBUG, "device_name: %s\n", device_name); + if (device_name && io_type) { + printf("%-7s %10llu %10llu %d %c %s\n", + stage, + counter.finish_count, + counter.duration, + io_dump[key], + io_type, + device_name + ); + fflush(stdout); + } + } + + return 0; +} + +int init_map(int *map_fd, const char *map_name, int *map_size, DeviceInfo *devices) { + struct stage_data init_data = {0}; + + memset(init_data.io_type, 0, sizeof(init_data.io_type)); + + for (int i = 0; i < map_size; i++) { + init_data.major = devices[i].major; + init_data.first_minor = devices[i].minor; + if (bpf_map_update_elem(map_fd, &i, &init_data, BPF_ANY) != 0) { + logMessage(LOG_LEVEL_ERROR, "Failed to initialize map %s at index %d\n", map_name, i); + return 1; + } + } + + return 0; +} + +char *read_config_value(const char *file, const char *section, const char *key) { + FILE *fp = fopen(file, "r"); + if (fp == NULL) { + logMessage(LOG_LEVEL_ERROR, "Failed to open config file.\n"); + return NULL; + } + + char line[MAX_LINE_LENGTH]; + char current_section[MAX_SECTION_NAME_LENGTH] = {0}; + char *value = NULL; + + while (fgets(line, sizeof(line), fp) != NULL) { + line[strcspn(line, "\n")] = 0; + + if (line[0] == '\0' || line[0] == ';' || line[0] == '#') { + continue; + } + + if (line[0] == '[') { + sscanf(line, "[%255[^]]", current_section); + continue; + } + + if (strcmp(current_section, section) == 0) { + char *delimiter = "="; + char *token = strtok(line, delimiter); + if (token != NULL) { + if (strcmp(token, key) == 0) { + token = strtok(NULL, delimiter); + if (token != NULL) { + value = strdup(token); + break; + } + } + } + } + } + + fclose(fp); + return value; +} + +void setLogLevel(const char *levelStr) { + if (strcmp(levelStr, "info") == 0) { + currentLogLevel = LOG_LEVEL_INFO; + } + else if (strcmp(levelStr, "warning") == 0) { + currentLogLevel = LOG_LEVEL_WARNING; + } + else if (strcmp(levelStr, "error") == 0) { + currentLogLevel = LOG_LEVEL_ERROR; + } + else if (strcmp(levelStr, "debug") == 0) { + currentLogLevel = LOG_LEVEL_DEBUG; + } + else if (strcmp(levelStr, "none") == 0) { + currentLogLevel = LOG_LEVEL_NONE; + } + else { + logMessage(LOG_LEVEL_ERROR, "unknown log level: %s\n", levelStr); + } +} + +void logMessage(LogLevel level, const char *format, ...){ + if (level >= currentLogLevel) { + va_list args; + va_start(args, format); + vprintf(format, args); + va_end(args); + } +} + +int check_for_device(const char *device_name) { + char path[256]; + snprintf(path, sizeof(path), "/sys/block/%s", device_name); + + DIR *dir = opendir(path); + if (dir == NULL) { + return 0; + } + + struct dirent *entry; + while ((entry = readdir(dir)) != NULL) { + struct stat statbuf; + if (stat(path, &statbuf) == 0) { + if (S_ISDIR(statbuf.st_mode)) { + closedir(dir); + return 1; + } + } + } + closedir(dir); + return 0; +} + +int main(int argc, char **argv) { + struct partitions *partitions = NULL; + const struct partition *partition; + static const struct argp argp = { + .options = opts, + .parser = parse_arg, + .doc = argp_program_doc, + }; + int err; + char filename[256]; + DIR *dir; + struct dirent *entry; + char path[1024]; + int major, minor; + DeviceInfo devices[MAP_SIZE]; + int device_count = 0; + struct rlimit r = {RLIM_INFINITY, RLIM_INFINITY}; + setrlimit(RLIMIT_MEMLOCK, &r); + + char *level = read_config_value(CONFIG_FILE, "log", "level"); + if (level != NULL) { + if (level[strlen(level) - 1] == '\r') { + size_t len = strlen(level); + level[len - 1] = '\0'; + } + setLogLevel(level); + free(level); + } else { + logMessage(LOG_LEVEL_INFO, "the log level is incorrectly configured. the default log level is info.\n"); + } + logMessage(LOG_LEVEL_DEBUG, "finish config parse.\n"); + + err = argp_parse(&argp, argc, argv, 0, NULL, NULL); + if (err) { + logMessage(LOG_LEVEL_ERROR, "argp parse failed.\n"); + return err; + } + + snprintf(filename, sizeof(filename), BPF_FILE); + + if (load_bpf_file(filename)) { + logMessage(LOG_LEVEL_ERROR, "load_bpf_file failed.\n"); + return 1; + } + + signal(SIGINT, sig_handler); + + dir = opendir("/dev"); + if (dir == NULL) { + logMessage(LOG_LEVEL_ERROR, "Failed to open /dev directory.\n"); + return EXIT_FAILURE; + } + + while ((entry = readdir(dir)) != NULL) { + if (entry->d_type != DT_BLK) { + continue; + } + snprintf(path, sizeof(path), "/dev/%s", entry->d_name); + struct stat statbuf; + if (lstat(path, &statbuf) != 0 && !S_ISBLK(statbuf.st_mode)) { + continue; + } + if (!strncmp(entry->d_name, "dm-", 3) || !strncmp(entry->d_name, "loop", 4) || + !strncmp(entry->d_name, "md", 2)) { + continue; + } + if (!check_for_device(entry->d_name)) { + continue; + } + + devices[device_count].major = major(statbuf.st_rdev); + devices[device_count].minor = minor(statbuf.st_rdev); + device_count++; + if (device_count >= MAP_SIZE) { + logMessage(LOG_LEVEL_DEBUG, " device_count moren than MAP_SIZE.\n"); + break; + } + } + + closedir(dir); + + if (init_map(BLK_RES, "blk_res_map", device_count, devices) != 0) { + logMessage(LOG_LEVEL_ERROR, "blk_res_map failed.\n"); + return 1; + } + if (init_map(BIO_RES, "blo_res_map", device_count, devices) != 0) { + logMessage(LOG_LEVEL_ERROR, "blo_res_map failed.\n"); + return 1; + } + if (init_map(WBT_RES, "wbt_res_map", device_count, devices) != 0) { + logMessage(LOG_LEVEL_ERROR, "wbt_res_map failed.\n"); + return 1; + } + if (init_map(TAG_RES, "tag_res_map", device_count, devices) != 0) { + logMessage(LOG_LEVEL_ERROR, "tag_res_map failed.\n"); + return 1; + } + + for (;;) { + + sleep(1); + + int io_dump_blk[MAP_SIZE] = {0}; + update_io_dump(BLK_RES_2, io_dump_blk, device_count,"rq_driver"); + err = print_map_res(BLK_RES, "rq_driver", device_count, io_dump_blk); + if (err) { + logMessage(LOG_LEVEL_ERROR, "print_map_res rq_driver error.\n"); + break; + } + + int io_dump_bio[MAP_SIZE] = {0}; + update_io_dump(BIO_RES_2, io_dump_bio, device_count,"bio"); + err = print_map_res(BIO_RES, "bio", device_count, io_dump_bio); + if (err) { + logMessage(LOG_LEVEL_ERROR, "print_map_res bio error.\n"); + break; + } + + int io_dump_tag[MAP_SIZE] = {0}; + update_io_dump(TAG_RES_2, io_dump_tag, device_count,"gettag"); + err = print_map_res(TAG_RES, "gettag", device_count, io_dump_tag); + if (err) { + logMessage(LOG_LEVEL_ERROR, "print_map_res gettag error.\n"); + break; + } + + int io_dump_wbt[MAP_SIZE] = {0}; + update_io_dump(WBT_RES_2, io_dump_wbt, device_count,"wbt"); + err = print_map_res(WBT_RES, "wbt", device_count, io_dump_wbt); + if (err) { + logMessage(LOG_LEVEL_ERROR, "print_map_res wbt error.\n"); + break; + } + + if (exiting) { + logMessage(LOG_LEVEL_DEBUG, "exit program.\n"); + break; + } + } + + return -err; +} diff --git a/src/c/ebpf_collector/ebpf_collector.h b/src/c/ebpf_collector/ebpf_collector.h new file mode 100644 index 0000000000000000000000000000000000000000..fcebc93a2c99170ebc198040efce6c92a8b7f868 --- /dev/null +++ b/src/c/ebpf_collector/ebpf_collector.h @@ -0,0 +1,77 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2024. All rights reserved. + * Description: ebpf collector program + * Author: Zhang Nan + * Create: 2024-09-27 + */ +#ifndef __EBPFCOLLECTOR_H +#define __EBPFCOLLECTOR_H + +typedef long long unsigned int u64; +typedef unsigned int u32; + +#define MAX_IO_TIME 130 +#define IO_DUMP_THRESHOLD 120 +#define THRESHOLD 1000000000 +#define DURATION_THRESHOLD 500000000 + +#define RWBS_LEN 8 + +#define REQ_OP_BITS 8 +#define REQ_OP_MASK ((1 << REQ_OP_BITS) - 1) +#define REQ_FUA (1ULL << __REQ_FUA) +#define REQ_RAHEAD (1ULL << __REQ_RAHEAD) +#define REQ_SYNC (1ULL << __REQ_SYNC) +#define REQ_META (1ULL << __REQ_META) +#define REQ_PREFLUSH (1ULL << __REQ_PREFLUSH) +#define REQ_OP_READ 0 +#define REQ_OP_WRITE 1 +#define REQ_OP_FLUSH 2 +#define REQ_OP_DISCARD 3 +#define REQ_OP_SECURE_ERASE 5 +#define REQ_OP_WRITE_SAME 7 +#define MAP_SIZE 15 + +enum stage_type { + BIO=0, + WBT, + GET_TAG, + DEADLINE, + BFQ, + KYBER, + RQ_DRIVER, + MAX_STAGE_TYPE, +}; + +struct stage_data { + u64 start_count; + u64 finish_count; + u64 finish_over_time; + u64 duration; + int major; + int first_minor; + char io_type[RWBS_LEN]; +}; + +struct io_counter { + u64 duration; + u64 start_time; + u32 isend; + int major; + int first_minor; +}; + +struct update_params { + int major; + int first_minor; + unsigned int cmd_flags; + u64 curr_start_range; +}; + +struct time_range_io_count +{ + u32 count[MAP_SIZE]; +}; + +#endif /* __EBPFCOLLECTOR_H */ + diff --git a/src/libso/xalarm/register_xalarm.c b/src/libso/xalarm/register_xalarm.c index 152c078aa1c62c55b98a60336e63e63419418136..fe15dc8a0be6854e055364270c69fd73c3e9fc07 100644 --- a/src/libso/xalarm/register_xalarm.c +++ b/src/libso/xalarm/register_xalarm.c @@ -35,7 +35,7 @@ #define ALARM_SOCKET_PERMISSION 0700 #define TIME_UNIT_MILLISECONDS 1000 -#define MAX_PARAS_LEN 511 +#define MAX_PARAS_LEN 8191 #define MIN_ALARM_ID 1001 #define MAX_ALARM_ID (MIN_ALARM_ID + MAX_NUM_OF_ALARM_ID - 1) @@ -91,7 +91,7 @@ static int create_unix_socket(const char *path) return -1; } - fd = socket(AF_UNIX, SOCK_DGRAM, 0); + fd = socket(AF_UNIX, SOCK_STREAM, 0); if (fd < 0) { printf("socket failed:%s\n", strerror(errno)); return -1; @@ -103,14 +103,6 @@ static int create_unix_socket(const char *path) goto release_socket; } - if (access(PATH_REG_ALARM, F_OK) == 0) { - ret = unlink(PATH_REG_ALARM); - if (ret != 0) { - printf("unlink register socket file failed\n"); - goto release_socket; - } - } - if (access(DIR_XALARM, F_OK) == -1) { if (mkdir(DIR_XALARM, ALARM_DIR_PERMISSION) == -1) { printf("mkdir %s failed\n", DIR_XALARM); @@ -120,32 +112,22 @@ static int create_unix_socket(const char *path) if (memset(&alarm_addr, 0, sizeof(alarm_addr)) == NULL) { printf("create_unix_socket: memset alarm_addr failed, ret: %d\n", ret); - goto remove_dir; + goto release_socket; } alarm_addr.sun_family = AF_UNIX; strncpy(alarm_addr.sun_path, path, sizeof(alarm_addr.sun_path) - 1); - if (bind(fd, (struct sockaddr *)&alarm_addr, sizeof(alarm_addr.sun_family) + strlen(alarm_addr.sun_path)) < 0) { - printf("bind socket failed:%s\n", strerror(errno)); - goto remove_dir; + if (connect(fd, (struct sockaddr*)&alarm_addr, sizeof(alarm_addr)) == -1) { + printf("create_unix_socket: connect alarm_addr failed, ret: %d\n", ret); + goto release_socket; } if (chmod(path, ALARM_SOCKET_PERMISSION) < 0) { printf("chmod %s failed: %s\n", path, strerror(errno)); - goto unlink_sockfile; + goto release_socket; } return fd; -unlink_sockfile: - ret = unlink(PATH_REG_ALARM); - if (ret != 0) { - printf("unlink register socket file failed\n"); - } -remove_dir: - ret = rmdir(DIR_XALARM); - if (ret != 0) { - printf("rmdir %s failed: %s\n", path, strerror(errno)); - } release_socket: (void)close(fd); @@ -174,7 +156,11 @@ static void *alarm_recv(void *arg) continue; } printf("recv error len:%d errno:%d\n", recvlen, errno); - } + } else if (recvlen == 0) { + printf("connection closed by xalarmd, maybe connections reach max num or service stopped.\n"); + g_register_info.thread_should_stop = 1; + break; + } } return NULL; } @@ -229,6 +215,11 @@ bool xalarm_Upgrade(struct alarm_subscription_info id_filter, int client_id) printf("%s: invalid args\n", __func__); return false; } + + if (g_register_info.thread_should_stop) { + printf("%s: upgrade failed, alarm thread has stopped\n", __func__); + return false; + } set_alarm_id(id_filter); return true; @@ -271,8 +262,6 @@ int xalarm_Register(alarm_callback_func callback, struct alarm_subscription_info void xalarm_UnRegister(int client_id) { - int ret; - if (!g_register_info.is_registered) { printf("%s: alarm has not registered\n", __func__); return; @@ -292,10 +281,6 @@ void xalarm_UnRegister(int client_id) if (g_register_info.register_fd != -1) { (void)close(g_register_info.register_fd); g_register_info.register_fd = -1; - ret = unlink(PATH_REG_ALARM); - if (ret != 0) { - printf("%s: unlink register socket file failed\n", __func__); - } } memset(g_register_info.alarm_enable_bitmap, 0, MAX_NUM_OF_ALARM_ID * sizeof(char)); @@ -357,12 +342,17 @@ int xalarm_Report(unsigned short usAlarmId, unsigned char ucAlarmLevel, struct sockaddr_un alarm_addr; if ((usAlarmId < MIN_ALARM_ID || usAlarmId > MAX_ALARM_ID) || - (ucAlarmLevel < ALARM_LEVEL_FATAL || ucAlarmLevel > ALARM_LEVEL_DEBUG) || + (ucAlarmLevel < MINOR_ALM || ucAlarmLevel > CRITICAL_ALM) || (ucAlarmType < ALARM_TYPE_OCCUR || ucAlarmType > ALARM_TYPE_RECOVER)) { fprintf(stderr, "%s: alarm info invalid\n", __func__); return -1; } + if (pucParas == NULL || (int)strlen(pucParas) > MAX_PARAS_LEN) { + fprintf(stderr, "%s: alarm info invalid\n", __func__); + return -1; + } + if (memset(&info, 0, sizeof(struct alarm_info)) == NULL) { fprintf(stderr, "%s: memset info failed, ret: %d\n", __func__, ret); return -1; @@ -666,3 +656,4 @@ int report_result(const char *task_name, enum RESULT_LEVEL result_level, const c return RETURE_CODE_SUCCESS; } + diff --git a/src/libso/xalarm/register_xalarm.h b/src/libso/xalarm/register_xalarm.h index 1f26c6a06588a6b9b3b570f6e7e6bbbef02970c7..dcf4f03d9543da83d190d65de7a217919520d84d 100644 --- a/src/libso/xalarm/register_xalarm.h +++ b/src/libso/xalarm/register_xalarm.h @@ -11,7 +11,7 @@ #include #include -#define ALARM_INFO_MAX_PARAS_LEN 512 +#define ALARM_INFO_MAX_PARAS_LEN 8192 #define MAX_STRERROR_SIZE 1024 #define MAX_ALARM_TYEPS 1024 #define MIN_ALARM_ID 1001 @@ -19,11 +19,9 @@ #define MEMORY_ALARM_ID 1001 -#define ALARM_LEVEL_FATAL 1 -#define ALARM_LEVEL_ERROR 2 -#define ALARM_LEVEL_WARNING 3 -#define ALARM_LEVEL_INFO 4 -#define ALARM_LEVEL_DEBUG 5 +#define MINOR_ALM 1 +#define MAJOR_ALM 2 +#define CRITICAL_ALM 3 #define ALARM_TYPE_OCCUR 1 #define ALARM_TYPE_RECOVER 2 diff --git a/src/python/sentryCollector/__init__.py b/src/python/sentryCollector/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/src/python/sentryCollector/__main__.py b/src/python/sentryCollector/__main__.py new file mode 100644 index 0000000000000000000000000000000000000000..9c2ae50eea7db9db5ce1e7a0ef9e570f6be15f6d --- /dev/null +++ b/src/python/sentryCollector/__main__.py @@ -0,0 +1,17 @@ +# coding: utf-8 +# Copyright (c) 2023 Huawei Technologies Co., Ltd. +# sysSentry is licensed under the Mulan PSL v2. +# You can use this software according to the terms and conditions of the Mulan PSL v2. +# You may obtain a copy of Mulan PSL v2 at: +# http://license.coscl.org.cn/MulanPSL2 +# THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR +# PURPOSE. +# See the Mulan PSL v2 for more details. + +""" +main +""" +from collectd import collectd + +collectd.main() diff --git a/src/python/sentryCollector/collect_config.py b/src/python/sentryCollector/collect_config.py new file mode 100644 index 0000000000000000000000000000000000000000..7ca9898bf27197f331a0a302742b21fc6a31a9b7 --- /dev/null +++ b/src/python/sentryCollector/collect_config.py @@ -0,0 +1,148 @@ +# coding: utf-8 +# Copyright (c) 2024 Huawei Technologies Co., Ltd. +# sysSentry is licensed under the Mulan PSL v2. +# You can use this software according to the terms and conditions of the Mulan PSL v2. +# You may obtain a copy of Mulan PSL v2 at: +# http://license.coscl.org.cn/MulanPSL2 +# THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR +# PURPOSE. +# See the Mulan PSL v2 for more details. + +""" +Read and save collector.conf value. +""" +import configparser +import logging +import os +import re + + +COLLECT_CONF_PATH = "/etc/sysSentry/collector.conf" + +CONF_COMMON = 'common' +CONF_MODULES = 'modules' + +# io +CONF_IO = 'io' +CONF_IO_PERIOD_TIME = 'period_time' +CONF_IO_MAX_SAVE = 'max_save' +CONF_IO_DISK = 'disk' +CONF_IO_PERIOD_TIME_DEFAULT = 1 +CONF_IO_MAX_SAVE_DEFAULT = 10 +CONF_IO_DISK_DEFAULT = "default" + +# log +CONF_LOG = 'log' +CONF_LOG_LEVEL = 'level' +LogLevel = { + "debug": logging.DEBUG, + "info": logging.INFO, + "warning": logging.WARNING, + "error": logging.ERROR, + "critical": logging.CRITICAL +} + + +def get_log_level(filename=COLLECT_CONF_PATH): + if not os.path.exists(filename): + return logging.INFO + + try: + config = configparser.ConfigParser() + config.read(filename) + if not config.has_option(CONF_LOG, CONF_LOG_LEVEL): + return logging.INFO + log_level = config.get(CONF_LOG, CONF_LOG_LEVEL) + if log_level.lower() in LogLevel: + return LogLevel.get(log_level.lower()) + return logging.INFO + except configparser.Error: + return logging.INFO + + +class CollectConfig: + def __init__(self, filename=COLLECT_CONF_PATH): + + self.filename = filename + self.modules = [] + self.module_count = 0 + self.load_config() + + def load_config(self): + if not os.path.exists(self.filename): + logging.error("%s is not exists", self.filename) + return + + try: + self.config = configparser.ConfigParser() + self.config.read(self.filename) + except configparser.Error: + logging.error("collect configure file read failed") + return + + try: + common_config = self.config[CONF_COMMON] + modules_str = common_config[CONF_MODULES].lower() + # remove space + modules_list = set(modules_str.replace(" ", "").split(',')) + except KeyError as e: + logging.error("read config data failed, %s", e) + return + + pattern = r'^[a-zA-Z0-9-_]+$' + for module_name in modules_list: + if not re.match(pattern, module_name): + logging.warning("module_name: %s is invalid", module_name) + continue + if not self.config.has_section(module_name): + logging.warning("module_name: %s config is incorrect", module_name) + continue + self.modules.append(module_name) + + def load_module_config(self, module_name): + module_name = module_name.strip().lower() + if module_name in self.modules and self.config.has_section(module_name): + return {key.lower(): value for key, value in self.config[module_name].items()} + else: + raise ValueError(f"Module '{module_name}' not found in configuration") + + def get_io_config(self): + result_io_config = {} + io_map_value = self.load_module_config(CONF_IO) + # period_time + period_time = io_map_value.get(CONF_IO_PERIOD_TIME) + if period_time and period_time.isdigit() and int(period_time) >= 1 and int(period_time) <= 300: + result_io_config[CONF_IO_PERIOD_TIME] = int(period_time) + else: + logging.warning("module_name = %s section, field = %s is incorrect, use default %d", + CONF_IO, CONF_IO_PERIOD_TIME, CONF_IO_PERIOD_TIME_DEFAULT) + result_io_config[CONF_IO_PERIOD_TIME] = CONF_IO_PERIOD_TIME_DEFAULT + # max_save + max_save = io_map_value.get(CONF_IO_MAX_SAVE) + if max_save and max_save.isdigit() and int(max_save) >= 1 and int(max_save) <= 300: + result_io_config[CONF_IO_MAX_SAVE] = int(max_save) + else: + logging.warning("module_name = %s section, field = %s is incorrect, use default %d", + CONF_IO, CONF_IO_MAX_SAVE, CONF_IO_MAX_SAVE_DEFAULT) + result_io_config[CONF_IO_MAX_SAVE] = CONF_IO_MAX_SAVE_DEFAULT + # disk + disk = io_map_value.get(CONF_IO_DISK) + if disk: + disk_str = disk.lower().replace(" ", "") + pattern = r'^[a-zA-Z0-9-_,]+$' + if not re.match(pattern, disk_str): + logging.warning("module_name = %s section, field = %s is incorrect, use default %s", + CONF_IO, CONF_IO_DISK, CONF_IO_DISK_DEFAULT) + disk_str = CONF_IO_DISK_DEFAULT + disk_str = ",".join(set(disk_str.split(','))) + result_io_config[CONF_IO_DISK] = disk_str + else: + logging.warning("module_name = %s section, field = %s is incorrect, use default %s", + CONF_IO, CONF_IO_DISK, CONF_IO_DISK_DEFAULT) + result_io_config[CONF_IO_DISK] = CONF_IO_DISK_DEFAULT + logging.debug("config get_io_config: %s", result_io_config) + return result_io_config + + def get_common_config(self): + return {key.lower(): value for key, value in self.config['common'].items()} diff --git a/src/python/sentryCollector/collect_io.py b/src/python/sentryCollector/collect_io.py new file mode 100644 index 0000000000000000000000000000000000000000..a7e86cbfa92b27422adb96794271ec09ded42dec --- /dev/null +++ b/src/python/sentryCollector/collect_io.py @@ -0,0 +1,471 @@ +# coding: utf-8 +# Copyright (c) 2024 Huawei Technologies Co., Ltd. +# sysSentry is licensed under the Mulan PSL v2. +# You can use this software according to the terms and conditions of the Mulan PSL v2. +# You may obtain a copy of Mulan PSL v2 at: +# http://license.coscl.org.cn/MulanPSL2 +# THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR +# PURPOSE. +# See the Mulan PSL v2 for more details. + +""" +collect module +""" +import os +import time +import logging +import threading +import subprocess +from typing import Union + +from .collect_config import CollectConfig + +Io_Category = ["read", "write", "flush", "discard"] +IO_GLOBAL_DATA = {} +IO_CONFIG_DATA = [] +EBPF_GLOBAL_DATA = [] +EBPF_PROCESS = None +EBPF_STAGE_LIST = ["wbt", "rq_driver", "bio", "gettag"] +EBPF_SUPPORT_VERSION = ["4.19.90"] + +class IoStatus(): + TOTAL = 0 + FINISH = 1 + LATENCY = 2 + +class CollectIo(): + + def __init__(self, module_config): + + io_config = module_config.get_io_config() + + self.period_time = io_config['period_time'] + self.max_save = io_config['max_save'] + disk_str = io_config['disk'] + + self.disk_map_stage = {} + self.window_value = {} + + self.ebpf_base_path = 'ebpf_collector' + + self.loop_all = False + + if disk_str == "default": + self.loop_all = True + else: + self.disk_list = disk_str.strip().split(',') + + self.stop_event = threading.Event() + + IO_CONFIG_DATA.append(self.period_time) + IO_CONFIG_DATA.append(self.max_save) + + def get_blk_io_hierarchy(self, disk_name, stage_list): + stats_file = '/sys/kernel/debug/block/{}/blk_io_hierarchy/stats'.format(disk_name) + try: + with open(stats_file, 'r') as file: + lines = file.read() + except FileNotFoundError: + logging.error("The file %s does not exist", stats_file) + return -1 + except Exception as e: + logging.error("An error occurred: %s", e) + return -1 + + curr_value = lines.strip().split('\n') + + for stage_val in curr_value: + stage = stage_val.split(' ')[0] + if (len(self.window_value[disk_name][stage])) >= 2: + self.window_value[disk_name][stage].pop(0) + + curr_stage_value = stage_val.split(' ')[1:-1] + self.window_value[disk_name][stage].append(curr_stage_value) + return 0 + + def append_period_lat(self, disk_name, stage_list): + for stage in stage_list: + if len(self.window_value[disk_name][stage]) < 2: + return + curr_stage_value = self.window_value[disk_name][stage][-1] + last_stage_value = self.window_value[disk_name][stage][-2] + + for index in range(len(Io_Category)): + # read=0, write=1, flush=2, discard=3 + if (len(IO_GLOBAL_DATA[disk_name][stage][Io_Category[index]])) >= self.max_save: + IO_GLOBAL_DATA[disk_name][stage][Io_Category[index]].pop() + + curr_lat = self.get_latency_value(curr_stage_value, last_stage_value, index) + curr_iops = self.get_iops(curr_stage_value, last_stage_value, index) + curr_io_length = self.get_io_length(curr_stage_value, last_stage_value, index) + curr_io_dump = self.get_io_dump(disk_name, stage, index) + + IO_GLOBAL_DATA[disk_name][stage][Io_Category[index]].insert(0, [curr_lat, curr_io_dump, curr_io_length, curr_iops]) + + def get_iops(self, curr_stage_value, last_stage_value, category): + try: + finish = int(curr_stage_value[category * 3 + IoStatus.FINISH]) - int(last_stage_value[category * 3 + IoStatus.FINISH]) + except ValueError as e: + logging.error("get_iops convert to int failed, %s", e) + return 0 + value = finish / self.period_time + if value.is_integer(): + return int(value) + else: + return round(value, 1) + + def get_latency_value(self, curr_stage_value, last_stage_value, category): + try: + finish = int(curr_stage_value[category * 3 + IoStatus.FINISH]) - int(last_stage_value[category * 3 + IoStatus.FINISH]) + lat_time = (int(curr_stage_value[category * 3 + IoStatus.LATENCY]) - int(last_stage_value[category * 3 + IoStatus.LATENCY])) + except ValueError as e: + logging.error("get_latency_value convert to int failed, %s", e) + return 0 + if finish <= 0 or lat_time <= 0: + return 0 + value = lat_time / finish / 1000 + if value.is_integer(): + return int(value) + else: + return round(value, 1) + + def get_io_length(self, curr_stage_value, last_stage_value, category): + try: + lat_time = (int(curr_stage_value[category * 3 + IoStatus.LATENCY]) - int(last_stage_value[category * 3 + IoStatus.LATENCY])) + except ValueError as e: + logging.error("get_io_length convert to int failed, %s", e) + return 0 + if lat_time <= 0: + return 0 + # ns convert us + lat_time = lat_time / 1000 + # s convert us + period_time = self.period_time * 1000 * 1000 + value = lat_time / period_time + if value.is_integer(): + return int(value) + else: + return round(value, 1) + + def get_io_dump(self, disk_name, stage, category): + io_dump_file = '/sys/kernel/debug/block/{}/blk_io_hierarchy/{}/io_dump'.format(disk_name, stage) + count = 0 + try: + with open(io_dump_file, 'r') as file: + for line in file: + count += line.count('.op=' + Io_Category[category].upper()) + if count > 0: + logging.info(f"io_dump info : {disk_name}, {stage}, {Io_Category[category]}, {count}") + except FileNotFoundError: + logging.error("The file %s does not exist.", io_dump_file) + return count + except Exception as e: + logging.error("An error occurred1: %s", e) + return count + return count + + def extract_first_column(self, file_path): + column_names = [] + try: + with open(file_path, 'r') as file: + for line in file: + parts = line.strip().split() + if parts: + column_names.append(parts[0]) + except FileNotFoundError: + logging.error("The file %s does not exist.", file_path) + except Exception as e: + logging.error("An error occurred2: %s", e) + return column_names + + def is_kernel_avaliable(self): + base_path = '/sys/kernel/debug/block' + all_disk = [] + for disk_name in os.listdir(base_path): + disk_path = os.path.join(base_path, disk_name) + blk_io_hierarchy_path = os.path.join(disk_path, 'blk_io_hierarchy') + + if not os.path.exists(blk_io_hierarchy_path): + logging.warning("no blk_io_hierarchy directory found in %s, skipping.", disk_name) + continue + + for file_name in os.listdir(blk_io_hierarchy_path): + file_path = os.path.join(blk_io_hierarchy_path, file_name) + if file_name == 'stats': + all_disk.append(disk_name) + + if len(all_disk) == 0: + logging.debug("no blk_io_hierarchy disk, it is not lock-free collection") + return False + + if self.loop_all: + self.disk_list = all_disk + + for disk_name in self.disk_list: + if not self.loop_all and disk_name not in all_disk: + logging.warning("the %s disk not exist!", disk_name) + continue + stats_file = '/sys/kernel/debug/block/{}/blk_io_hierarchy/stats'.format(disk_name) + stage_list = self.extract_first_column(stats_file) + self.disk_map_stage[disk_name] = stage_list + self.window_value[disk_name] = {} + IO_GLOBAL_DATA[disk_name] = {} + + return len(IO_GLOBAL_DATA) != 0 + + def is_ebpf_avaliable(self): + with open('/proc/version', 'r') as f: + kernel_version = f.read().split()[2] + major_version = kernel_version.split('-')[0] + + base_path = '/sys/kernel/debug/block' + for disk_name in os.listdir(base_path): + if not self.loop_all and disk_name not in self.disk_list: + continue + self.disk_map_stage[disk_name] = EBPF_STAGE_LIST + self.window_value[disk_name] = {} + IO_GLOBAL_DATA[disk_name] = {} + + for disk_name, stage_list in self.disk_map_stage.items(): + for stage in stage_list: + self.window_value[disk_name][stage] = {} + IO_GLOBAL_DATA[disk_name][stage] = {} + for category in Io_Category: + IO_GLOBAL_DATA[disk_name][stage][category] = [] + self.window_value[disk_name][stage][category] = [[0,0,0], [0,0,0]] + + return major_version in EBPF_SUPPORT_VERSION and os.path.exists('/usr/bin/ebpf_collector') and len(IO_GLOBAL_DATA) != 0 + + def get_ebpf_raw_data( + self + ) -> None: + global EBPF_PROCESS + global EBPF_GLOBAL_DATA + + while True: + if self.stop_event.is_set(): + logging.debug("collect io thread exit") + return + line = EBPF_PROCESS.stdout.readline() + if not line: + logging.info("no ebpf data found, wait for collect") + break + EBPF_GLOBAL_DATA.append(line.strip()) + time.sleep(0.1) + + def update_ebpf_collector_data( + self, + ) -> None: + global EBPF_GLOBAL_DATA + + while True: + if self.stop_event.is_set(): + logging.debug("collect io thread exit") + return + if EBPF_GLOBAL_DATA: + for data in EBPF_GLOBAL_DATA: + data_list = data.split() + if len(data_list) != 6: + continue + stage, finish_count, latency, io_dump, io_type ,disk_name = data_list + if stage not in EBPF_STAGE_LIST: + continue + if disk_name not in self.window_value: + continue + io_type = self.get_ebpf_io_type(io_type) + if not io_type: + continue + if (len(self.window_value[disk_name][stage][io_type])) >= 2: + self.window_value[disk_name][stage][io_type].pop() + self.window_value[disk_name][stage][io_type].append([int(finish_count), int(latency), int(io_dump)]) + EBPF_GLOBAL_DATA.clear() + time.sleep(0.1) + + def get_ebpf_io_type( + self, + io_type: str + ) -> str: + io_type_mapping = { + "R": "read", + "W": "write", + "F": "flush", + "D": "discard" + } + io_type = io_type_mapping.get(io_type, None) + return io_type + + def append_ebpf_period_data( + self, + ) -> None: + global IO_GLOBAL_DATA + while True: + if self.stop_event.is_set(): + logging.debug("collect io thread exit") + return + start_time = time.time() + for disk_name, stage_list in self.disk_map_stage.items(): + for stage in stage_list: + for io_type in Io_Category: + if len(self.window_value[disk_name][stage][io_type]) < 2: + return + if (len(IO_GLOBAL_DATA[disk_name][stage][io_type])) >= self.max_save: + IO_GLOBAL_DATA[disk_name][stage][io_type].pop() + curr_finish_count, curr_latency, curr_io_dump_count = self.window_value[disk_name][stage][io_type][-1] + prev_finish_count, prev_latency, prev_io_dump_count = self.window_value[disk_name][stage][io_type][-2] + self.window_value[disk_name][stage][io_type].pop(0) + self.window_value[disk_name][stage][io_type].insert(1, self.window_value[disk_name][stage][io_type][0]) + curr_lat = self.get_ebpf_latency_value(curr_latency=curr_latency, prev_latency=prev_latency, curr_finish_count=curr_finish_count, prev_finish_count=prev_finish_count) + curr_iops = self.get_ebpf_iops(curr_finish_count=curr_finish_count, prev_finish_count=prev_finish_count) + curr_io_length = self.get_ebpf_io_length(curr_latency=curr_latency, prev_latency=prev_latency) + curr_io_dump = self.get_ebpf_io_dump(curr_io_dump_count=curr_io_dump_count, prev_io_dump_count=prev_io_dump_count) + if curr_io_dump > 0: + logging.info(f"ebpf io_dump info : {disk_name}, {stage}, {io_type}, {curr_io_dump}") + IO_GLOBAL_DATA[disk_name][stage][io_type].insert(0, [curr_lat, curr_io_dump, curr_io_length, curr_iops]) + + elapsed_time = time.time() - start_time + sleep_time = self.period_time - elapsed_time + if sleep_time < 0: + continue + while sleep_time > 1: + if self.stop_event.is_set(): + logging.debug("collect io thread exit") + return + time.sleep(1) + sleep_time -= 1 + time.sleep(sleep_time) + + def get_ebpf_latency_value( + self, + curr_latency: int, + prev_latency: int, + curr_finish_count: int, + prev_finish_count: int + ) -> Union[int, float]: + finish = curr_finish_count - prev_finish_count + lat_time = curr_latency - prev_latency + if finish <= 0 or lat_time <= 0: + return 0 + value = lat_time / finish / 1000 + if value.is_integer(): + return int(value) + else: + return round(value, 1) + + def get_ebpf_iops( + self, + curr_finish_count: int, + prev_finish_count: int + ) -> Union[int, float]: + finish = curr_finish_count - prev_finish_count + if finish <= 0: + return 0 + value = finish / self.period_time + if value.is_integer(): + return int(value) + else: + return round(value, 1) + + def get_ebpf_io_length( + self, + curr_latency: int, + prev_latency: int, + ) -> Union[int, float]: + lat_time = curr_latency - prev_latency + if lat_time <= 0: + return 0 + # ns convert us + lat_time = lat_time / 1000 + # s convert us + period_time = self.period_time * 1000 * 1000 + value = lat_time / period_time + if value.is_integer(): + return int(value) + else: + return round(value, 1) + + def get_ebpf_io_dump( + self, + curr_io_dump_count: int, + prev_io_dump_count: int + ) -> Union[int, float]: + io_dump_count = curr_io_dump_count + if io_dump_count <= 0: + return 0 + value = io_dump_count + return int(value) + + def start_ebpf_subprocess( + self + ) -> None: + global EBPF_PROCESS + EBPF_PROCESS = subprocess.Popen(self.ebpf_base_path, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True) + + def stop_ebpf_subprocess( + self + ) -> None: + global EBPF_PROCESS + if EBPF_PROCESS: + EBPF_PROCESS.terminate() + EBPF_PROCESS.wait() + logging.info("ebpf collector thread exit") + + def main_loop(self): + global IO_GLOBAL_DATA + logging.info("collect io thread start") + + if self.is_kernel_avaliable() and len(self.disk_map_stage) != 0: + for disk_name, stage_list in self.disk_map_stage.items(): + for stage in stage_list: + self.window_value[disk_name][stage] = [] + IO_GLOBAL_DATA[disk_name][stage] = {} + for category in Io_Category: + IO_GLOBAL_DATA[disk_name][stage][category] = [] + + while True: + start_time = time.time() + + if self.stop_event.is_set(): + logging.debug("collect io thread exit") + return + + for disk_name, stage_list in self.disk_map_stage.items(): + if self.get_blk_io_hierarchy(disk_name, stage_list) < 0: + continue + self.append_period_lat(disk_name, stage_list) + + elapsed_time = time.time() - start_time + sleep_time = self.period_time - elapsed_time + if sleep_time < 0: + continue + while sleep_time > 1: + if self.stop_event.is_set(): + logging.debug("collect io thread exit") + return + time.sleep(1) + sleep_time -= 1 + time.sleep(sleep_time) + elif self.is_ebpf_avaliable(): + logging.info("ebpf collector thread start") + self.start_ebpf_subprocess() + + thread_get_data = threading.Thread(target=self.get_ebpf_raw_data) + thread_update_data = threading.Thread(target=self.update_ebpf_collector_data) + thread_append_data = threading.Thread(target=self.append_ebpf_period_data) + + thread_get_data.start() + thread_update_data.start() + thread_append_data.start() + + thread_get_data.join() + thread_update_data.join() + thread_append_data.join() + + self.stop_ebpf_subprocess() + else: + logging.warning("fail to start ebpf collector thread. collect io thread exits") + return + + # set stop event, notify thread exit + def stop_thread(self): + self.stop_event.set() diff --git a/src/python/sentryCollector/collect_plugin.py b/src/python/sentryCollector/collect_plugin.py new file mode 100644 index 0000000000000000000000000000000000000000..9495d8b989c8ac890266e88c63f504ef35fda507 --- /dev/null +++ b/src/python/sentryCollector/collect_plugin.py @@ -0,0 +1,350 @@ +# coding: utf-8 +# Copyright (c) 2024 Huawei Technologies Co., Ltd. +# sysSentry is licensed under the Mulan PSL v2. +# You can use this software according to the terms and conditions of the Mulan PSL v2. +# You may obtain a copy of Mulan PSL v2 at: +# http://license.coscl.org.cn/MulanPSL2 +# THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR +# PURPOSE. +# See the Mulan PSL v2 for more details. + +""" +collect plugin +""" +import json +import socket +import logging +import re +import os + +COLLECT_SOCKET_PATH = "/var/run/sysSentry/collector.sock" + +# data length param +CLT_MSG_HEAD_LEN = 9 #3+2+4 +CLT_MSG_PRO_LEN = 2 +CLT_MSG_MAGIC_LEN = 3 +CLT_MSG_LEN_LEN = 4 + +CLT_MAGIC = "CLT" +RES_MAGIC = "RES" + +# disk limit +LIMIT_DISK_CHAR_LEN = 32 +LIMIT_DISK_LIST_LEN = 10 + +# stage limit +LIMIT_STAGE_CHAR_LEN = 20 +LIMIT_STAGE_LIST_LEN = 15 + +#iotype limit +LIMIT_IOTYPE_CHAR_LEN = 7 +LIMIT_IOTYPE_LIST_LEN = 4 + +#period limit +LIMIT_PERIOD_MIN_LEN = 1 +LIMIT_PERIOD_MAX_LEN = 300 + +# max_save +LIMIT_MAX_SAVE_LEN = 300 + +# interface protocol +class ClientProtocol(): + IS_IOCOLLECT_VALID = 0 + GET_IO_DATA = 1 + PRO_END = 3 + +class ResultMessage(): + RESULT_SUCCEED = 0 + RESULT_UNKNOWN = 1 # unknown error + RESULT_NOT_PARAM = 2 # the parameter does not exist or the type does not match. + RESULT_INVALID_LENGTH = 3 # invalid parameter length. + RESULT_EXCEED_LIMIT = 4 # the parameter length exceeds the limit. + RESULT_PARSE_FAILED = 5 # parse failed + RESULT_INVALID_CHAR = 6 # invalid char + RESULT_DISK_NOEXIST = 7 # disk is not exist + RESULT_DISK_TYPE_MISMATCH= 8 # disk type mismatch + +Result_Messages = { + ResultMessage.RESULT_SUCCEED: "Succeed", + ResultMessage.RESULT_UNKNOWN: "Unknown error", + ResultMessage.RESULT_NOT_PARAM: "The parameter does not exist or the type does not match", + ResultMessage.RESULT_INVALID_LENGTH: "Invalid parameter length", + ResultMessage.RESULT_EXCEED_LIMIT: "The parameter length exceeds the limit", + ResultMessage.RESULT_PARSE_FAILED: "Parse failed", + ResultMessage.RESULT_INVALID_CHAR: "Invalid char", + ResultMessage.RESULT_DISK_NOEXIST: "Disk is not exist", + ResultMessage.RESULT_DISK_TYPE_MISMATCH: "Disk type mismatch" +} + +class DiskType(): + TYPE_NVME_SSD = 0 + TYPE_SATA_SSD = 1 + TYPE_SATA_HDD = 2 + +Disk_Type = { + DiskType.TYPE_NVME_SSD: "nvme_ssd", + DiskType.TYPE_SATA_SSD: "sata_ssd", + DiskType.TYPE_SATA_HDD: "sata_hdd" +} + +def client_send_and_recv(request_data, data_str_len, protocol): + """client socket send and recv message""" + try: + client_socket = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) + except socket.error: + logging.debug("collect_plugin: client create socket error") + return None + + try: + client_socket.connect(COLLECT_SOCKET_PATH) + except OSError: + client_socket.close() + logging.debug("collect_plugin: client connect error") + return None + + req_data_len = len(request_data) + request_msg = CLT_MAGIC + str(protocol).zfill(CLT_MSG_PRO_LEN) + str(req_data_len).zfill(CLT_MSG_LEN_LEN) + request_data + + try: + client_socket.send(request_msg.encode()) + res_data = client_socket.recv(len(RES_MAGIC) + CLT_MSG_PRO_LEN + data_str_len) + res_data = res_data.decode() + except (OSError, UnicodeError): + client_socket.close() + logging.debug("collect_plugin: client communicate error") + return None + + res_magic = res_data[:CLT_MSG_MAGIC_LEN] + if res_magic != "RES": + logging.debug("res msg format error") + return None + + protocol_str = res_data[CLT_MSG_MAGIC_LEN:CLT_MSG_MAGIC_LEN+CLT_MSG_PRO_LEN] + try: + protocol_id = int(protocol_str) + except ValueError: + logging.debug("recv msg protocol id is invalid %s", protocol_str) + return None + + if protocol_id >= ClientProtocol.PRO_END: + logging.debug("protocol id is invalid") + return None + + try: + res_data_len = int(res_data[CLT_MSG_MAGIC_LEN+CLT_MSG_PRO_LEN:]) + res_msg_data = client_socket.recv(res_data_len) + res_msg_data = res_msg_data.decode() + return res_msg_data + except (OSError, ValueError, UnicodeError): + logging.debug("collect_plugin: client recv res msg error") + finally: + client_socket.close() + + return None + +def validate_parameters(param, len_limit, char_limit): + ret = ResultMessage.RESULT_SUCCEED + if not param: + logging.error("param is invalid, param = %s", param) + ret = ResultMessage.RESULT_NOT_PARAM + return [False, ret] + + if not isinstance(param, list): + logging.error("%s is not list type.", param) + ret = ResultMessage.RESULT_NOT_PARAM + return [False, ret] + + if len(param) <= 0: + logging.error("%s length is 0.", param) + ret = ResultMessage.RESULT_INVALID_LENGTH + return [False, ret] + + pattern = r'^[a-zA-Z0-9_-]+$' + for info in param: + if not re.match(pattern, info): + logging.error("%s is invalid char", info) + ret = ResultMessage.RESULT_INVALID_CHAR + return [False, ret] + + # length of len_limit is exceeded, keep len_limit + if len(param) > len_limit: + logging.error("%s length more than %d, keep the first %d", param, len_limit, len_limit) + param[:] = param[0:len_limit] + + # only keep elements under the char_limit length + param[:] = [elem for elem in param if len(elem) <= char_limit] + + return [True, ret] + +def is_iocollect_valid(period, disk_list=None, stage=None): + result = inter_is_iocollect_valid(period, disk_list, stage) + error_code = result['ret'] + if error_code != ResultMessage.RESULT_SUCCEED: + result['message'] = Result_Messages[error_code] + return result + +def inter_is_iocollect_valid(period, disk_list=None, stage=None): + result = {} + result['ret'] = ResultMessage.RESULT_UNKNOWN + result['message'] = "" + + if not period or not isinstance(period, int): + result['ret'] = ResultMessage.RESULT_NOT_PARAM + return result + if period < LIMIT_PERIOD_MIN_LEN or period > LIMIT_PERIOD_MAX_LEN * LIMIT_MAX_SAVE_LEN: + result['ret'] = ResultMessage.RESULT_INVALID_LENGTH + return result + + if not disk_list: + disk_list = [] + else: + res = validate_parameters(disk_list, LIMIT_DISK_LIST_LEN, LIMIT_DISK_CHAR_LEN) + if not res[0]: + result['ret'] = res[1] + return result + + if not stage: + stage = [] + else: + res = validate_parameters(stage, LIMIT_STAGE_LIST_LEN, LIMIT_STAGE_CHAR_LEN) + if not res[0]: + result['ret'] = res[1] + return result + + req_msg_struct = { + 'disk_list': json.dumps(disk_list), + 'period': period, + 'stage': json.dumps(stage) + } + request_message = json.dumps(req_msg_struct) + result_message = client_send_and_recv(request_message, CLT_MSG_LEN_LEN, ClientProtocol.IS_IOCOLLECT_VALID) + if not result_message: + logging.error("collect_plugin: client_send_and_recv failed") + return result + + try: + json.loads(result_message) + except json.JSONDecodeError: + logging.error("is_iocollect_valid: json decode error") + result['ret'] = ResultMessage.RESULT_PARSE_FAILED + return result + + result['ret'] = ResultMessage.RESULT_SUCCEED + result['message'] = result_message + return result + +def get_io_data(period, disk_list, stage, iotype): + result = inter_get_io_data(period, disk_list, stage, iotype) + error_code = result['ret'] + if error_code != ResultMessage.RESULT_SUCCEED: + result['message'] = Result_Messages[error_code] + return result + +def inter_get_io_data(period, disk_list, stage, iotype): + result = {} + result['ret'] = ResultMessage.RESULT_UNKNOWN + result['message'] = "" + + if not isinstance(period, int): + result['ret'] = ResultMessage.RESULT_NOT_PARAM + return result + if period < LIMIT_PERIOD_MIN_LEN or period > LIMIT_PERIOD_MAX_LEN * LIMIT_MAX_SAVE_LEN: + result['ret'] = ResultMessage.RESULT_INVALID_LENGTH + return result + + res = validate_parameters(disk_list, LIMIT_DISK_LIST_LEN, LIMIT_DISK_CHAR_LEN) + if not res[0]: + result['ret'] = res[1] + return result + + res = validate_parameters(stage, LIMIT_STAGE_LIST_LEN, LIMIT_STAGE_CHAR_LEN) + if not res[0]: + result['ret'] = res[1] + return result + + res = validate_parameters(iotype, LIMIT_IOTYPE_LIST_LEN, LIMIT_IOTYPE_CHAR_LEN) + if not res[0]: + result['ret'] = res[1] + return result + + req_msg_struct = { + 'disk_list': json.dumps(disk_list), + 'period': period, + 'stage': json.dumps(stage), + 'iotype' : json.dumps(iotype) + } + + request_message = json.dumps(req_msg_struct) + result_message = client_send_and_recv(request_message, CLT_MSG_LEN_LEN, ClientProtocol.GET_IO_DATA) + if not result_message: + logging.error("collect_plugin: client_send_and_recv failed") + return result + try: + json.loads(result_message) + except json.JSONDecodeError: + logging.error("get_io_data: json decode error") + result['ret'] = ResultMessage.RESULT_PARSE_FAILED + return result + + result['ret'] = ResultMessage.RESULT_SUCCEED + result['message'] = result_message + return result + +def get_disk_type(disk): + result = {} + result['ret'] = ResultMessage.RESULT_UNKNOWN + result['message'] = "" + if not disk: + logging.error("param is invalid") + result['ret'] = ResultMessage.RESULT_NOT_PARAM + return result + if len(disk) <= 0 or len(disk) > LIMIT_DISK_CHAR_LEN: + logging.error("invalid disk length") + result['ret'] = ResultMessage.RESULT_INVALID_LENGTH + return result + pattern = r'^[a-zA-Z0-9_-]+$' + if not re.match(pattern, disk): + logging.error("%s is invalid char", disk) + result['ret'] = ResultMessage.RESULT_INVALID_CHAR + return result + + base_path = '/sys/block' + all_disk = [] + for disk_name in os.listdir(base_path): + all_disk.append(disk_name) + + if disk not in all_disk: + logging.error("disk %s is not exist", disk) + result['ret'] = ResultMessage.RESULT_DISK_NOEXIST + return result + + if disk[0:4] == "nvme": + result['message'] = str(DiskType.TYPE_NVME_SSD) + elif disk[0:2] == "sd": + disk_file = '/sys/block/{}/queue/rotational'.format(disk) + try: + with open(disk_file, 'r') as file: + num = int(file.read()) + if num == 1: + result['message'] = str(DiskType.TYPE_SATA_SSD) + elif num == 0: + result['message'] = str(DiskType.TYPE_SATA_HDD) + else: + logging.error("disk %s is not support, num = %d", disk, num) + result['ret'] = ResultMessage.RESULT_DISK_TYPE_MISMATCH + return result + except FileNotFoundError: + logging.error("The disk_file [%s] does not exist", disk_file) + result['ret'] = ResultMessage.RESULT_DISK_NOEXIST + return result + except Exception as e: + logging.error("open disk_file %s happen an error: %s", disk_file, e) + return result + else: + logging.error("disk %s is not support", disk) + result['ret'] = ResultMessage.RESULT_DISK_TYPE_MISMATCH + return result + + result['ret'] = ResultMessage.RESULT_SUCCEED + return result \ No newline at end of file diff --git a/src/python/sentryCollector/collect_server.py b/src/python/sentryCollector/collect_server.py new file mode 100644 index 0000000000000000000000000000000000000000..ad3ac0e3b6ca09b5be3a5fdbdda9e0cad034cc13 --- /dev/null +++ b/src/python/sentryCollector/collect_server.py @@ -0,0 +1,282 @@ +# coding: utf-8 +# Copyright (c) 2024 Huawei Technologies Co., Ltd. +# sysSentry is licensed under the Mulan PSL v2. +# You can use this software according to the terms and conditions of the Mulan PSL v2. +# You may obtain a copy of Mulan PSL v2 at: +# http://license.coscl.org.cn/MulanPSL2 +# THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR +# PURPOSE. +# See the Mulan PSL v2 for more details. + +""" +listen module +""" +import sys +import signal +import traceback +import socket +import os +import json +import logging +import fcntl +import select +import threading +import time + +from .collect_io import IO_GLOBAL_DATA, IO_CONFIG_DATA +from .collect_config import CollectConfig + +SENTRY_RUN_DIR = "/var/run/sysSentry" +COLLECT_SOCKET_PATH = "/var/run/sysSentry/collector.sock" + +# socket param +CLT_LISTEN_QUEUE_LEN = 5 +SERVER_EPOLL_TIMEOUT = 0.3 + +# data length param +CLT_MSG_HEAD_LEN = 9 #3+2+4 +CLT_MSG_PRO_LEN = 2 +CLT_MSG_MAGIC_LEN = 3 +CLT_MSG_LEN_LEN = 4 + +# data flag param +CLT_MAGIC = "CLT" +RES_MAGIC = "RES" + +# interface protocol +class ServerProtocol(): + IS_IOCOLLECT_VALID = 0 + GET_IO_DATA = 1 + PRO_END = 3 + +class CollectServer(): + + def __init__(self): + + self.io_global_data = {} + + self.stop_event = threading.Event() + + def is_iocollect_valid(self, data_struct): + + result_rev = {} + self.io_global_data = IO_GLOBAL_DATA + + if len(IO_CONFIG_DATA) == 0: + logging.error("the collect thread is not started, the data is invalid.") + return json.dumps(result_rev) + + period_time = IO_CONFIG_DATA[0] + max_save = IO_CONFIG_DATA[1] + + disk_list = json.loads(data_struct['disk_list']) + period = int(data_struct['period']) + stage_list = json.loads(data_struct['stage']) + + if (period < period_time) or (period > period_time * max_save) or (period % period_time): + logging.error("is_iocollect_valid: period time is invalid, user period: %d, config period_time: %d", period, period_time) + return json.dumps(result_rev) + + for disk_name, stage_info in self.io_global_data.items(): + if len(disk_list) > 0 and disk_name not in disk_list: + continue + result_rev[disk_name] = [] + if len(stage_list) == 0: + result_rev[disk_name] = list(stage_info.keys()) + continue + for stage_name, stage_data in stage_info.items(): + if stage_name in stage_list: + result_rev[disk_name].append(stage_name) + + return json.dumps(result_rev) + + def get_io_data(self, data_struct): + result_rev = {} + self.io_global_data = IO_GLOBAL_DATA + + if len(IO_CONFIG_DATA) == 0: + logging.error("the collect thread is not started, the data is invalid.") + return json.dumps(result_rev) + period_time = IO_CONFIG_DATA[0] + max_save = IO_CONFIG_DATA[1] + + period = int(data_struct['period']) + disk_list = json.loads(data_struct['disk_list']) + stage_list = json.loads(data_struct['stage']) + iotype_list = json.loads(data_struct['iotype']) + + if (period < period_time) or (period > period_time * max_save) or (period % period_time): + logging.error("get_io_data: period time is invalid, user period: %d, config period_time: %d", period, period_time) + return json.dumps(result_rev) + + collect_index = period // period_time - 1 + logging.debug("user period: %d, config period_time: %d, collect_index: %d", period, period_time, collect_index) + + for disk_name, stage_info in self.io_global_data.items(): + if disk_name not in disk_list: + continue + result_rev[disk_name] = {} + for stage_name, iotype_info in stage_info.items(): + if len(stage_list) > 0 and stage_name not in stage_list: + continue + result_rev[disk_name][stage_name] = {} + for iotype_name, iotype_info in iotype_info.items(): + if iotype_name not in iotype_list: + continue + if len(iotype_info) - 1 < collect_index: + continue + result_rev[disk_name][stage_name][iotype_name] = iotype_info[collect_index] + + return json.dumps(result_rev) + + def msg_data_process(self, msg_data, protocal_id): + """message data process""" + logging.debug("msg_data %s", msg_data) + protocol_name = msg_data[0] + try: + data_struct = json.loads(msg_data) + except json.JSONDecodeError: + logging.error("msg data process: json decode error") + return "Request message decode failed" + + if protocal_id == ServerProtocol.IS_IOCOLLECT_VALID: + res_msg = self.is_iocollect_valid(data_struct) + elif protocal_id == ServerProtocol.GET_IO_DATA: + res_msg = self.get_io_data(data_struct) + + return res_msg + + def msg_head_process(self, msg_head): + """message head process""" + ctl_magic = msg_head[:CLT_MSG_MAGIC_LEN] + if ctl_magic != CLT_MAGIC: + logging.error("recv msg head magic invalid") + return None + + protocol_str = msg_head[CLT_MSG_MAGIC_LEN:CLT_MSG_MAGIC_LEN+CLT_MSG_PRO_LEN] + try: + protocol_id = int(protocol_str) + except ValueError: + logging.error("recv msg protocol id is invalid") + return None + + data_len_str = msg_head[CLT_MSG_MAGIC_LEN+CLT_MSG_PRO_LEN:CLT_MSG_HEAD_LEN] + try: + data_len = int(data_len_str) + except ValueError: + logging.error("recv msg data len is invalid %s", data_len_str) + return None + + return [protocol_id, data_len] + + def server_recv(self, server_socket: socket.socket): + """server receive""" + try: + client_socket, _ = server_socket.accept() + logging.debug("server_fd listen ok") + except socket.error: + logging.error("server accept failed, %s", socket.error) + return + + try: + msg_head = client_socket.recv(CLT_MSG_HEAD_LEN) + logging.debug("recv msg head: %s", msg_head.decode()) + head_info = self.msg_head_process(msg_head.decode()) + except (OSError, UnicodeError): + client_socket.close() + logging.error("server recv HEAD failed") + return + + protocol_id = head_info[0] + data_len = head_info[1] + logging.debug("msg protocol id: %d, data length: %d", protocol_id, data_len) + if protocol_id >= ServerProtocol.PRO_END: + client_socket.close() + logging.error("protocol id is invalid") + return + + if data_len < 0: + client_socket.close() + logging.error("msg head parse failed") + return + + try: + msg_data = client_socket.recv(data_len) + msg_data_decode = msg_data.decode() + logging.debug("msg data %s", msg_data_decode) + except (OSError, UnicodeError): + client_socket.close() + logging.error("server recv MSG failed") + return + + res_data = self.msg_data_process(msg_data_decode, protocol_id) + logging.debug("res data %s", res_data) + + # server send + res_head = RES_MAGIC + res_head += str(protocol_id).zfill(CLT_MSG_PRO_LEN) + res_data_len = str(len(res_data)).zfill(CLT_MSG_LEN_LEN) + res_head += res_data_len + logging.debug("res head %s", res_head) + + res_msg = res_head + res_data + logging.debug("res msg %s", res_msg) + + try: + client_socket.send(res_msg.encode()) + except OSError: + logging.error("server recv failed") + finally: + client_socket.close() + return + + def server_fd_create(self): + """create server fd""" + if not os.path.exists(SENTRY_RUN_DIR): + logging.error("%s not exist, failed", SENTRY_RUN_DIR) + return None + + try: + server_fd = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) + server_fd.setblocking(False) + if os.path.exists(COLLECT_SOCKET_PATH): + os.remove(COLLECT_SOCKET_PATH) + + server_fd.bind(COLLECT_SOCKET_PATH) + os.chmod(COLLECT_SOCKET_PATH, 0o600) + server_fd.listen(CLT_LISTEN_QUEUE_LEN) + logging.debug("%s bind and listen", COLLECT_SOCKET_PATH) + except socket.error: + logging.error("server fd create failed") + server_fd = None + return server_fd + + def server_loop(self): + """main loop""" + logging.info("collect listen thread start") + server_fd = self.server_fd_create() + if not server_fd: + return + + epoll_fd = select.epoll() + epoll_fd.register(server_fd.fileno(), select.EPOLLIN) + + logging.debug("start server_loop loop") + while True: + if self.stop_event.is_set(): + logging.debug("collect listen thread exit") + server_fd = None + return + try: + events_list = epoll_fd.poll(SERVER_EPOLL_TIMEOUT) + for event_fd, _ in events_list: + if event_fd == server_fd.fileno(): + self.server_recv(server_fd) + else: + continue + except Exception: + logging.error('collect listen exception : %s', traceback.format_exc()) + + def stop_thread(self): + self.stop_event.set() diff --git a/src/python/sentryCollector/collectd.py b/src/python/sentryCollector/collectd.py new file mode 100644 index 0000000000000000000000000000000000000000..33f4b04da11368354aa641832d0449e85f4252a4 --- /dev/null +++ b/src/python/sentryCollector/collectd.py @@ -0,0 +1,99 @@ +# coding: utf-8 +# Copyright (c) 2024 Huawei Technologies Co., Ltd. +# sysSentry is licensed under the Mulan PSL v2. +# You can use this software according to the terms and conditions of the Mulan PSL v2. +# You may obtain a copy of Mulan PSL v2 at: +# http://license.coscl.org.cn/MulanPSL2 +# THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR +# PURPOSE. +# See the Mulan PSL v2 for more details. + +""" +main loop for collect. +""" +import sys +import signal +import traceback +import socket +import os +import json +import logging +import fcntl +import select + +import threading + +from .collect_io import CollectIo +from .collect_server import CollectServer +from .collect_config import CollectConfig, get_log_level + +SENTRY_RUN_DIR = "/var/run/sysSentry" +COLLECT_SOCKET_PATH = "/var/run/sysSentry/collector.sock" +SENTRY_RUN_DIR_PERM = 0o750 + +COLLECT_LOG_FILE = "/var/log/sysSentry/collector.log" +Thread_List = [] +Module_Map_Class = {"io" : CollectIo} + +def remove_sock_file(): + try: + os.unlink(COLLECT_SOCKET_PATH) + except FileNotFoundError: + pass + +def sig_handler(signum, _f): + if signum not in (signal.SIGINT, signal.SIGTERM): + return + for i in range(len(Thread_List)): + Thread_List[i][0].stop_thread() + + remove_sock_file() + +def main(): + """main + """ + if not os.path.exists(SENTRY_RUN_DIR): + os.mkdir(SENTRY_RUN_DIR) + os.chmod(SENTRY_RUN_DIR, mode=SENTRY_RUN_DIR_PERM) + + log_level = get_log_level() + log_format = "%(asctime)s - %(levelname)s - [%(filename)s:%(lineno)d] - %(message)s" + logging.basicConfig(filename=COLLECT_LOG_FILE, level=log_level, format=log_format) + os.chmod(COLLECT_LOG_FILE, 0o600) + + try: + signal.signal(signal.SIGINT, sig_handler) + signal.signal(signal.SIGTERM, sig_handler) + + logging.info("finish main parse_args") + + module_config = CollectConfig() + module_list = module_config.modules + + # listen thread + cs = CollectServer() + listen_thread = threading.Thread(target=cs.server_loop) + listen_thread.start() + Thread_List.append([cs, listen_thread]) + + # collect thread + for info in module_list: + class_name = Module_Map_Class.get(info) + if not class_name: + logging.info("%s correspond to class is not exist", info) + continue + cn = class_name(module_config) + collect_thread = threading.Thread(target=cn.main_loop) + collect_thread.start() + Thread_List.append([cn, collect_thread]) + + for i in range(len(Thread_List)): + Thread_List[i][1].join() + + except Exception: + logging.error('%s', traceback.format_exc()) + finally: + pass + + logging.info("all threads have finished. main thread exit.") \ No newline at end of file diff --git a/src/python/sentryPlugins/__init__.py b/src/python/sentryPlugins/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/src/python/sentryPlugins/ai_block_io/README.md b/src/python/sentryPlugins/ai_block_io/README.md new file mode 100644 index 0000000000000000000000000000000000000000..95c11116e5ed8ac544818f567b62ad3e6a75281d --- /dev/null +++ b/src/python/sentryPlugins/ai_block_io/README.md @@ -0,0 +1 @@ +# slow_io_detection diff --git a/src/python/sentryPlugins/ai_block_io/__init__.py b/src/python/sentryPlugins/ai_block_io/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/src/python/sentryPlugins/ai_block_io/ai_block_io.py b/src/python/sentryPlugins/ai_block_io/ai_block_io.py new file mode 100644 index 0000000000000000000000000000000000000000..8075f5ff7c3319119fc1f410bccb4d25a8c536fe --- /dev/null +++ b/src/python/sentryPlugins/ai_block_io/ai_block_io.py @@ -0,0 +1,239 @@ +# coding: utf-8 +# Copyright (c) 2024 Huawei Technologies Co., Ltd. +# sysSentry is licensed under the Mulan PSL v2. +# You can use this software according to the terms and conditions of the Mulan PSL v2. +# You may obtain a copy of Mulan PSL v2 at: +# http://license.coscl.org.cn/MulanPSL2 +# THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR +# PURPOSE. +# See the Mulan PSL v2 for more details. + +import time +import signal +import logging +from collections import defaultdict + +from .detector import Detector, DiskDetector +from .threshold import ThresholdFactory, ThresholdType +from .sliding_window import SlidingWindowFactory +from .utils import get_data_queue_size_and_update_size +from .config_parser import ConfigParser +from .data_access import ( + get_io_data_from_collect_plug, + check_collect_valid, + get_disk_type, + check_disk_is_available +) +from .io_data import MetricName +from .alarm_report import Xalarm, Report + +CONFIG_FILE = "/etc/sysSentry/plugins/ai_block_io.ini" + + +def sig_handler(signum, frame): + Report.report_pass(f"receive signal: {signum}, exiting...") + logging.info("Finished ai_block_io plugin running.") + exit(signum) + + +class SlowIODetection: + _config_parser = None + _disk_list = [] + _detector_name_list = defaultdict(list) + _disk_detectors = {} + + def __init__(self, config_parser: ConfigParser): + self._config_parser = config_parser + self.__init_detector_name_list() + self.__init_detector() + + def __init_detector_name_list(self): + disks: list = self._config_parser.disks_to_detection + stages: list = self._config_parser.stage + iotypes: list = self._config_parser.iotype + + if disks is None: + logging.warning("you not specify any disk or use default, so ai_block_io will enable all available disk.") + all_available_disk_list = check_collect_valid(self._config_parser.period_time) + if all_available_disk_list is None: + Report.report_pass("get available disk error, please check if the collector plug is enable. exiting...") + logging.critical("get available disk error, please check if the collector plug is enable. exiting...") + exit(1) + if len(all_available_disk_list) == 0: + Report.report_pass("not found available disk. exiting...") + logging.critical("not found available disk. exiting...") + exit(1) + disks = all_available_disk_list + logging.info(f"available disk list is follow: {disks}.") + + for disk in disks: + tmp_disk = [disk] + ret = check_disk_is_available(self._config_parser.period_time, tmp_disk) + if not ret: + logging.warning(f"disk: {disk} is not available, it will be ignored.") + continue + + disk_type_result = get_disk_type(disk) + if disk_type_result["ret"] == 0 and disk_type_result["message"] in ( + '0', + '1', + '2', + ): + disk_type = int(disk_type_result["message"]) + else: + logging.warning( + "%s get disk type error, return %s, so it will be ignored.", + disk, + disk_type_result, + ) + continue + self._disk_list.append(disk) + for stage in stages: + for iotype in iotypes: + self._detector_name_list[disk].append(MetricName(disk, disk_type, stage, iotype, "latency")) + self._detector_name_list[disk].append(MetricName(disk, disk_type, stage, iotype, "io_dump")) + + if not self._detector_name_list: + Report.report_pass("the disks to detection is empty, ai_block_io will exit.") + logging.critical("the disks to detection is empty, ai_block_io will exit.") + exit(1) + + def __init_detector(self): + train_data_duration, train_update_duration = ( + self._config_parser.get_train_data_duration_and_train_update_duration() + ) + slow_io_detection_frequency = self._config_parser.period_time + threshold_type = self._config_parser.algorithm_type + data_queue_size, update_size = get_data_queue_size_and_update_size( + train_data_duration, train_update_duration, slow_io_detection_frequency + ) + sliding_window_type = self._config_parser.sliding_window_type + window_size, window_threshold = ( + self._config_parser.get_window_size_and_window_minimum_threshold() + ) + + for disk, metric_name_list in self._detector_name_list.items(): + disk_detector = DiskDetector(disk) + for metric_name in metric_name_list: + + if metric_name.metric_name == 'latency': + threshold = ThresholdFactory().get_threshold( + threshold_type, + boxplot_parameter=self._config_parser.boxplot_parameter, + n_sigma_paramter=self._config_parser.n_sigma_parameter, + data_queue_size=data_queue_size, + data_queue_update_size=update_size, + ) + tot_lim = self._config_parser.get_tot_lim( + metric_name.disk_type, metric_name.io_access_type_name + ) + avg_lim = self._config_parser.get_avg_lim( + metric_name.disk_type, metric_name.io_access_type_name + ) + if tot_lim is None: + logging.warning( + "disk %s, disk type %s, io type %s, get tot lim error, so it will be ignored.", + disk, + metric_name.disk_type, + metric_name.io_access_type_name, + ) + sliding_window = SlidingWindowFactory().get_sliding_window( + sliding_window_type, + queue_length=window_size, + threshold=window_threshold, + abs_threshold=tot_lim, + avg_lim=avg_lim + ) + detector = Detector(metric_name, threshold, sliding_window) + disk_detector.add_detector(detector) + continue + + elif metric_name.metric_name == 'io_dump': + threshold = ThresholdFactory().get_threshold(ThresholdType.AbsoluteThreshold) + abs_threshold = None + if metric_name.io_access_type_name == 'read': + abs_threshold = self._config_parser.read_iodump_lim + elif metric_name.io_access_type_name == 'write': + abs_threshold = self._config_parser.write_iodump_lim + sliding_window = SlidingWindowFactory().get_sliding_window( + sliding_window_type, + queue_length=window_size, + threshold=window_threshold + ) + detector = Detector(metric_name, threshold, sliding_window) + threshold.set_threshold(abs_threshold) + disk_detector.add_detector(detector) + + logging.info(f"disk: [{disk}] add detector:\n [{disk_detector}]") + self._disk_detectors[disk] = disk_detector + + def launch(self): + while True: + logging.debug("step0. AI threshold slow io event detection is looping.") + + # Step1：获取IO数据 + io_data_dict_with_disk_name = get_io_data_from_collect_plug( + self._config_parser.period_time, self._disk_list + ) + logging.debug(f"step1. Get io data: {str(io_data_dict_with_disk_name)}") + if io_data_dict_with_disk_name is None: + Report.report_pass( + "get io data error, please check if the collector plug is enable. exitting..." + ) + exit(1) + + # Step2：慢IO检测 + logging.debug("step2. Start to detection slow io event.") + slow_io_event_list = [] + for disk, disk_detector in self._disk_detectors.items(): + result = disk_detector.is_slow_io_event(io_data_dict_with_disk_name) + if result[0]: + slow_io_event_list.append(result) + logging.debug("step2. End to detection slow io event.") + + # Step3：慢IO事件上报 + logging.debug("step3. Report slow io event to sysSentry.") + for slow_io_event in slow_io_event_list: + alarm_content = { + "alarm_source": "ai_block_io", + "driver_name": slow_io_event[1], + "io_type": slow_io_event[4], + "reason": slow_io_event[2], + "block_stack": slow_io_event[3], + "alarm_type": slow_io_event[5], + "details": slow_io_event[6] + } + Xalarm.major(alarm_content) + tmp_alarm_content = alarm_content.copy() + del tmp_alarm_content["details"] + logging.warning("[SLOW IO] " + str(tmp_alarm_content)) + logging.warning(f'[SLOW IO] disk: {str(tmp_alarm_content.get("driver_name"))}, ' + f'stage: {str(tmp_alarm_content.get("driver_name"))}, ' + f'iotype: {str(tmp_alarm_content.get("io_type"))}, ' + f'type: {str(tmp_alarm_content.get("alarm_type"))}, ' + f'reason: {str(tmp_alarm_content.get("reason"))}') + logging.warning(f"latency: " + str(alarm_content.get("details").get("latency"))) + logging.warning(f"iodump: " + str(alarm_content.get("details").get("iodump"))) + + # Step4：等待检测时间 + logging.debug("step4. Wait to start next slow io event detection loop.") + time.sleep(self._config_parser.period_time) + + +def main(): + # Step1：注册消息处理函数 + signal.signal(signal.SIGINT, sig_handler) + signal.signal(signal.SIGTERM, sig_handler) + + # Step2：断点恢复 + # todo: + + # Step3：读取配置 + config_file_name = CONFIG_FILE + config = ConfigParser(config_file_name) + config.read_config_from_file() + + # Step4：启动慢IO检测 + slow_io_detection = SlowIODetection(config) + slow_io_detection.launch() diff --git a/src/python/sentryPlugins/ai_block_io/alarm_report.py b/src/python/sentryPlugins/ai_block_io/alarm_report.py new file mode 100644 index 0000000000000000000000000000000000000000..61bb145bdead8ac9c8e688be5b4fa4cd9c030729 --- /dev/null +++ b/src/python/sentryPlugins/ai_block_io/alarm_report.py @@ -0,0 +1,80 @@ +# coding: utf-8 +# Copyright (c) 2024 Huawei Technologies Co., Ltd. +# sysSentry is licensed under the Mulan PSL v2. +# You can use this software according to the terms and conditions of the Mulan PSL v2. +# You may obtain a copy of Mulan PSL v2 at: +# http://license.coscl.org.cn/MulanPSL2 +# THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR +# PURPOSE. +# See the Mulan PSL v2 for more details. + +import logging +import json + +from xalarm.sentry_notify import ( + xalarm_report, + MINOR_ALM, + MAJOR_ALM, + CRITICAL_ALM, + ALARM_TYPE_OCCUR, + ALARM_TYPE_RECOVER, +) + +from syssentry.result import ResultLevel, report_result + + +class Report: + TASK_NAME = "ai_block_io" + + @staticmethod + def report_pass(info: str): + report_result(Report.TASK_NAME, ResultLevel.PASS, json.dumps({"msg": info})) + logging.debug(f'Report {Report.TASK_NAME} PASS: {info}') + + @staticmethod + def report_fail(info: str): + report_result(Report.TASK_NAME, ResultLevel.FAIL, json.dumps({"msg": info})) + logging.debug(f'Report {Report.TASK_NAME} FAIL: {info}') + + @staticmethod + def report_skip(info: str): + report_result(Report.TASK_NAME, ResultLevel.SKIP, json.dumps({"msg": info})) + logging.debug(f'Report {Report.TASK_NAME} SKIP: {info}') + + +class Xalarm: + ALARM_ID = 1002 + + @staticmethod + def minor(info: dict): + info_str = json.dumps(info) + xalarm_report(Xalarm.ALARM_ID, MINOR_ALM, ALARM_TYPE_OCCUR, info_str) + logging.debug(f"Report {Xalarm.ALARM_ID} MINOR_ALM: {info_str}") + + @staticmethod + def major(info: dict): + info_str = json.dumps(info) + xalarm_report(Xalarm.ALARM_ID, MAJOR_ALM, ALARM_TYPE_OCCUR, info_str) + logging.debug(f"Report {Xalarm.ALARM_ID} MAJOR_ALM: {info_str}") + + @staticmethod + def critical(info: dict): + info_str = json.dumps(info) + xalarm_report(Xalarm.ALARM_ID, CRITICAL_ALM, ALARM_TYPE_OCCUR, info_str) + logging.debug(f"Report {Xalarm.ALARM_ID} CRITICAL_ALM: {info_str}") + + def minor_recover(info: dict): + info_str = json.dumps(info) + xalarm_report(Xalarm.ALARM_ID, MINOR_ALM, ALARM_TYPE_RECOVER, info_str) + logging.debug(f"Report {Xalarm.ALARM_ID} MINOR_ALM Recover: {info_str}") + + def major_recover(info: dict): + info_str = json.dumps(info) + xalarm_report(Xalarm.ALARM_ID, MAJOR_ALM, ALARM_TYPE_RECOVER, info_str) + logging.debug(f"Report {Xalarm.ALARM_ID} MAJOR_ALM Recover: {info_str}") + + def critical_recover(info: dict): + info_str = json.dumps(info) + xalarm_report(Xalarm.ALARM_ID, CRITICAL_ALM, ALARM_TYPE_RECOVER, info_str) + logging.debug(f"Report {Xalarm.ALARM_ID} CRITICAL_ALM Recover: {info_str}") diff --git a/src/python/sentryPlugins/ai_block_io/config_parser.py b/src/python/sentryPlugins/ai_block_io/config_parser.py new file mode 100644 index 0000000000000000000000000000000000000000..1bbb6091ea962240b45864683caeb70f38e0ecf2 --- /dev/null +++ b/src/python/sentryPlugins/ai_block_io/config_parser.py @@ -0,0 +1,742 @@ +# coding: utf-8 +# Copyright (c) 2024 Huawei Technologies Co., Ltd. +# sysSentry is licensed under the Mulan PSL v2. +# You can use this software according to the terms and conditions of the Mulan PSL v2. +# You may obtain a copy of Mulan PSL v2 at: +# http://license.coscl.org.cn/MulanPSL2 +# THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR +# PURPOSE. +# See the Mulan PSL v2 for more details. + +import os +import configparser +import logging + +from .alarm_report import Report +from .threshold import ThresholdType +from .utils import get_threshold_type_enum, get_sliding_window_type_enum, get_log_level +from .data_access import check_detect_frequency_is_valid + + +LOG_FORMAT = "%(asctime)s - %(levelname)s - [%(filename)s:%(lineno)d] - %(message)s" + +ALL_STAGE_LIST = [ + "throtl", + "wbt", + "gettag", + "plug", + "deadline", + "hctx", + "requeue", + "rq_driver", + "bio", +] +ALL_IOTPYE_LIST = ["read", "write"] +DISK_TYPE_MAP = { + 0: "nvme_ssd", + 1: "sata_ssd", + 2: "sata_hdd", +} + + +def init_log_format(log_level: str): + logging.basicConfig(level=get_log_level(log_level.lower()), format=LOG_FORMAT) + if log_level.lower() not in ("info", "warning", "error", "debug"): + logging.warning( + "the log_level: %s you set is invalid, use default value: info.", log_level + ) + + +class ConfigParser: + DEFAULT_CONF = { + "log": {"level": "info"}, + "common": { + "period_time": 1, + "disk": None, + "stage": "throtl,wbt,gettag,plug,deadline,hctx,requeue,rq_driver,bio", + "iotype": "read,write", + }, + "algorithm": { + "train_data_duration": 24.0, + "train_update_duration": 2.0, + "algorithm_type": get_threshold_type_enum("boxplot"), + "boxplot_parameter": 1.5, + "n_sigma_parameter": 3.0, + "win_type": get_sliding_window_type_enum("not_continuous"), + "win_size": 30, + "win_threshold": 6, + }, + "latency_sata_ssd": { + "read_avg_lim": 10000, + "write_avg_lim": 10000, + "read_tot_lim": 50000, + "write_tot_lim": 50000 + }, + "latency_nvme_ssd": { + "read_avg_lim": 10000, + "write_avg_lim": 10000, + "read_tot_lim": 50000, + "write_tot_lim": 50000 + }, + "latency_sata_hdd": { + "read_avg_lim": 15000, + "write_avg_lim": 15000, + "read_tot_lim": 50000, + "write_tot_lim": 50000 + }, + "iodump": { + "read_iodump_lim": 0, + "write_iodump_lim": 0 + } + } + + def __init__(self, config_file_name): + self._conf = ConfigParser.DEFAULT_CONF + self._config_file_name = config_file_name + + def _get_config_value( + self, + config_items: dict, + key: str, + value_type, + default_value=None, + gt=None, + ge=None, + lt=None, + le=None, + section=None + ): + if section is not None: + print_key = section + "." + key + else: + print_key = key + value = config_items.get(key) + if value is None: + logging.warning( + "config of %s not found, the default value %s will be used.", + print_key, + default_value, + ) + value = default_value + if not value: + logging.critical( + "the value of %s is empty, ai_block_io plug will exit.", print_key + ) + Report.report_pass( + f"the value of {print_key} is empty, ai_block_io plug will exit." + ) + exit(1) + try: + value = value_type(value) + except ValueError: + logging.critical( + "the value of %s is not a valid %s, ai_block_io plug will exit.", + print_key, + value_type, + ) + Report.report_pass( + f"the value of {print_key} is not a valid {value_type}, ai_block_io plug will exit." + ) + exit(1) + if gt is not None and value <= gt: + logging.critical( + "the value of %s is not greater than %s, ai_block_io plug will exit.", + print_key, + gt, + ) + Report.report_pass( + f"the value of {print_key} is not greater than {gt}, ai_block_io plug will exit." + ) + exit(1) + if ge is not None and value < ge: + logging.critical( + "the value of %s is not greater than or equal to %s, ai_block_io plug will exit.", + print_key, + ge, + ) + Report.report_pass( + f"the value of {print_key} is not greater than or equal to {ge}, ai_block_io plug will exit." + ) + exit(1) + if lt is not None and value >= lt: + logging.critical( + "the value of %s is not less than %s, ai_block_io plug will exit.", + print_key, + lt, + ) + Report.report_pass( + f"the value of {print_key} is not less than {lt}, ai_block_io plug will exit." + ) + exit(1) + if le is not None and value > le: + logging.critical( + "the value of %s is not less than or equal to %s, ai_block_io plug will exit.", + print_key, + le, + ) + Report.report_pass( + f"the value of {print_key} is not less than or equal to {le}, ai_block_io plug will exit." + ) + exit(1) + + return value + + def _read_period_time(self, items_common: dict): + self._conf["common"]["period_time"] = self._get_config_value( + items_common, + "period_time", + int, + self.DEFAULT_CONF["common"]["period_time"], + gt=0 + ) + frequency = self._conf["common"]["period_time"] + ret = check_detect_frequency_is_valid(frequency) + if ret is None: + log = f"period_time: {frequency} is invalid, "\ + f"Check whether the value range is too large or is not an "\ + f"integer multiple of period_time.. exiting..." + Report.report_pass(log) + logging.critical(log) + exit(1) + + def _read_disks_to_detect(self, items_common: dict): + disks_to_detection = items_common.get("disk") + if disks_to_detection is None: + logging.warning("config of disk not found, the default value will be used.") + self._conf["common"]["disk"] = None + return + disks_to_detection = disks_to_detection.strip() + disks_to_detection = disks_to_detection.lower() + if not disks_to_detection: + logging.critical("the value of disk is empty, ai_block_io plug will exit.") + Report.report_pass( + "the value of disk is empty, ai_block_io plug will exit." + ) + exit(1) + disk_list = disks_to_detection.split(",") + disk_list = [disk.strip() for disk in disk_list] + if len(disk_list) == 1 and disk_list[0] == "default": + self._conf["common"]["disk"] = None + return + if len(disk_list) > 10: + ten_disk_list = disk_list[0:10] + other_disk_list = disk_list[10:] + logging.warning(f"disk only support maximum is 10, disks: {ten_disk_list} will be retained, other: {other_disk_list} will be ignored.") + else: + ten_disk_list = disk_list + set_ten_disk_list = set(ten_disk_list) + if len(ten_disk_list) > len(set_ten_disk_list): + tmp = ten_disk_list + ten_disk_list = list(set_ten_disk_list) + logging.warning(f"disk exist duplicate, it will be deduplicate, before: {tmp}, after: {ten_disk_list}") + self._conf["common"]["disk"] = ten_disk_list + + def _read_train_data_duration(self, items_algorithm: dict): + self._conf["algorithm"]["train_data_duration"] = self._get_config_value( + items_algorithm, + "train_data_duration", + float, + self.DEFAULT_CONF["algorithm"]["train_data_duration"], + gt=0, + le=720, + ) + + def _read_train_update_duration(self, items_algorithm: dict): + default_train_update_duration = self.DEFAULT_CONF["algorithm"][ + "train_update_duration" + ] + if default_train_update_duration > self._conf["algorithm"]["train_data_duration"]: + default_train_update_duration = ( + self._conf["algorithm"]["train_data_duration"] / 2 + ) + self._conf["algorithm"]["train_update_duration"] = self._get_config_value( + items_algorithm, + "train_update_duration", + float, + default_train_update_duration, + gt=0, + le=self._conf["algorithm"]["train_data_duration"], + ) + + def _read_algorithm_type_and_parameter(self, items_algorithm: dict): + algorithm_type = items_algorithm.get("algorithm_type") + if algorithm_type is None: + default_algorithm_type = self._conf["algorithm"]["algorithm_type"] + logging.warning(f"algorithm_type not found, it will be set default: {default_algorithm_type}") + else: + self._conf["algorithm"]["algorithm_type"] = get_threshold_type_enum(algorithm_type) + + if self._conf["algorithm"]["algorithm_type"] is None: + logging.critical( + "the algorithm_type: %s you set is invalid. ai_block_io plug will exit.", + algorithm_type, + ) + Report.report_pass( + f"the algorithm_type: {algorithm_type} you set is invalid. ai_block_io plug will exit." + ) + exit(1) + + elif self._conf["algorithm"]["algorithm_type"] == ThresholdType.NSigmaThreshold: + self._conf["algorithm"]["n_sigma_parameter"] = self._get_config_value( + items_algorithm, + "n_sigma_parameter", + float, + self.DEFAULT_CONF["algorithm"]["n_sigma_parameter"], + gt=0, + le=10, + ) + elif ( + self._conf["algorithm"]["algorithm_type"] == ThresholdType.BoxplotThreshold + ): + self._conf["algorithm"]["boxplot_parameter"] = self._get_config_value( + items_algorithm, + "boxplot_parameter", + float, + self.DEFAULT_CONF["algorithm"]["boxplot_parameter"], + gt=0, + le=10, + ) + + def _read_stage(self, items_algorithm: dict): + stage_str = items_algorithm.get("stage") + if stage_str is None: + stage_str = self.DEFAULT_CONF["common"]["stage"] + logging.warning(f"stage not found, it will be set default: {stage_str}") + else: + stage_str = stage_str.strip() + + stage_str = stage_str.lower() + stage_list = stage_str.split(",") + stage_list = [stage.strip() for stage in stage_list] + if len(stage_list) == 1 and stage_list[0] == "": + logging.critical("stage value not allow is empty, exiting...") + exit(1) + if len(stage_list) == 1 and stage_list[0] == "default": + logging.warning( + "stage will enable default value: %s", + self.DEFAULT_CONF["common"]["stage"], + ) + self._conf["common"]["stage"] = ALL_STAGE_LIST + return + for stage in stage_list: + if stage not in ALL_STAGE_LIST: + logging.critical( + "stage: %s is not valid stage, ai_block_io will exit...", stage + ) + exit(1) + dup_stage_list = set(stage_list) + if "bio" not in dup_stage_list: + logging.critical("stage must contains bio stage, exiting...") + exit(1) + self._conf["common"]["stage"] = dup_stage_list + + def _read_iotype(self, items_algorithm: dict): + iotype_str = items_algorithm.get("iotype") + if iotype_str is None: + iotype_str = self.DEFAULT_CONF["common"]["iotype"] + logging.warning(f"iotype not found, it will be set default: {iotype_str}") + else: + iotype_str = iotype_str.strip() + + iotype_str = iotype_str.lower() + iotype_list = iotype_str.split(",") + iotype_list = [iotype.strip() for iotype in iotype_list] + if len(iotype_list) == 1 and iotype_list[0] == "": + logging.critical("iotype value not allow is empty, exiting...") + exit(1) + if len(iotype_list) == 1 and iotype_list[0] == "default": + logging.warning( + "iotype will enable default value: %s", + self.DEFAULT_CONF["common"]["iotype"], + ) + self._conf["common"]["iotype"] = ALL_IOTPYE_LIST + return + for iotype in iotype_list: + if iotype not in ALL_IOTPYE_LIST: + logging.critical( + "iotype: %s is not valid iotype, ai_block_io will exit...", iotype + ) + exit(1) + dup_iotype_list = set(iotype_list) + self._conf["common"]["iotype"] = dup_iotype_list + + def _read_sliding_window_type(self, items_sliding_window: dict): + sliding_window_type = items_sliding_window.get("win_type") + + if sliding_window_type is None: + default_sliding_window_type = self._conf["algorithm"]["win_type"] + logging.warning(f"win_type not found, it will be set default: {default_sliding_window_type}") + return + + sliding_window_type = sliding_window_type.strip() + if sliding_window_type is not None: + self._conf["algorithm"]["win_type"] = ( + get_sliding_window_type_enum(sliding_window_type) + ) + if self._conf["algorithm"]["win_type"] is None: + logging.critical( + "the win_type: %s you set is invalid. ai_block_io plug will exit.", + sliding_window_type, + ) + Report.report_pass( + f"the win_type: {sliding_window_type} you set is invalid. ai_block_io plug will exit." + ) + exit(1) + + def _read_window_size(self, items_sliding_window: dict): + self._conf["algorithm"]["win_size"] = self._get_config_value( + items_sliding_window, + "win_size", + int, + self.DEFAULT_CONF["algorithm"]["win_size"], + gt=0, + le=300, + ) + + def _read_window_minimum_threshold(self, items_sliding_window: dict): + default_window_minimum_threshold = self.DEFAULT_CONF["algorithm"]["win_threshold"] + self._conf["algorithm"]["win_threshold"] = ( + self._get_config_value( + items_sliding_window, + "win_threshold", + int, + default_window_minimum_threshold, + gt=0, + le=self._conf["algorithm"]["win_size"], + ) + ) + + def read_config_from_file(self): + if not os.path.exists(self._config_file_name): + init_log_format(self._conf["log"]["level"]) + logging.critical( + "config file %s not found, ai_block_io plug will exit.", + self._config_file_name, + ) + Report.report_pass( + f"config file {self._config_file_name} not found, ai_block_io plug will exit." + ) + exit(1) + + con = configparser.ConfigParser() + try: + con.read(self._config_file_name, encoding="utf-8") + except configparser.Error as e: + init_log_format(self._conf["log"]["level"]) + logging.critical( + "config file read error: %s, ai_block_io plug will exit.", e + ) + Report.report_pass( + f"config file read error: {e}, ai_block_io plug will exit." + ) + exit(1) + + if con.has_section("log"): + items_log = dict(con.items("log")) + # 情况一：没有log，则使用默认值 + # 情况二：有log，值为空或异常，使用默认值 + # 情况三：有log，值正常，则使用该值 + self._conf["log"]["level"] = items_log.get( + "level", self.DEFAULT_CONF["log"]["level"] + ) + init_log_format(self._conf["log"]["level"]) + else: + init_log_format(self._conf["log"]["level"]) + logging.warning( + "log section parameter not found, it will be set to default value." + ) + + if con.has_section("common"): + items_common = dict(con.items("common")) + + self._read_period_time(items_common) + self._read_disks_to_detect(items_common) + self._read_stage(items_common) + self._read_iotype(items_common) + else: + Report.report_pass("not found common section. exiting...") + logging.critical("not found common section. exiting...") + exit(1) + + if con.has_section("algorithm"): + items_algorithm = dict(con.items("algorithm")) + self._read_train_data_duration(items_algorithm) + self._read_train_update_duration(items_algorithm) + self._read_algorithm_type_and_parameter(items_algorithm) + self._read_sliding_window_type(items_algorithm) + self._read_window_size(items_algorithm) + self._read_window_minimum_threshold(items_algorithm) + + if con.has_section("latency_sata_ssd"): + items_latency_sata_ssd = dict(con.items("latency_sata_ssd")) + self._conf["latency_sata_ssd"]["read_tot_lim"] = self._get_config_value( + items_latency_sata_ssd, + "read_tot_lim", + int, + self.DEFAULT_CONF["latency_sata_ssd"]["read_tot_lim"], + gt=0, + section="latency_sata_ssd" + ) + self._conf["latency_sata_ssd"]["write_tot_lim"] = self._get_config_value( + items_latency_sata_ssd, + "write_tot_lim", + int, + self.DEFAULT_CONF["latency_sata_ssd"]["write_tot_lim"], + gt=0, + section="latency_sata_ssd" + ) + self._conf["latency_sata_ssd"]["read_avg_lim"] = self._get_config_value( + items_latency_sata_ssd, + "read_avg_lim", + int, + self.DEFAULT_CONF["latency_sata_ssd"]["read_avg_lim"], + gt=0, + section="latency_sata_ssd" + ) + self._conf["latency_sata_ssd"]["write_avg_lim"] = self._get_config_value( + items_latency_sata_ssd, + "write_avg_lim", + int, + self.DEFAULT_CONF["latency_sata_ssd"]["write_avg_lim"], + gt=0, + section="latency_sata_ssd" + ) + if self._conf["latency_sata_ssd"]["read_avg_lim"] >= self._conf["latency_sata_ssd"]["read_tot_lim"]: + Report.report_pass("latency_sata_ssd.read_avg_lim must < latency_sata_ssd.read_tot_lim . exiting...") + logging.critical("latency_sata_ssd.read_avg_lim must < latency_sata_ssd.read_tot_lim . exiting...") + exit(1) + if self._conf["latency_sata_ssd"]["write_avg_lim"] >= self._conf["latency_sata_ssd"]["write_tot_lim"]: + Report.report_pass("latency_sata_ssd.write_avg_lim must < latency_sata_ssd.write_tot_lim . exiting...") + logging.critical("latency_sata_ssd.read_avg_lim must < latency_sata_ssd.read_tot_lim . exiting...") + exit(1) + else: + Report.report_pass("not found latency_sata_ssd section. exiting...") + logging.critical("not found latency_sata_ssd section. exiting...") + exit(1) + + if con.has_section("latency_nvme_ssd"): + items_latency_nvme_ssd = dict(con.items("latency_nvme_ssd")) + self._conf["latency_nvme_ssd"]["read_tot_lim"] = self._get_config_value( + items_latency_nvme_ssd, + "read_tot_lim", + int, + self.DEFAULT_CONF["latency_nvme_ssd"]["read_tot_lim"], + gt=0, + section="latency_nvme_ssd" + ) + self._conf["latency_nvme_ssd"]["write_tot_lim"] = self._get_config_value( + items_latency_nvme_ssd, + "write_tot_lim", + int, + self.DEFAULT_CONF["latency_nvme_ssd"]["write_tot_lim"], + gt=0, + section="latency_nvme_ssd" + ) + self._conf["latency_nvme_ssd"]["read_avg_lim"] = self._get_config_value( + items_latency_nvme_ssd, + "read_avg_lim", + int, + self.DEFAULT_CONF["latency_nvme_ssd"]["read_avg_lim"], + gt=0, + section="latency_nvme_ssd" + ) + self._conf["latency_nvme_ssd"]["write_avg_lim"] = self._get_config_value( + items_latency_nvme_ssd, + "write_avg_lim", + int, + self.DEFAULT_CONF["latency_nvme_ssd"]["write_avg_lim"], + gt=0, + section="latency_nvme_ssd" + ) + if self._conf["latency_nvme_ssd"]["read_avg_lim"] >= self._conf["latency_nvme_ssd"]["read_tot_lim"]: + Report.report_pass("latency_nvme_ssd.read_avg_lim must < latency_nvme_ssd.read_tot_lim . exiting...") + logging.critical("latency_nvme_ssd.read_avg_lim must < latency_nvme_ssd.read_tot_lim . exiting...") + exit(1) + if self._conf["latency_nvme_ssd"]["write_avg_lim"] >= self._conf["latency_nvme_ssd"]["write_tot_lim"]: + Report.report_pass("latency_nvme_ssd.write_avg_lim must < latency_nvme_ssd.write_tot_lim . exiting...") + logging.critical("latency_nvme_ssd.write_avg_lim must < latency_nvme_ssd.write_tot_lim . exiting...") + exit(1) + else: + Report.report_pass("not found latency_nvme_ssd section. exiting...") + logging.critical("not found latency_nvme_ssd section. exiting...") + exit(1) + + if con.has_section("latency_sata_hdd"): + items_latency_sata_hdd = dict(con.items("latency_sata_hdd")) + self._conf["latency_sata_hdd"]["read_tot_lim"] = self._get_config_value( + items_latency_sata_hdd, + "read_tot_lim", + int, + self.DEFAULT_CONF["latency_sata_hdd"]["read_tot_lim"], + gt=0, + section="latency_sata_hdd" + ) + self._conf["latency_sata_hdd"]["write_tot_lim"] = self._get_config_value( + items_latency_sata_hdd, + "write_tot_lim", + int, + self.DEFAULT_CONF["latency_sata_hdd"]["write_tot_lim"], + gt=0, + section="latency_sata_hdd" + ) + self._conf["latency_sata_hdd"]["read_avg_lim"] = self._get_config_value( + items_latency_sata_hdd, + "read_avg_lim", + int, + self.DEFAULT_CONF["latency_sata_hdd"]["read_avg_lim"], + gt=0, + section="latency_sata_hdd" + ) + self._conf["latency_sata_hdd"]["write_avg_lim"] = self._get_config_value( + items_latency_sata_hdd, + "write_avg_lim", + int, + self.DEFAULT_CONF["latency_sata_hdd"]["write_avg_lim"], + gt=0, + section="latency_sata_hdd" + ) + if self._conf["latency_sata_hdd"]["read_avg_lim"] >= self._conf["latency_sata_hdd"]["read_tot_lim"]: + Report.report_pass("latency_sata_hdd.read_avg_lim must < latency_sata_hdd.read_tot_lim . exiting...") + logging.critical("latency_sata_hdd.read_avg_lim must < latency_sata_hdd.read_tot_lim . exiting...") + exit(1) + if self._conf["latency_sata_hdd"]["write_avg_lim"] >= self._conf["latency_sata_hdd"]["write_tot_lim"]: + Report.report_pass("latency_sata_hdd.write_avg_lim must < latency_sata_hdd.write_tot_lim . exiting...") + logging.critical("latency_sata_hdd.write_avg_lim must < latency_sata_hdd.write_tot_lim . exiting...") + exit(1) + else: + Report.report_pass("not found latency_sata_hdd section. exiting...") + logging.critical("not found latency_sata_hdd section. exiting...") + exit(1) + + if con.has_section("iodump"): + items_iodump = dict(con.items("iodump")) + self._conf["iodump"]["read_iodump_lim"] = self._get_config_value( + items_iodump, + "read_iodump_lim", + int, + self.DEFAULT_CONF["iodump"]["read_iodump_lim"], + ge=0 + ) + self._conf["iodump"]["write_iodump_lim"] = self._get_config_value( + items_iodump, + "write_iodump_lim", + int, + self.DEFAULT_CONF["iodump"]["write_iodump_lim"], + ge=0 + ) + else: + Report.report_pass("not found iodump section. exiting...") + logging.critical("not found iodump section. exiting...") + exit(1) + + self.__print_all_config_value() + + def __repr__(self) -> str: + return str(self._conf) + + def __str__(self) -> str: + return str(self._conf) + + def __print_all_config_value(self): + logging.info("all config is follow:\n %s", self) + + def get_tot_lim(self, disk_type, io_type): + if io_type == "read": + return self._conf.get( + f"latency_{DISK_TYPE_MAP.get(disk_type, '')}", {} + ).get("read_tot_lim", None) + elif io_type == "write": + return self._conf.get( + f"latency_{DISK_TYPE_MAP.get(disk_type, '')}", {} + ).get("write_tot_lim", None) + else: + return None + + def get_avg_lim(self, disk_type, io_type): + if io_type == "read": + return self._conf.get( + f"latency_{DISK_TYPE_MAP.get(disk_type, '')}", {} + ).get("read_avg_lim", None) + elif io_type == "write": + return self._conf.get( + f"latency_{DISK_TYPE_MAP.get(disk_type, '')}", {} + ).get("write_avg_lim", None) + else: + return None + + def get_train_data_duration_and_train_update_duration(self): + return ( + self._conf["algorithm"]["train_data_duration"], + self._conf["algorithm"]["train_update_duration"], + ) + + def get_window_size_and_window_minimum_threshold(self): + return ( + self._conf["algorithm"]["win_size"], + self._conf["algorithm"]["win_threshold"], + ) + + @property + def period_time(self): + return self._conf["common"]["period_time"] + + @property + def algorithm_type(self): + return self._conf["algorithm"]["algorithm_type"] + + @property + def sliding_window_type(self): + return self._conf["algorithm"]["win_type"] + + @property + def train_data_duration(self): + return self._conf["algorithm"]["train_data_duration"] + + @property + def train_update_duration(self): + return self._conf["algorithm"]["train_update_duration"] + + @property + def window_size(self): + return self._conf["algorithm"]["win_size"] + + @property + def window_minimum_threshold(self): + return self._conf["algorithm"]["win_threshold"] + + @property + def absolute_threshold(self): + return self._conf["common"]["absolute_threshold"] + + @property + def log_level(self): + return self._conf["log"]["level"] + + @property + def disks_to_detection(self): + return self._conf["common"]["disk"] + + @property + def stage(self): + return self._conf["common"]["stage"] + + @property + def iotype(self): + return self._conf["common"]["iotype"] + + @property + def boxplot_parameter(self): + return self._conf["algorithm"]["boxplot_parameter"] + + @property + def n_sigma_parameter(self): + return self._conf["algorithm"]["n_sigma_parameter"] + + @property + def read_iodump_lim(self): + return self._conf["iodump"]["read_iodump_lim"] + + @property + def write_iodump_lim(self): + return self._conf["iodump"]["write_iodump_lim"] \ No newline at end of file diff --git a/src/python/sentryPlugins/ai_block_io/data_access.py b/src/python/sentryPlugins/ai_block_io/data_access.py new file mode 100644 index 0000000000000000000000000000000000000000..2f2d607499f9bc9b2fd8919eb0b3f8bca8e8c9f0 --- /dev/null +++ b/src/python/sentryPlugins/ai_block_io/data_access.py @@ -0,0 +1,127 @@ +# coding: utf-8 +# Copyright (c) 2024 Huawei Technologies Co., Ltd. +# sysSentry is licensed under the Mulan PSL v2. +# You can use this software according to the terms and conditions of the Mulan PSL v2. +# You may obtain a copy of Mulan PSL v2 at: +# http://license.coscl.org.cn/MulanPSL2 +# THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR +# PURPOSE. +# See the Mulan PSL v2 for more details. + +import json +import logging + +from sentryCollector.collect_plugin import ( + Result_Messages, + get_io_data, + is_iocollect_valid, + get_disk_type +) + + +from .io_data import IOStageData, IOData + +COLLECT_STAGES = [ + "throtl", + "wbt", + "gettag", + "plug", + "bfq", + "hctx", + "requeue", + "rq_driver", + "bio", + "iocost", +] + + +def check_collect_valid(period): + data_raw = is_iocollect_valid(period) + if data_raw["ret"] == 0: + try: + data = json.loads(data_raw["message"]) + except Exception as e: + logging.warning(f"get valid devices failed, occur exception: {e}") + return None + if not data: + logging.warning(f"get valid devices failed, return {data_raw}") + return None + return [k for k in data.keys()] + else: + logging.warning(f"get valid devices failed, return {data_raw}") + return None + + +def check_detect_frequency_is_valid(period): + data_raw = is_iocollect_valid(period) + if data_raw["ret"] == 0: + try: + data = json.loads(data_raw["message"]) + except Exception as e: + return None + if not data: + return None + return [k for k in data.keys()] + else: + return None + + +def check_disk_is_available(period_time, disk): + data_raw = is_iocollect_valid(period_time, disk) + if data_raw["ret"] == 0: + try: + data = json.loads(data_raw["message"]) + except Exception as e: + return False + if not data: + return False + return True + else: + return False + + +def _get_raw_data(period, disk_list): + return get_io_data( + period, + disk_list, + COLLECT_STAGES, + ["read", "write", "flush", "discard"], + ) + + +def _get_io_stage_data(data): + io_stage_data = IOStageData() + for data_type in ("read", "write", "flush", "discard"): + if data_type in data: + getattr(io_stage_data, data_type).latency = data[data_type][0] + getattr(io_stage_data, data_type).io_dump = data[data_type][1] + getattr(io_stage_data, data_type).io_length = data[data_type][2] + getattr(io_stage_data, data_type).iops = data[data_type][3] + return io_stage_data + + +def get_io_data_from_collect_plug(period, disk_list): + data_raw = _get_raw_data(period, disk_list) + if data_raw["ret"] == 0: + ret = {} + try: + data = json.loads(data_raw["message"]) + except json.decoder.JSONDecodeError as e: + logging.warning(f"get io data failed, {e}") + return None + + for disk in data: + disk_data = data[disk] + disk_ret = IOData() + for k, v in disk_data.items(): + try: + getattr(disk_ret, k) + setattr(disk_ret, k, _get_io_stage_data(v)) + except AttributeError: + logging.debug(f"no attr {k}") + continue + ret[disk] = disk_ret + return ret + logging.warning(f'get io data failed with message: {data_raw["message"]}') + return None diff --git a/src/python/sentryPlugins/ai_block_io/detector.py b/src/python/sentryPlugins/ai_block_io/detector.py new file mode 100644 index 0000000000000000000000000000000000000000..27fb7f7d9a3afeabd627203ab3ad3f577adafd15 --- /dev/null +++ b/src/python/sentryPlugins/ai_block_io/detector.py @@ -0,0 +1,156 @@ +# coding: utf-8 +# Copyright (c) 2024 Huawei Technologies Co., Ltd. +# sysSentry is licensed under the Mulan PSL v2. +# You can use this software according to the terms and conditions of the Mulan PSL v2. +# You may obtain a copy of Mulan PSL v2 at: +# http://license.coscl.org.cn/MulanPSL2 +# THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR +# PURPOSE. +# See the Mulan PSL v2 for more details. +import logging +from datetime import datetime + +from .io_data import MetricName +from .threshold import Threshold +from .sliding_window import SlidingWindow +from .utils import get_metric_value_from_io_data_dict_by_metric_name + + +class Detector: + + def __init__(self, metric_name: MetricName, threshold: Threshold, sliding_window: SlidingWindow): + self._metric_name = metric_name + self._threshold = threshold + # for when threshold update, it can print latest threshold with metric name + self._threshold.set_metric_name(self._metric_name) + self._slidingWindow = sliding_window + self._threshold.attach_observer(self._slidingWindow) + self._count = None + + @property + def metric_name(self): + return self._metric_name + + def get_sliding_window_data(self): + return self._slidingWindow.get_data() + + def is_slow_io_event(self, io_data_dict_with_disk_name: dict): + if self._count is None: + self._count = datetime.now() + else: + now_time = datetime.now() + time_diff = (now_time - self._count).total_seconds() + if time_diff >= 60: + logging.info(f"({self._metric_name}) 's latest ai threshold is: {self._threshold.get_threshold()}.") + self._count = None + + logging.debug(f'enter Detector: {self}') + metric_value = get_metric_value_from_io_data_dict_by_metric_name(io_data_dict_with_disk_name, self._metric_name) + if metric_value is None: + logging.debug('not found metric value, so return None.') + return (False, False), None, None, None, None + logging.debug(f'input metric value: {str(metric_value)}') + self._threshold.push_latest_data_to_queue(metric_value) + detection_result = self._slidingWindow.is_slow_io_event(metric_value) + # 检测到慢周期，由Detector负责打印info级别日志 + if detection_result[0][1]: + logging.info(f'[abnormal_period]: disk: {self._metric_name.disk_name}, ' + f'stage: {self._metric_name.stage_name}, ' + f'iotype: {self._metric_name.io_access_type_name}, ' + f'type: {self._metric_name.metric_name}, ' + f'ai_threshold: {round(detection_result[2], 3)}, ' + f'curr_val: {metric_value}') + else: + logging.debug(f'Detection result: {str(detection_result)}') + logging.debug(f'exit Detector: {self}') + return detection_result + + def __repr__(self): + return (f'disk_name: {self._metric_name.disk_name}, stage_name: {self._metric_name.stage_name},' + f' io_type_name: {self._metric_name.io_access_type_name},' + f' metric_name: {self._metric_name.metric_name}, threshold_type: {self._threshold},' + f' sliding_window_type: {self._slidingWindow}') + + +def set_to_str(parameter: set): + ret = "" + parameter = list(parameter) + length = len(parameter) + for i in range(length): + if i == 0: + ret += parameter[i] + else: + ret += "," + parameter[i] + return ret + + +class DiskDetector: + + def __init__(self, disk_name: str): + self._disk_name = disk_name + self._detector_list = [] + + def add_detector(self, detector: Detector): + self._detector_list.append(detector) + + def get_detector_list_window(self): + latency_wins = {"read": {}, "write": {}} + iodump_wins = {"read": {}, "write": {}} + for detector in self._detector_list: + if detector.metric_name.metric_name == 'latency': + latency_wins[detector.metric_name.io_access_type_name][detector.metric_name.stage_name] = detector.get_sliding_window_data() + elif detector.metric_name.metric_name == 'io_dump': + iodump_wins[detector.metric_name.io_access_type_name][detector.metric_name.stage_name] = detector.get_sliding_window_data() + return latency_wins, iodump_wins + + def is_slow_io_event(self, io_data_dict_with_disk_name: dict): + diagnosis_info = {"bio": [], "rq_driver": [], "kernel_stack": []} + for detector in self._detector_list: + # result返回内容：(是否检测到慢IO，是否检测到慢周期)、窗口、ai阈值、绝对阈值 + # 示例： (False, False), self._io_data_queue, self._ai_threshold, self._abs_threshold + result = detector.is_slow_io_event(io_data_dict_with_disk_name) + if result[0][0]: + if detector.metric_name.stage_name == "bio": + diagnosis_info["bio"].append(detector.metric_name) + elif detector.metric_name.stage_name == "rq_driver": + diagnosis_info["rq_driver"].append(detector.metric_name) + else: + diagnosis_info["kernel_stack"].append(detector.metric_name) + + if len(diagnosis_info["bio"]) == 0: + return False, None, None, None, None, None, None + + driver_name = self._disk_name + reason = "unknown" + block_stack = set() + io_type = set() + alarm_type = set() + + for key, value in diagnosis_info.items(): + for metric_name in value: + block_stack.add(metric_name.stage_name) + io_type.add(metric_name.io_access_type_name) + alarm_type.add(metric_name.metric_name) + + latency_wins, iodump_wins = self.get_detector_list_window() + details = {"latency": latency_wins, "iodump": iodump_wins} + + io_press = {"throtl", "wbt", "iocost", "bfq"} + driver_slow = {"rq_driver"} + kernel_slow = {"gettag", "plug", "deadline", "hctx", "requeue"} + + if not io_press.isdisjoint(block_stack): + reason = "io_press" + elif not driver_slow.isdisjoint(block_stack): + reason = "driver_slow" + elif not kernel_slow.isdisjoint(block_stack): + reason = "kernel_slow" + + return True, driver_name, reason, set_to_str(block_stack), set_to_str(io_type), set_to_str(alarm_type), details + + def __repr__(self): + msg = f'disk: {self._disk_name}, ' + for detector in self._detector_list: + msg += f'\n detector: [{detector}]' + return msg diff --git a/src/python/sentryPlugins/ai_block_io/io_data.py b/src/python/sentryPlugins/ai_block_io/io_data.py new file mode 100644 index 0000000000000000000000000000000000000000..604291157ae0a9d160e7adb10c9360cbcbf68a1e --- /dev/null +++ b/src/python/sentryPlugins/ai_block_io/io_data.py @@ -0,0 +1,54 @@ +# coding: utf-8 +# Copyright (c) 2024 Huawei Technologies Co., Ltd. +# sysSentry is licensed under the Mulan PSL v2. +# You can use this software according to the terms and conditions of the Mulan PSL v2. +# You may obtain a copy of Mulan PSL v2 at: +# http://license.coscl.org.cn/MulanPSL2 +# THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR +# PURPOSE. +# See the Mulan PSL v2 for more details. + +from dataclasses import dataclass, field +from datetime import datetime +from typing import Optional + + +@dataclass +class BaseData: + latency: Optional[float] = field(default_factory=lambda: None) + io_dump: Optional[int] = field(default_factory=lambda: None) + io_length: Optional[int] = field(default_factory=lambda: None) + iops: Optional[int] = field(default_factory=lambda: None) + + +@dataclass +class IOStageData: + read: BaseData = field(default_factory=lambda: BaseData()) + write: BaseData = field(default_factory=lambda: BaseData()) + flush: BaseData = field(default_factory=lambda: BaseData()) + discard: BaseData = field(default_factory=lambda: BaseData()) + + +@dataclass +class IOData: + throtl: IOStageData = field(default_factory=lambda: IOStageData()) + wbt: IOStageData = field(default_factory=lambda: IOStageData()) + gettag: IOStageData = field(default_factory=lambda: IOStageData()) + iocost: IOStageData = field(default_factory=lambda: IOStageData()) + plug: IOStageData = field(default_factory=lambda: IOStageData()) + bfq: IOStageData = field(default_factory=lambda: IOStageData()) + hctx: IOStageData = field(default_factory=lambda: IOStageData()) + requeue: IOStageData = field(default_factory=lambda: IOStageData()) + rq_driver: IOStageData = field(default_factory=lambda: IOStageData()) + bio: IOStageData = field(default_factory=lambda: IOStageData()) + time_stamp: float = field(default_factory=lambda: datetime.now().timestamp()) + + +@dataclass(frozen=True) +class MetricName: + disk_name: str + disk_type: int + stage_name: str + io_access_type_name: str + metric_name: str diff --git a/src/python/sentryPlugins/ai_block_io/sliding_window.py b/src/python/sentryPlugins/ai_block_io/sliding_window.py new file mode 100644 index 0000000000000000000000000000000000000000..a13033f2b354372b4a5d812ca5dd590035433845 --- /dev/null +++ b/src/python/sentryPlugins/ai_block_io/sliding_window.py @@ -0,0 +1,129 @@ +# coding: utf-8 +# Copyright (c) 2024 Huawei Technologies Co., Ltd. +# sysSentry is licensed under the Mulan PSL v2. +# You can use this software according to the terms and conditions of the Mulan PSL v2. +# You may obtain a copy of Mulan PSL v2 at: +# http://license.coscl.org.cn/MulanPSL2 +# THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR +# PURPOSE. +# See the Mulan PSL v2 for more details. + +from enum import Enum, unique +import numpy as np + + +@unique +class SlidingWindowType(Enum): + NotContinuousSlidingWindow = 0 + ContinuousSlidingWindow = 1 + MedianSlidingWindow = 2 + + +class SlidingWindow: + def __init__(self, queue_length: int, threshold: int, abs_threshold: int = None, avg_lim: int = None): + self._queue_length = queue_length + self._queue_threshold = threshold + self._ai_threshold = None + self._abs_threshold = abs_threshold + self._avg_lim = avg_lim + self._io_data_queue = [] + self._io_data_queue_abnormal_tag = [] + + def is_abnormal(self, data): + if self._avg_lim is not None and data < self._avg_lim: + return False + if self._ai_threshold is not None and data > self._ai_threshold: + return True + if self._abs_threshold is not None and data > self._abs_threshold: + return True + + def push(self, data: float): + if len(self._io_data_queue) == self._queue_length: + self._io_data_queue.pop(0) + self._io_data_queue_abnormal_tag.pop(0) + self._io_data_queue.append(data) + tag = self.is_abnormal(data) + self._io_data_queue_abnormal_tag.append(tag) + return tag + + def update(self, threshold): + if self._ai_threshold == threshold: + return + self._ai_threshold = threshold + self._io_data_queue_abnormal_tag.clear() + for data in self._io_data_queue: + self._io_data_queue_abnormal_tag.append(self.is_abnormal(data)) + + def is_slow_io_event(self, data): + return False, None, None, None + + def get_data(self): + return self._io_data_queue + + def __repr__(self): + return "[SlidingWindow]" + + +class NotContinuousSlidingWindow(SlidingWindow): + def is_slow_io_event(self, data): + is_abnormal_period = super().push(data) + is_slow_io_event = False + if len(self._io_data_queue) < self._queue_length or (self._ai_threshold is None and self._abs_threshold is None): + is_slow_io_event = False + if self._io_data_queue_abnormal_tag.count(True) >= self._queue_threshold: + is_slow_io_event = True + return (is_slow_io_event, is_abnormal_period), self._io_data_queue, self._ai_threshold, self._abs_threshold, self._avg_lim + + def __repr__(self): + return f"[NotContinuousSlidingWindow, window size: {self._queue_length}, threshold: {self._queue_threshold}]" + + +class ContinuousSlidingWindow(SlidingWindow): + def is_slow_io_event(self, data): + is_abnormal_period = super().push(data) + is_slow_io_event = False + if len(self._io_data_queue) < self._queue_length or (self._ai_threshold is None and self._abs_threshold is None): + is_slow_io_event = False + consecutive_count = 0 + for tag in self._io_data_queue_abnormal_tag: + if tag: + consecutive_count += 1 + if consecutive_count >= self._queue_threshold: + is_slow_io_event = True + break + else: + consecutive_count = 0 + return (is_slow_io_event, is_abnormal_period), self._io_data_queue, self._ai_threshold, self._abs_threshold, self._avg_lim + + def __repr__(self): + return f"[ContinuousSlidingWindow, window size: {self._queue_length}, threshold: {self._queue_threshold}]" + + +class MedianSlidingWindow(SlidingWindow): + def is_slow_io_event(self, data): + is_abnormal_period = super().push(data) + is_slow_io_event = False + if len(self._io_data_queue) < self._queue_length or (self._ai_threshold is None and self._abs_threshold is None): + is_slow_io_event = False + median = np.median(self._io_data_queue) + if (self._ai_threshold is not None and median > self._ai_threshold) or (self._abs_threshold is not None and median > self._abs_threshold): + is_slow_io_event = True + return (is_slow_io_event, is_abnormal_period), self._io_data_queue, self._ai_threshold, self._abs_threshold, self._avg_lim + + def __repr__(self): + return f"[MedianSlidingWindow, window size: {self._queue_length}]" + + +class SlidingWindowFactory: + def get_sliding_window( + self, sliding_window_type: SlidingWindowType, *args, **kwargs + ): + if sliding_window_type == SlidingWindowType.NotContinuousSlidingWindow: + return NotContinuousSlidingWindow(*args, **kwargs) + elif sliding_window_type == SlidingWindowType.ContinuousSlidingWindow: + return ContinuousSlidingWindow(*args, **kwargs) + elif sliding_window_type == SlidingWindowType.MedianSlidingWindow: + return MedianSlidingWindow(*args, **kwargs) + else: + return NotContinuousSlidingWindow(*args, **kwargs) diff --git a/src/python/sentryPlugins/ai_block_io/threshold.py b/src/python/sentryPlugins/ai_block_io/threshold.py new file mode 100644 index 0000000000000000000000000000000000000000..e202bb86cd4fe57576a70be2cf21dd7d4fb07239 --- /dev/null +++ b/src/python/sentryPlugins/ai_block_io/threshold.py @@ -0,0 +1,178 @@ +# coding: utf-8 +# Copyright (c) 2024 Huawei Technologies Co., Ltd. +# sysSentry is licensed under the Mulan PSL v2. +# You can use this software according to the terms and conditions of the Mulan PSL v2. +# You may obtain a copy of Mulan PSL v2 at: +# http://license.coscl.org.cn/MulanPSL2 +# THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR +# PURPOSE. +# See the Mulan PSL v2 for more details. +import logging +from enum import Enum +import queue +import numpy as np +import math + +from .sliding_window import SlidingWindow + + +class ThresholdState(Enum): + INIT = 0 + START = 1 + + +class Threshold: + + def __init__(self, data_queue_size: int = 10000, data_queue_update_size: int = 1000): + self._observer = None + self.data_queue = queue.Queue(data_queue_size) + self.data_queue_update_size = data_queue_update_size + self.new_data_size = 0 + self.threshold_state = ThresholdState.INIT + self.threshold = math.inf + self.metric_name = None + + def set_threshold(self, threshold): + self.threshold = threshold + self.threshold_state = ThresholdState.START + self.notify_observer() + + def set_metric_name(self, metric_name): + self.metric_name = metric_name + + def get_threshold(self): + if self.threshold_state == ThresholdState.INIT: + return None + return self.threshold + + def is_abnormal(self, data): + if self.threshold_state == ThresholdState.INIT: + return False + return data >= self.threshold + + # 使用观察者模式，当阈值更新时，自动同步刷新滑窗中的阈值 + def attach_observer(self, observer: SlidingWindow): + self._observer = observer + + def notify_observer(self): + if self._observer is not None: + self._observer.update(self.threshold) + + def push_latest_data_to_queue(self, data): + pass + + def __repr__(self): + return "Threshold" + + def __str__(self): + return "Threshold" + + +class AbsoluteThreshold(Threshold): + def __init__(self, data_queue_size: int = 10000, data_queue_update_size: int = 1000, **kwargs): + super().__init__(data_queue_size, data_queue_update_size) + + def push_latest_data_to_queue(self, data): + pass + + def __repr__(self): + return "[AbsoluteThreshold]" + + def __str__(self): + return "absolute" + + +class BoxplotThreshold(Threshold): + def __init__(self, boxplot_parameter: float = 1.5, data_queue_size: int = 10000, data_queue_update_size: int = 1000, **kwargs): + super().__init__(data_queue_size, data_queue_update_size) + self.parameter = boxplot_parameter + + def _update_threshold(self): + old_threshold = self.threshold + data = list(self.data_queue.queue) + q1 = np.percentile(data, 25) + q3 = np.percentile(data, 75) + iqr = q3 - q1 + self.threshold = q3 + self.parameter * iqr + if self.threshold_state == ThresholdState.INIT: + self.threshold_state = ThresholdState.START + logging.info(f"MetricName: [{self.metric_name}]'s threshold update, old is: {old_threshold} -> new is: {self.threshold}") + self.notify_observer() + + def push_latest_data_to_queue(self, data): + if data < 1e-6: + return + try: + self.data_queue.put(data, block=False) + except queue.Full: + self.data_queue.get() + self.data_queue.put(data) + self.new_data_size += 1 + if (self.data_queue.full() and (self.threshold_state == ThresholdState.INIT or + (self.threshold_state == ThresholdState.START and + self.new_data_size >= self.data_queue_update_size))): + self._update_threshold() + self.new_data_size = 0 + + def __repr__(self): + return f"[BoxplotThreshold, param is: {self.parameter}, train_size: {self.data_queue.maxsize}, update_size: {self.data_queue_update_size}]" + + def __str__(self): + return "boxplot" + + +class NSigmaThreshold(Threshold): + def __init__(self, n_sigma_parameter: float = 3.0, data_queue_size: int = 10000, data_queue_update_size: int = 1000, **kwargs): + super().__init__(data_queue_size, data_queue_update_size) + self.parameter = n_sigma_parameter + + def _update_threshold(self): + old_threshold = self.threshold + data = list(self.data_queue.queue) + mean = np.mean(data) + std = np.std(data) + self.threshold = mean + self.parameter * std + if self.threshold_state == ThresholdState.INIT: + self.threshold_state = ThresholdState.START + logging.info(f"MetricName: [{self.metric_name}]'s threshold update, old is: {old_threshold} -> new is: {self.threshold}") + self.notify_observer() + + def push_latest_data_to_queue(self, data): + if data < 1e-6: + return + try: + self.data_queue.put(data, block=False) + except queue.Full: + self.data_queue.get() + self.data_queue.put(data) + self.new_data_size += 1 + if (self.data_queue.full() and (self.threshold_state == ThresholdState.INIT or + (self.threshold_state == ThresholdState.START and + self.new_data_size >= self.data_queue_update_size))): + self._update_threshold() + self.new_data_size = 0 + + def __repr__(self): + return f"[NSigmaThreshold, param is: {self.parameter}, train_size: {self.data_queue.maxsize}, update_size: {self.data_queue_update_size}]" + + def __str__(self): + return "n_sigma" + + +class ThresholdType(Enum): + AbsoluteThreshold = 0 + BoxplotThreshold = 1 + NSigmaThreshold = 2 + + +class ThresholdFactory: + def get_threshold(self, threshold_type: ThresholdType, *args, **kwargs): + if threshold_type == ThresholdType.AbsoluteThreshold: + return AbsoluteThreshold(*args, **kwargs) + elif threshold_type == ThresholdType.BoxplotThreshold: + return BoxplotThreshold(*args, **kwargs) + elif threshold_type == ThresholdType.NSigmaThreshold: + return NSigmaThreshold(*args, **kwargs) + else: + raise ValueError(f"Invalid threshold type: {threshold_type}") diff --git a/src/python/sentryPlugins/ai_block_io/utils.py b/src/python/sentryPlugins/ai_block_io/utils.py new file mode 100644 index 0000000000000000000000000000000000000000..7d2390b9a0a788af367b52c1ed6015a4239cd55a --- /dev/null +++ b/src/python/sentryPlugins/ai_block_io/utils.py @@ -0,0 +1,73 @@ +# coding: utf-8 +# Copyright (c) 2024 Huawei Technologies Co., Ltd. +# sysSentry is licensed under the Mulan PSL v2. +# You can use this software according to the terms and conditions of the Mulan PSL v2. +# You may obtain a copy of Mulan PSL v2 at: +# http://license.coscl.org.cn/MulanPSL2 +# THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR +# PURPOSE. +# See the Mulan PSL v2 for more details. + +import logging +from dataclasses import asdict + + +from .threshold import ThresholdType +from .sliding_window import SlidingWindowType +from .io_data import MetricName, IOData + + +def get_threshold_type_enum(algorithm_type: str): + if algorithm_type.lower() == "boxplot": + return ThresholdType.BoxplotThreshold + if algorithm_type.lower() == "n_sigma": + return ThresholdType.NSigmaThreshold + return None + + +def get_sliding_window_type_enum(sliding_window_type: str): + if sliding_window_type.lower() == "not_continuous": + return SlidingWindowType.NotContinuousSlidingWindow + if sliding_window_type.lower() == "continuous": + return SlidingWindowType.ContinuousSlidingWindow + if sliding_window_type.lower() == "median": + return SlidingWindowType.MedianSlidingWindow + return None + + +def get_metric_value_from_io_data_dict_by_metric_name( + io_data_dict: dict, metric_name: MetricName +): + try: + io_data: IOData = io_data_dict[metric_name.disk_name] + io_stage_data = asdict(io_data)[metric_name.stage_name] + base_data = io_stage_data[metric_name.io_access_type_name] + metric_value = base_data[metric_name.metric_name] + return metric_value + except KeyError: + return None + + +def get_data_queue_size_and_update_size( + training_data_duration: float, + train_update_duration: float, + slow_io_detect_frequency: int, +): + data_queue_size = int(training_data_duration * 60 * 60 / slow_io_detect_frequency) + update_size = int(train_update_duration * 60 * 60 / slow_io_detect_frequency) + return data_queue_size, update_size + + +def get_log_level(log_level: str): + if log_level.lower() == "debug": + return logging.DEBUG + elif log_level.lower() == "info": + return logging.INFO + elif log_level.lower() == "warning": + return logging.WARNING + elif log_level.lower() == "error": + return logging.ERROR + elif log_level.lower() == "critical": + return logging.CRITICAL + return logging.INFO diff --git a/src/python/sentryPlugins/avg_block_io/__init__.py b/src/python/sentryPlugins/avg_block_io/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/src/python/sentryPlugins/avg_block_io/avg_block_io.py b/src/python/sentryPlugins/avg_block_io/avg_block_io.py new file mode 100644 index 0000000000000000000000000000000000000000..899d517735d488c3f54029d1f675e3a23d32f235 --- /dev/null +++ b/src/python/sentryPlugins/avg_block_io/avg_block_io.py @@ -0,0 +1,189 @@ +# coding: utf-8 +# Copyright (c) 2024 Huawei Technologies Co., Ltd. +# sysSentry is licensed under the Mulan PSL v2. +# You can use this software according to the terms and conditions of the Mulan PSL v2. +# You may obtain a copy of Mulan PSL v2 at: +# http://license.coscl.org.cn/MulanPSL2 +# THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR +# PURPOSE. +# See the Mulan PSL v2 for more details. +import logging +import signal +import configparser +import time + +from .config import read_config_log, read_config_common, read_config_algorithm, read_config_latency, read_config_iodump, read_config_stage +from .stage_window import IoWindow, IoDumpWindow +from .module_conn import avg_is_iocollect_valid, avg_get_io_data, report_alarm_fail, process_report_data, sig_handler, get_disk_type_by_name, check_disk_list_validation +from .utils import update_avg_and_check_abnormal + +CONFIG_FILE = "/etc/sysSentry/plugins/avg_block_io.ini" + + +def init_io_win(io_dic, config, common_param): + """initialize windows of latency, iodump, and dict of avg_value""" + iotype_list = io_dic["iotype_list"] + io_data = {} + io_avg_value = {} + for disk_name in io_dic["disk_list"]: + io_data[disk_name] = {} + io_avg_value[disk_name] = {} + curr_disk_type = get_disk_type_by_name(disk_name) + for stage_name in io_dic["stage_list"]: + io_data[disk_name][stage_name] = {} + io_avg_value[disk_name][stage_name] = {} + # 解析stage配置 + curr_stage_param = read_config_stage(config, stage_name, iotype_list, curr_disk_type) + for rw in iotype_list: + io_data[disk_name][stage_name][rw] = {} + io_avg_value[disk_name][stage_name][rw] = [0, 0] + + # 对每个rw创建latency和iodump窗口 + avg_lim_key = "{}_avg_lim".format(rw) + avg_time_key = "{}_avg_time".format(rw) + tot_lim_key = "{}_tot_lim".format(rw) + iodump_lim_key = "{}_iodump_lim".format(rw) + + # 获取值，优先从 curr_stage_param 获取，如果不存在，则从 common_param 获取 + avg_lim_value = curr_stage_param.get(avg_lim_key, common_param.get(curr_disk_type, {}).get(avg_lim_key)) + avg_time_value = curr_stage_param.get(avg_time_key, common_param.get(curr_disk_type, {}).get(avg_time_key)) + tot_lim_value = curr_stage_param.get(tot_lim_key, common_param.get(curr_disk_type, {}).get(tot_lim_key)) + iodump_lim_value = curr_stage_param.get(iodump_lim_key, common_param.get("iodump", {}).get(iodump_lim_key)) + + if avg_lim_value and avg_time_value and tot_lim_value: + io_data[disk_name][stage_name][rw]["latency"] = IoWindow(window_size=io_dic["win_size"], window_threshold=io_dic["win_threshold"], abnormal_multiple=avg_time_value, abnormal_multiple_lim=avg_lim_value, abnormal_time=tot_lim_value) + logging.debug("Successfully create {}-{}-{}-latency window".format(disk_name, stage_name, rw)) + + if iodump_lim_value is not None: + io_data[disk_name][stage_name][rw]["iodump"] = IoDumpWindow(window_size=io_dic["win_size"], window_threshold=io_dic["win_threshold"], abnormal_time=iodump_lim_value) + logging.debug("Successfully create {}-{}-{}-iodump window".format(disk_name, stage_name, rw)) + return io_data, io_avg_value + + +def get_valid_disk_stage_list(io_dic, config_disk, config_stage): + """get disk_list and stage_list by sentryCollector""" + json_data = avg_is_iocollect_valid(io_dic, config_disk, config_stage) + + all_disk_set = json_data.keys() + all_stage_set = set() + for disk_stage_list in json_data.values(): + all_stage_set.update(disk_stage_list) + + disk_list = [key for key in all_disk_set if key in config_disk] + not_in_disk_list = [key for key in config_disk if key not in all_disk_set] + + if not config_disk and not not_in_disk_list: + disk_list = [key for key in all_disk_set] + + if not disk_list: + report_alarm_fail("Cannot get valid disk name") + + disk_list = check_disk_list_validation(disk_list) + + disk_list = disk_list[:10] if len(disk_list) > 10 else disk_list + + if not config_disk: + logging.info(f"Default common.disk using disk={disk_list}") + elif sorted(disk_list) != sorted(config_disk): + logging.warning(f"Set common.disk to {disk_list}") + + stage_list = [key for key in all_stage_set if key in config_stage] + not_in_stage_list = [key for key in config_stage if key not in all_stage_set] + + if not_in_stage_list: + report_alarm_fail(f"Invalid common.stage_list config, cannot set {not_in_stage_list}") + + if not config_stage: + stage_list = [key for key in all_stage_set] + + if not stage_list: + report_alarm_fail("Cannot get valid stage name.") + + if not config_stage: + logging.info(f"Default common.stage using stage={stage_list}") + + return disk_list, stage_list + + +def main_loop(io_dic, io_data, io_avg_value): + """main loop of avg_block_io""" + period_time = io_dic["period_time"] + disk_list = io_dic["disk_list"] + stage_list = io_dic["stage_list"] + iotype_list = io_dic["iotype_list"] + win_size = io_dic["win_size"] + # 开始循环 + while True: + # 等待x秒 + time.sleep(period_time) + + # 采集模块对接，获取周期数据 + is_success, curr_period_data = avg_get_io_data(io_dic) + if not is_success: + logging.error(f"{curr_period_data['msg']}") + continue + + # 处理周期数据 + reach_size = False + for disk_name in disk_list: + for stage_name in stage_list: + for rw in iotype_list: + if disk_name in curr_period_data and stage_name in curr_period_data[disk_name] and rw in curr_period_data[disk_name][stage_name]: + io_key = (disk_name, stage_name, rw) + reach_size = update_avg_and_check_abnormal(curr_period_data, io_key, win_size, io_avg_value, io_data) + + # win_size不满时不进行告警判断 + if not reach_size: + continue + + # 判断异常窗口、异常场景 + for disk_name in disk_list: + for rw in iotype_list: + process_report_data(disk_name, rw, io_data) + + +def main(): + """main func""" + # 注册停止信号-2/-15 + signal.signal(signal.SIGINT, sig_handler) + signal.signal(signal.SIGTERM, sig_handler) + + log_level = read_config_log(CONFIG_FILE) + log_format = "%(asctime)s - %(levelname)s - [%(filename)s:%(lineno)d] - %(message)s" + logging.basicConfig(level=log_level, format=log_format) + + # 初始化配置读取 + config = configparser.ConfigParser(comment_prefixes=('#', ';')) + try: + config.read(CONFIG_FILE) + except configparser.Error: + report_alarm_fail("Failed to read config file") + + io_dic = {} + + # 读取配置文件 -- common段 + io_dic["period_time"], disk, stage, io_dic["iotype_list"] = read_config_common(config) + + # 采集模块对接，is_iocollect_valid() + io_dic["disk_list"], io_dic["stage_list"] = get_valid_disk_stage_list(io_dic, disk, stage) + + logging.debug(f"disk={io_dic['disk_list']}, stage={io_dic['stage_list']}") + + if "bio" not in io_dic["stage_list"]: + report_alarm_fail("Cannot run avg_block_io without bio stage") + + # 初始化窗口 -- config读取，对应is_iocollect_valid返回的结果 + # step1. 解析公共配置 --- algorithm + io_dic["win_size"], io_dic["win_threshold"] = read_config_algorithm(config) + + # step2. 解析公共配置 --- latency_xxx + common_param = read_config_latency(config) + + # step3. 解析公共配置 --- iodump + common_param['iodump'] = read_config_iodump(config) + + # step4. 循环创建窗口 + io_data, io_avg_value = init_io_win(io_dic, config, common_param) + + main_loop(io_dic, io_data, io_avg_value) diff --git a/src/python/sentryPlugins/avg_block_io/config.py b/src/python/sentryPlugins/avg_block_io/config.py new file mode 100644 index 0000000000000000000000000000000000000000..c1e8ab10425ce8f19f246aeac277a51945aede80 --- /dev/null +++ b/src/python/sentryPlugins/avg_block_io/config.py @@ -0,0 +1,208 @@ +import configparser +import logging +import os + +from .module_conn import report_alarm_fail +from sentryCollector.collect_plugin import Disk_Type + + +CONF_LOG = 'log' +CONF_LOG_LEVEL = 'level' +LogLevel = { + "debug": logging.DEBUG, + "info": logging.INFO, + "warning": logging.WARNING, + "error": logging.ERROR, + "critical": logging.CRITICAL +} + +CONF_COMMON = 'common' +CONF_COMMON_DISK = 'disk' +CONF_COMMON_STAGE = 'stage' +CONF_COMMON_IOTYPE = 'iotype' +CONF_COMMON_PER_TIME = 'period_time' + +CONF_ALGO = 'algorithm' +CONF_ALGO_SIZE = 'win_size' +CONF_ALGO_THRE = 'win_threshold' + +CONF_LATENCY = 'latency_{}' +CONF_IODUMP = 'iodump' + + +DEFAULT_PARAM = { + CONF_LOG: { + CONF_LOG_LEVEL: 'info' + }, CONF_COMMON: { + CONF_COMMON_DISK: 'default', + CONF_COMMON_STAGE: 'default', + CONF_COMMON_IOTYPE: 'read,write', + CONF_COMMON_PER_TIME: 1 + }, CONF_ALGO: { + CONF_ALGO_SIZE: 30, + CONF_ALGO_THRE: 6 + }, 'latency_nvme_ssd': { + 'read_avg_lim': 10000, + 'write_avg_lim': 10000, + 'read_avg_time': 3, + 'write_avg_time': 3, + 'read_tot_lim': 50000, + 'write_tot_lim': 50000, + }, 'latency_sata_ssd' : { + 'read_avg_lim': 10000, + 'write_avg_lim': 10000, + 'read_avg_time': 3, + 'write_avg_time': 3, + 'read_tot_lim': 50000, + 'write_tot_lim': 50000, + }, 'latency_sata_hdd' : { + 'read_avg_lim': 15000, + 'write_avg_lim': 15000, + 'read_avg_time': 3, + 'write_avg_time': 3, + 'read_tot_lim': 50000, + 'write_tot_lim': 50000 + }, CONF_IODUMP: { + 'read_iodump_lim': 0, + 'write_iodump_lim': 0 + } +} + + +def get_section_value(section_name, config): + common_param = {} + config_sec = config[section_name] + for config_key in DEFAULT_PARAM[section_name]: + if config_key in config_sec: + if not config_sec[config_key].isdecimal(): + report_alarm_fail(f"Invalid {section_name}.{config_key} config.") + common_param[config_key] = int(config_sec[config_key]) + else: + common_param[config_key] = DEFAULT_PARAM[section_name][config_key] + logging.warning(f"Unset {section_name}.{config_key} in config file, use {common_param[config_key]} as default") + return common_param + + +def read_config_log(filename): + """read config file, get [log] section value""" + default_log_level = DEFAULT_PARAM[CONF_LOG][CONF_LOG_LEVEL] + if not os.path.exists(filename): + return LogLevel.get(default_log_level) + + config = configparser.ConfigParser() + config.read(filename) + + log_level = config.get(CONF_LOG, CONF_LOG_LEVEL, fallback=default_log_level) + if log_level.lower() in LogLevel: + return LogLevel.get(log_level.lower()) + return LogLevel.get(default_log_level) + + +def read_config_common(config): + """read config file, get [common] section value""" + if not config.has_section(CONF_COMMON): + report_alarm_fail(f"Cannot find {CONF_COMMON} section in config file") + + try: + disk_name = config.get(CONF_COMMON, CONF_COMMON_DISK).lower() + disk = [] if disk_name == "default" else disk_name.split(",") + except configparser.NoOptionError: + disk = [] + logging.warning(f"Unset {CONF_COMMON}.{CONF_COMMON_DISK}, set to default") + + try: + stage_name = config.get(CONF_COMMON, CONF_COMMON_STAGE).lower() + stage = [] if stage_name == "default" else stage_name.split(",") + except configparser.NoOptionError: + stage = [] + logging.warning(f"Unset {CONF_COMMON}.{CONF_COMMON_STAGE}, set to default") + + if len(disk) > 10: + logging.warning(f"Too many {CONF_COMMON}.disks, record only max 10 disks") + disk = disk[:10] + + try: + iotype_name = config.get(CONF_COMMON, CONF_COMMON_IOTYPE).lower().split(",") + iotype_list = [rw.lower() for rw in iotype_name if rw.lower() in ['read', 'write']] + err_iotype = [rw.lower() for rw in iotype_name if rw.lower() not in ['read', 'write']] + + if err_iotype: + report_alarm_fail(f"Invalid {CONF_COMMON}.{CONF_COMMON_IOTYPE} config") + + except configparser.NoOptionError: + iotype_list = DEFAULT_PARAM[CONF_COMMON][CONF_COMMON_IOTYPE] + logging.warning(f"Unset {CONF_COMMON}.{CONF_COMMON_IOTYPE}, use {iotupe_list} as default") + + try: + period_time = int(config.get(CONF_COMMON, CONF_COMMON_PER_TIME)) + if not (1 <= period_time <= 300): + raise ValueError("Invalid period_time") + except ValueError: + report_alarm_fail(f"Invalid {CONF_COMMON}.{CONF_COMMON_PER_TIME}") + except configparser.NoOptionError: + period_time = DEFAULT_PARAM[CONF_COMMON][CONF_COMMON_PER_TIME] + logging.warning(f"Unset {CONF_COMMON}.{CONF_COMMON_PER_TIME}, use {period_time} as default") + + return period_time, disk, stage, iotype_list + + +def read_config_algorithm(config): + """read config file, get [algorithm] section value""" + if not config.has_section(CONF_ALGO): + report_alarm_fail(f"Cannot find {CONF_ALGO} section in config file") + + try: + win_size = int(config.get(CONF_ALGO, CONF_ALGO_SIZE)) + if not (1 <= win_size <= 300): + raise ValueError(f"Invalid {CONF_ALGO}.{CONF_ALGO_SIZE}") + except ValueError: + report_alarm_fail(f"Invalid {CONF_ALGO}.{CONF_ALGO_SIZE} config") + except configparser.NoOptionError: + win_size = DEFAULT_PARAM[CONF_ALGO][CONF_ALGO_SIZE] + logging.warning(f"Unset {CONF_ALGO}.{CONF_ALGO_SIZE}, use {win_size} as default") + + try: + win_threshold = int(config.get(CONF_ALGO, CONF_ALGO_THRE)) + if win_threshold < 1 or win_threshold > 300 or win_threshold > win_size: + raise ValueError(f"Invalid {CONF_ALGO}.{CONF_ALGO_THRE}") + except ValueError: + report_alarm_fail(f"Invalid {CONF_ALGO}.{CONF_ALGO_THRE} config") + except configparser.NoOptionError: + win_threshold = DEFAULT_PARAM[CONF_ALGO]['win_threshold'] + logging.warning(f"Unset {CONF_ALGO}.{CONF_ALGO_THRE}, use {win_threshold} as default") + + return win_size, win_threshold + + +def read_config_latency(config): + """read config file, get [latency_xxx] section value""" + common_param = {} + for type_name in Disk_Type: + section_name = CONF_LATENCY.format(Disk_Type[type_name]) + if not config.has_section(section_name): + report_alarm_fail(f"Cannot find {section_name} section in config file") + + common_param[Disk_Type[type_name]] = get_section_value(section_name, config) + return common_param + + +def read_config_iodump(config): + """read config file, get [iodump] section value""" + if not config.has_section(CONF_IODUMP): + report_alarm_fail(f"Cannot find {CONF_IODUMP} section in config file") + + return get_section_value(CONF_IODUMP, config) + + +def read_config_stage(config, stage, iotype_list, curr_disk_type): + """read config file, get [STAGE_NAME_diskType] section value""" + res = {} + section_name = f"{stage}_{curr_disk_type}" + if not config.has_section(section_name): + return res + + for key in config[section_name]: + if config[stage][key].isdecimal(): + res[key] = int(config[stage][key]) + + return res diff --git a/src/python/sentryPlugins/avg_block_io/module_conn.py b/src/python/sentryPlugins/avg_block_io/module_conn.py new file mode 100644 index 0000000000000000000000000000000000000000..a67ef45c5bf473645bbb5c877962c3a5472fc6c5 --- /dev/null +++ b/src/python/sentryPlugins/avg_block_io/module_conn.py @@ -0,0 +1,145 @@ +# coding: utf-8 +# Copyright (c) 2024 Huawei Technologies Co., Ltd. +# sysSentry is licensed under the Mulan PSL v2. +# You can use this software according to the terms and conditions of the Mulan PSL v2. +# You may obtain a copy of Mulan PSL v2 at: +# http://license.coscl.org.cn/MulanPSL2 +# THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR +# PURPOSE. +# See the Mulan PSL v2 for more details. +import json +import logging +import sys +import time + +from .utils import is_abnormal, get_win_data, log_slow_win +from sentryCollector.collect_plugin import is_iocollect_valid, get_io_data, Result_Messages, get_disk_type, Disk_Type +from syssentry.result import ResultLevel, report_result +from xalarm.sentry_notify import xalarm_report, MINOR_ALM, ALARM_TYPE_OCCUR + + +TASK_NAME = "avg_block_io" + +def sig_handler(signum, _f): + """stop avg_block_io""" + report_result(TASK_NAME, ResultLevel.PASS, json.dumps({})) + logging.info("Finished avg_block_io plugin running.") + sys.exit(0) + +def avg_get_io_data(io_dic): + """get_io_data from sentryCollector""" + logging.debug(f"send to sentryCollector get_io_data: period={io_dic['period_time']}, " + f"disk={io_dic['disk_list']}, stage={io_dic['stage_list']}, iotype={io_dic['iotype_list']}") + res = get_io_data(io_dic["period_time"], io_dic["disk_list"], io_dic["stage_list"], io_dic["iotype_list"]) + return check_result_validation(res, 'get io data') + + +def avg_is_iocollect_valid(io_dic, config_disk, config_stage): + """is_iocollect_valid from sentryCollector""" + logging.debug(f"send to sentryCollector is_iocollect_valid: period={io_dic['period_time']}, " + f"disk={config_disk}, stage={config_stage}") + res = is_iocollect_valid(io_dic["period_time"], config_disk, config_stage) + is_success, data = check_result_validation(res, 'check config validation') + if not is_success: + report_alarm_fail(f"{data['msg']}") + return data + + +def check_result_validation(res, reason): + """check validation of result from sentryCollector""" + if not 'ret' in res or not 'message' in res: + return False, {'msg': f"Failed to {reason}: Cannot connect to sentryCollector"} + if res['ret'] != 0: + return False, {'msg': f"Failed to {reason}: {Result_Messages[res['ret']]}"} + + try: + json_data = json.loads(res['message']) + except json.JSONDecodeError: + return False, {'msg': f"Failed to {reason}: invalid return message"} + + return True, json_data + + +def report_alarm_fail(alarm_info): + """report result to xalarmd""" + report_result(TASK_NAME, ResultLevel.FAIL, json.dumps({"msg": alarm_info})) + logging.critical(alarm_info) + sys.exit(1) + + +def process_report_data(disk_name, rw, io_data): + """check abnormal window and report to xalarm""" + abnormal, abnormal_list = is_abnormal((disk_name, 'bio', rw), io_data) + if not abnormal: + return + + msg = { + "alarm_source": TASK_NAME, "driver_name": disk_name, "io_type": rw, + "reason": "unknown", "block_stack": "bio", "alarm_type": abnormal_list, + "details": get_win_data(disk_name, rw, io_data) + } + + # io press + ctrl_stage = ['throtl', 'wbt', 'iocost', 'bfq'] + for stage_name in ctrl_stage: + abnormal, abnormal_list = is_abnormal((disk_name, stage_name, rw), io_data) + if not abnormal: + continue + msg["reason"] = "IO press" + msg["block_stack"] = f"bio,{stage_name}" + msg["alarm_type"] = abnormal_list + log_slow_win(msg, "IO press") + xalarm_report(1002, MINOR_ALM, ALARM_TYPE_OCCUR, json.dumps(msg)) + return + + # driver slow + abnormal, abnormal_list = is_abnormal((disk_name, 'rq_driver', rw), io_data) + if abnormal: + msg["reason"] = "driver slow" + msg["block_stack"] = "bio,rq_driver" + msg["alarm_type"] = abnormal_list + log_slow_win(msg, "driver slow") + xalarm_report(1002, MINOR_ALM, ALARM_TYPE_OCCUR, json.dumps(msg)) + return + + # kernel slow + kernel_stage = ['gettag', 'plug', 'deadline', 'hctx', 'requeue'] + for stage_name in kernel_stage: + abnormal, abnormal_list = is_abnormal((disk_name, stage_name, rw), io_data) + if not abnormal: + continue + msg["reason"] = "kernel slow" + msg["block_stack"] = f"bio,{stage_name}" + msg["alarm_type"] = abnormal_list + log_slow_win(msg, "kernel slow") + xalarm_report(1002, MINOR_ALM, ALARM_TYPE_OCCUR, json.dumps(msg)) + return + + log_slow_win(msg, "unknown") + xalarm_report(1002, MINOR_ALM, ALARM_TYPE_OCCUR, json.dumps(msg)) + + +def check_disk_list_validation(disk_list): + valid_disk_list = [] + for disk_name in disk_list: + is_success, _ = check_result_validation(get_disk_type(disk_name), "") + if not is_success: + continue + valid_disk_list.append(disk_name) + return valid_disk_list + + +def get_disk_type_by_name(disk_name): + logging.debug(f"send to sentryCollector get_disk_type: disk_name={disk_name}") + is_success, disk_type_str = check_result_validation(get_disk_type(disk_name), f'Invalid disk type {disk_name}') + if not is_success: + report_alarm_fail(f"{disk_type_str['msg']}") + try: + curr_disk_type = int(disk_type_str) + if curr_disk_type not in Disk_Type: + raise ValueError + except ValueError: + report_alarm_fail(f"Failed to get disk type for {disk_name}") + + return Disk_Type[curr_disk_type] diff --git a/src/python/sentryPlugins/avg_block_io/stage_window.py b/src/python/sentryPlugins/avg_block_io/stage_window.py new file mode 100644 index 0000000000000000000000000000000000000000..587bd497ae71b189cb60c5f2915562edff5f7848 --- /dev/null +++ b/src/python/sentryPlugins/avg_block_io/stage_window.py @@ -0,0 +1,55 @@ +# coding: utf-8 +# Copyright (c) 2024 Huawei Technologies Co., Ltd. +# sysSentry is licensed under the Mulan PSL v2. +# You can use this software according to the terms and conditions of the Mulan PSL v2. +# You may obtain a copy of Mulan PSL v2 at: +# http://license.coscl.org.cn/MulanPSL2 +# THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR +# PURPOSE. +# See the Mulan PSL v2 for more details. + +class AbnormalWindowBase: + def __init__(self, window_size=10, window_threshold=7): + self.window_size = window_size + self.window_threshold = window_threshold + self.abnormal_window = [False] * window_size + self.window_data = [-1] * window_size + + def append_new_data(self, ab_res): + self.window_data.pop(0) + self.window_data.append(ab_res) + + def append_new_period(self, ab_res, avg_val=0): + self.abnormal_window.pop(0) + if self.is_abnormal_period(ab_res, avg_val): + self.abnormal_window.append(True) + else: + self.abnormal_window.append(False) + + def is_abnormal_window(self): + return sum(self.abnormal_window) >= self.window_threshold + + def window_data_to_string(self): + return ",".join(str(x) for x in self.window_data) + + +class IoWindow(AbnormalWindowBase): + def __init__(self, window_size=10, window_threshold=7, abnormal_multiple=5, abnormal_multiple_lim=30, abnormal_time=40): + super().__init__(window_size, window_threshold) + self.abnormal_multiple = abnormal_multiple + self.abnormal_multiple_lim = abnormal_multiple_lim + self.abnormal_time = abnormal_time + + def is_abnormal_period(self, value, avg_val): + return (value > avg_val * self.abnormal_multiple and value > self.abnormal_multiple_lim) or \ + (value > self.abnormal_time) + + +class IoDumpWindow(AbnormalWindowBase): + def __init__(self, window_size=10, window_threshold=7, abnormal_time=40): + super().__init__(window_size, window_threshold) + self.abnormal_time = abnormal_time + + def is_abnormal_period(self, value, avg_val=0): + return value > self.abnormal_time diff --git a/src/python/sentryPlugins/avg_block_io/utils.py b/src/python/sentryPlugins/avg_block_io/utils.py new file mode 100644 index 0000000000000000000000000000000000000000..1bfd4e82aa2283326bdfd5a20ca25cf0f56fcfb6 --- /dev/null +++ b/src/python/sentryPlugins/avg_block_io/utils.py @@ -0,0 +1,140 @@ +# coding: utf-8 +# Copyright (c) 2024 Huawei Technologies Co., Ltd. +# sysSentry is licensed under the Mulan PSL v2. +# You can use this software according to the terms and conditions of the Mulan PSL v2. +# You may obtain a copy of Mulan PSL v2 at: +# http://license.coscl.org.cn/MulanPSL2 +# THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR +# PURPOSE. +# See the Mulan PSL v2 for more details. +import logging +import os + +AVG_VALUE = 0 +AVG_COUNT = 1 + + +def get_nested_value(data, keys): + """get data from nested dict""" + for key in keys: + if key in data: + data = data[key] + else: + return None + return data + + +def set_nested_value(data, keys, value): + """set data to nested dict""" + for key in keys[:-1]: + if key in data: + data = data[key] + else: + return False + data[keys[-1]] = value + return True + + +def get_win_data(disk_name, rw, io_data): + """get latency and iodump win data""" + latency = '' + iodump = '' + for stage_name in io_data[disk_name]: + if 'latency' in io_data[disk_name][stage_name][rw]: + latency_list = io_data[disk_name][stage_name][rw]['latency'].window_data_to_string() + latency += f'{stage_name}: [{latency_list}], ' + if 'iodump' in io_data[disk_name][stage_name][rw]: + iodump_list = io_data[disk_name][stage_name][rw]['iodump'].window_data_to_string() + iodump += f'{stage_name}: [{iodump_list}], ' + return {"latency": latency[:-2], "iodump": iodump[:-2]} + + +def is_abnormal(io_key, io_data): + """check if latency and iodump win abnormal""" + abnormal_list = '' + for key in ['latency', 'iodump']: + all_keys = get_nested_value(io_data, io_key) + if all_keys and key in all_keys: + win = get_nested_value(io_data, io_key + (key,)) + if win and win.is_abnormal_window(): + abnormal_list += key + ', ' + if not abnormal_list: + return False, abnormal_list + return True, abnormal_list[:-2] + + +def update_io_avg(old_avg, period_value, win_size): + """update average of latency window""" + if old_avg[AVG_COUNT] < win_size: + new_avg_count = old_avg[AVG_COUNT] + 1 + new_avg_value = (old_avg[AVG_VALUE] * old_avg[AVG_COUNT] + period_value[0]) / new_avg_count + else: + new_avg_count = old_avg[AVG_COUNT] + new_avg_value = (old_avg[AVG_VALUE] * (old_avg[AVG_COUNT] - 1) + period_value[0]) / new_avg_count + return [new_avg_value, new_avg_count] + + +def update_io_period(old_avg, period_value, io_data, io_key): + """update period of latency and iodump window""" + all_wins = get_nested_value(io_data, io_key) + if all_wins and "latency" in all_wins: + io_data[io_key[0]][io_key[1]][io_key[2]]["latency"].append_new_period(period_value[0], old_avg[AVG_VALUE]) + if all_wins and "iodump" in all_wins: + io_data[io_key[0]][io_key[1]][io_key[2]]["iodump"].append_new_period(period_value[1]) + + +def update_io_data(period_value, io_data, io_key): + """update data of latency and iodump window""" + all_wins = get_nested_value(io_data, io_key) + if all_wins and "latency" in all_wins: + io_data[io_key[0]][io_key[1]][io_key[2]]["latency"].append_new_data(period_value[0]) + if all_wins and "iodump" in all_wins: + io_data[io_key[0]][io_key[1]][io_key[2]]["iodump"].append_new_data(period_value[1]) + + +def log_abnormal_period(old_avg, period_value, io_data, io_key): + """record log of abnormal period""" + all_wins = get_nested_value(io_data, io_key) + if all_wins and "latency" in all_wins: + if all_wins["latency"].is_abnormal_period(period_value[0], old_avg[AVG_VALUE]): + logging.info(f"[abnormal_period] disk: {io_key[0]}, stage: {io_key[1]}, iotype: {io_key[2]}, " + f"type: latency, avg: {round(old_avg[AVG_VALUE], 3)}, curr_val: {period_value[0]}") + if all_wins and "iodump" in all_wins: + if all_wins["iodump"].is_abnormal_period(period_value[1]): + logging.info(f"[abnormal_period] disk: {io_key[0]}, stage: {io_key[1]}, iotype: {io_key[2]}, " + f"type: iodump, curr_val: {period_value[1]}") + + +def log_slow_win(msg, reason): + """record log of slow win""" + logging.warning(f"[SLOW IO] disk: {msg['driver_name']}, stage: {msg['block_stack']}, " + f"iotype: {msg['io_type']}, type: {msg['alarm_type']}, reason: {reason}") + logging.info(f"latency: {msg['details']['latency']}") + logging.info(f"iodump: {msg['details']['iodump']}") + + +def update_avg_and_check_abnormal(data, io_key, win_size, io_avg_value, io_data): + """update avg and check abonrmal, return true if win_size full""" + period_value = get_nested_value(data, io_key) + old_avg = get_nested_value(io_avg_value, io_key) + + # 更新avg数据 + update_io_data(period_value, io_data, io_key) + if old_avg[AVG_COUNT] < win_size: + set_nested_value(io_avg_value, io_key, update_io_avg(old_avg, period_value, win_size)) + return False + + # 打印异常周期数据 + log_abnormal_period(old_avg, period_value, io_data, io_key) + + # 更新win数据 -- 判断异常周期 + update_io_period(old_avg, period_value, io_data, io_key) + all_wins = get_nested_value(io_data, io_key) + if not all_wins or 'latency' not in all_wins: + return True + period = get_nested_value(io_data, io_key + ("latency",)) + if period and period.is_abnormal_period(period_value[0], old_avg[AVG_VALUE]): + return True + set_nested_value(io_avg_value, io_key, update_io_avg(old_avg, period_value, win_size)) + return True diff --git a/src/python/setup.py b/src/python/setup.py index 21dbe9f6276dbf7939a0fc8bdf38e7af36992244..d332e36174fec1e78ca1e6cf07070be85bfc6746 100644 --- a/src/python/setup.py +++ b/src/python/setup.py @@ -17,7 +17,7 @@ from setuptools import setup, find_packages setup( name="syssentry", - version="1.0.1", + version="1.0.2", description="System inspection framework tool set", packages=find_packages(), include_package_data=True, @@ -29,9 +29,11 @@ setup( ], entry_points={ 'console_scripts': [ - 'cpu_sentry=syssentry.cpu_sentry:main', 'syssentry=syssentry.syssentry:main', - 'xalarmd=xalarm.xalarm_daemon:alarm_process_create' + 'xalarmd=xalarm.xalarm_daemon:alarm_process_create', + 'sentryCollector=sentryCollector.collectd:main', + 'avg_block_io=sentryPlugins.avg_block_io.avg_block_io:main', + 'ai_block_io=sentryPlugins.ai_block_io.ai_block_io:main' ] }, ) diff --git a/src/python/syssentry/alarm.py b/src/python/syssentry/alarm.py new file mode 100644 index 0000000000000000000000000000000000000000..e07867bd2ac95ed7dccf84db0cc8997cd7a4c28a --- /dev/null +++ b/src/python/syssentry/alarm.py @@ -0,0 +1,209 @@ +# coding: utf-8 +# Copyright (c) 2024 Huawei Technologies Co., Ltd. +# sysSentry is licensed under the Mulan PSL v2. +# You can use this software according to the terms and conditions of the Mulan PSL v2. +# You may obtain a copy of Mulan PSL v2 at: +# http://license.coscl.org.cn/MulanPSL2 +# THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR +# PURPOSE. +# See the Mulan PSL v2 for more details. + +""" +use for report alarm +""" +import threading +from typing import Dict, List +from datetime import datetime +import time +import logging +import json +import sys + +from xalarm.register_xalarm import xalarm_register,xalarm_getid,xalarm_getlevel,xalarm_gettype,xalarm_gettime,xalarm_getdesc +from xalarm.xalarm_api import Xalarm + +from .global_values import InspectTask +from .task_map import TasksMap + +# 告警ID映射字典，key为插件名，value为告警ID（类型为数字） +task_alarm_id_dict: Dict[str, int] = {} + +# 告警老化时间字典，key为告警ID，value为老化时间（类型为数字，单位为秒） +alarm_id_clear_time_dict: Dict[int, int] = {} + +# 告警事件列表，key为告警ID，value为告警ID对应的告警事件列表（类型为list） +alarm_list_dict: Dict[int, List[Xalarm]] = {} +# 告警事件列表锁 +alarm_list_lock = threading.Lock() + +id_filter = [] +id_base = 1001 +clientId = -1 + +MILLISECONDS_UNIT_SECONDS = 1000 +MAX_NUM_OF_ALARM_ID = 128 +MIN_ALARM_ID = 1001 +MAX_ALARM_ID = (MIN_ALARM_ID + MAX_NUM_OF_ALARM_ID - 1) + +def update_alarm_list(alarm_info: Xalarm): + alarm_id = xalarm_getid(alarm_info) + if alarm_id < MIN_ALARM_ID or alarm_id > MAX_ALARM_ID: + logging.warning(f"Invalid alarm_id {alarm_id}") + return + timestamp = xalarm_gettime(alarm_info) + if not timestamp: + logging.error("Retrieve timestamp failed") + return + alarm_list_lock.acquire() + try: + # new alarm is inserted into list head + if alarm_id not in alarm_list_dict: + logging.warning(f"update_alarm_list: alarm_id {alarm_id} not found in alarm_list_dict") + return + alarm_list = alarm_list_dict[alarm_id] + + alarm_list.insert(0, alarm_info) + # clear alarm_info older than clear time threshold + clear_index = -1 + clear_time = alarm_id_clear_time_dict[alarm_id] + for i in range(len(alarm_list)): + if (timestamp - xalarm_gettime(alarm_list[i])) / MILLISECONDS_UNIT_SECONDS > clear_time: + clear_index = i + break + if clear_index >= 0: + alarm_list_dict[alarm_id] = alarm_list[:clear_index] + finally: + alarm_list_lock.release() + +def validate_alarm_id(alarm_id): + if alarm_id is None: + return False + try: + alarm_id = int(alarm_id) + if MIN_ALARM_ID <= alarm_id <= MAX_ALARM_ID: + return True + else: + return False + except ValueError: + return False + +def validate_alarm_clear_time(alarm_clear_time): + try: + alarm_clear_time = int(alarm_clear_time) + if alarm_clear_time > 0 and alarm_clear_time <= sys.maxsize: + return True + else: + return False + except ValueError: + return False + +def alarm_register(): + logging.debug(f"alarm_register: enter") + # 初始化告警ID映射字典、告警老化时间字典 + for task_type in TasksMap.tasks_dict: + for task_name in TasksMap.tasks_dict[task_type]: + task = TasksMap.tasks_dict[task_type][task_name] + if not validate_alarm_id(task.alarm_id): + logging.warning(f"Invalid alarm_id {task.alarm_id}: ignore {task_name} alarm") + continue + if not validate_alarm_clear_time(task.alarm_clear_time): + logging.warning(f"Invalid alarm_clear_time {task.alarm_clear_time}: ignore {task_name} alarm") + continue + task.alarm_id = int(task.alarm_id) + task.alarm_clear_time = int(task.alarm_clear_time) + alarm_id = task.alarm_id + alarm_clear_time = task.alarm_clear_time + + alarm_list_dict[alarm_id] = [] + task_alarm_id_dict[task_name] = alarm_id + if alarm_id not in alarm_id_clear_time_dict: + alarm_id_clear_time_dict[alarm_id] = alarm_clear_time + else: + alarm_id_clear_time_dict[alarm_id] = max(alarm_clear_time, alarm_id_clear_time_dict[alarm_id]) + logging.info(f"alarm_register: {task_name} is registered") + # 注册告警回调 + id_filter = [True] * 128 + clientId = xalarm_register(update_alarm_list, id_filter) + if clientId < 0: + logging.info(f'register xalarm: failed') + return clientId + logging.info('register xalarm: success') + return clientId + +def get_alarm_result(task_name: str, time_range: int, detailed: bool) -> List[Dict]: + alarm_list_lock.acquire() + try: + if task_name not in task_alarm_id_dict: + logging.debug("task_name does not exist") + return [] + alarm_id = task_alarm_id_dict[task_name] + clear_time = alarm_id_clear_time_dict[alarm_id] + if alarm_id not in alarm_list_dict: + logging.debug("alarm_id does not exist") + return [] + alarm_list = alarm_list_dict[alarm_id] + logging.debug(f"get_alarm_result: alarm_list of {alarm_id} has {len(alarm_list)} elements") + # clear alarm_info older than clear time threshold + stop_index = -1 + timestamp = datetime.now().timestamp() + for i in range(len(alarm_list)): + logging.debug(f"timestamp, alarm_list[{i}].timestamp: {timestamp}, {xalarm_gettime(alarm_list[i])}") + if timestamp - (xalarm_gettime(alarm_list[i])) / MILLISECONDS_UNIT_SECONDS > time_range: + stop_index = i + break + if timestamp - (xalarm_gettime(alarm_list[i])) / MILLISECONDS_UNIT_SECONDS > clear_time: + stop_index = i + break + if stop_index >= 0: + alarm_list = alarm_list[:stop_index] + logging.debug(f"get_alarm_result: final alarm_list of {alarm_id} has {len(alarm_list)} elements") + + def xalarm_to_dict(alarm_info: Xalarm) -> dict: + timestamp = alarm_info.timetamp.tv_sec + alarm_info.timetamp.tv_usec / 1000000 + dt_object = datetime.fromtimestamp(int(timestamp)) + return { + 'alarm_id': xalarm_getid(alarm_info), + 'alarm_type': xalarm_gettype(alarm_info), + 'alarm_level': xalarm_getlevel(alarm_info), + 'timestamp': dt_object.strftime("%Y-%m-%d %H:%M:%S"), + 'msg1': xalarm_getdesc(alarm_info) + } + + alarm_list = [xalarm_to_dict(alarm) for alarm in alarm_list] + + # keep detail + for alarm in alarm_list: + alarm_info = alarm['msg1'] + alarm_info = json.loads(alarm_info) + if not detailed: + if 'details' in alarm_info: + alarm_info.pop('details', None) + alarm.pop('msg1', None) + + # dump each {key,value} of details in one line + if 'details' in alarm_info and isinstance(alarm_info['details'], dict): + for key in alarm_info['details']: + alarm_info['details'][key] = str(alarm_info['details'][key]) + + alarm['alarm_info'] = alarm_info + alarm_list = [alarm for alarm in alarm_list if 'alarm_source' in alarm['alarm_info'] and alarm['alarm_info']['alarm_source'] == task_name] + + alarm_level_mapping = { + 1: 'MINOR_ALM', + 2: 'MAJOR_ALM', + 3: 'CRITICAL_ALM' + } + + alarm_type_mapping = { + 1: 'ALARM_TYPE_OCCUR', + 2: 'ALARM_TYPE_RECOVER' + } + + for alarm in alarm_list: + alarm['alarm_level'] = alarm_level_mapping.get(alarm['alarm_level'], 'UNKNOWN_LEVEL') + alarm['alarm_type'] = alarm_type_mapping.get(alarm['alarm_type'], 'UNKNOWN_TYPE') + return alarm_list + + finally: + alarm_list_lock.release() diff --git a/src/python/syssentry/callbacks.py b/src/python/syssentry/callbacks.py index d0d0719bf7d79c770b0945783a35b9f8a74bbcc9..d2dff24bd6b5beeb1b5e0745f9283d5bcb61a36f 100644 --- a/src/python/syssentry/callbacks.py +++ b/src/python/syssentry/callbacks.py @@ -18,6 +18,7 @@ import logging from .task_map import TasksMap, ONESHOT_TYPE, PERIOD_TYPE from .mod_status import EXITED_STATUS, RUNNING_STATUS, WAITING_STATUS, set_runtime_status +from .alarm import get_alarm_result def task_get_status(mod_name): @@ -41,6 +42,22 @@ def task_get_result(mod_name): return "success", task.get_result() +def task_get_alarm(data): + """get alarm by mod name""" + task_name = data['task_name'] + time_range = data['time_range'] + try: + detailed = data['detailed'] + except KeyError: + logging.debug("Key 'detailed' does not exist in the dictionary") + detailed = None + task = TasksMap.get_task_by_name(task_name) + if not task: + return "failed", f"cannot find task by name {task_name}" + if not task.load_enabled: + return "failed", f"mod {task_name} is not enabled" + + return "success", get_alarm_result(task_name, time_range, detailed) def task_stop(mod_name): """stop by mod name""" diff --git a/src/python/syssentry/cpu_alarm.py b/src/python/syssentry/cpu_alarm.py index d972c42cefe48de564e90f39434548f1d8cd81a1..0b1642ba15ba22c24652027486b9f0bdfd3cf004 100644 --- a/src/python/syssentry/cpu_alarm.py +++ b/src/python/syssentry/cpu_alarm.py @@ -1,6 +1,7 @@ import re import math import logging +import socket from enum import Enum from .utils import execute_command @@ -15,6 +16,12 @@ BINARY = 2 MIN_DATA_LEN = 0 MAX_DATA_LEN = 999 +PARAM_REP_LEN = 3 +PARAM_TYPE_LEN = 1 +PARAM_MODULE_LEN = 1 +PARAM_TRANS_TO_LEN = 2 +PARAM_DATA_LEN = 3 + class Type(Enum): CE = 0x00 @@ -207,3 +214,38 @@ def check_fixed_param(data, expect): raise ValueError("expected str param is not valid") return data raise NotImplementedError("unexpected param type") + + +def cpu_alarm_recv(server_socket: socket.socket): + try: + client_socket, _ = server_socket.accept() + logging.debug("cpu alarm fd listen ok") + + data = client_socket.recv(PARAM_REP_LEN) + check_fixed_param(data, "REP") + + data = client_socket.recv(PARAM_TYPE_LEN) + _type = check_fixed_param(data, Type) + + data = client_socket.recv(PARAM_MODULE_LEN) + module = check_fixed_param(data, Module) + + data = client_socket.recv(PARAM_TRANS_TO_LEN) + trans_to = check_fixed_param(data, TransTo) + + data = client_socket.recv(PARAM_DATA_LEN) + data_len = check_fixed_param(data, (MIN_DATA_LEN, MAX_DATA_LEN)) + + data = client_socket.recv(data_len) + + command, event_type, socket_id, core_id = parser_cpu_alarm_info(data) + except socket.error: + logging.error("socket error") + return + except (ValueError, OSError, UnicodeError, TypeError, NotImplementedError): + logging.error("server recv cpu alarm msg failed!") + client_socket.close() + return + + upload_bmc(_type, module, command, event_type, socket_id, core_id) + diff --git a/src/python/syssentry/cpu_sentry.py b/src/python/syssentry/cpu_sentry.py index 7e77654fa49a6c992c8c03a845e9843f061ac7c9..99af1270e1783a18856a32af598c70f997604cd7 100644 --- a/src/python/syssentry/cpu_sentry.py +++ b/src/python/syssentry/cpu_sentry.py @@ -87,11 +87,26 @@ class CpuSentry: } def handle_cpu_output(self, stdout: str): - if "" in stdout: + if not stdout: + logging.error("%s process output is None, it may be killed!", LOW_LEVEL_INSPECT_CMD) + self.send_result["result"] = ResultLevel.FAIL + self.send_result["details"]["code"] = 1005 + self.send_result["details"]["msg"] = "cpu_sentry task is killed!" + return + + if "ERROR" in stdout: self.send_result["result"] = ResultLevel.FAIL self.send_result["details"]["code"] = 1004 - self.send_result["details"]["msg"] = stdout.split("\n")[0] + + # Remove ANSI escape sequences + error_info = stdout.split("\n")[0] + if error_info.startswith("\u001b"): + ansi_escape = r'\x1b\[([0-9]+)(;[0-9]+)*([A-Za-z])' + error_info = re.sub(ansi_escape, '', error_info) + + self.send_result["details"]["msg"] = error_info return + out_split = stdout.split("\n") isolated_cores_number = 0 found_fault_cores_list = [] @@ -133,6 +148,7 @@ class CpuSentry: result_level = self.send_result.get("result", ResultLevel.FAIL) report_result(task_name, result_level, details) + self.init_send_result() def kill_process(signum, _f, cpu_sentry_obj): """kill process by 'pkill -9'""" @@ -179,6 +195,6 @@ def main(): cpu_sentry_task.send_result["result"] = ResultLevel.FAIL cpu_sentry_task.send_result["details"]["code"] = 1004 cpu_sentry_task.send_result["details"]["msg"] = "run cmd [%s] raise Error" % cpu_sentry_task_cmd - finally: cpu_sentry_task.cpu_report_result() - cpu_sentry_task.init_send_result() + else: + cpu_sentry_task.cpu_report_result() diff --git a/src/python/syssentry/global_values.py b/src/python/syssentry/global_values.py index 483d5447d97b36b1af61c1a2214bef68349f46aa..b123b2dcb7bcf8f94a30b43339f8f21dae5a038f 100644 --- a/src/python/syssentry/global_values.py +++ b/src/python/syssentry/global_values.py @@ -27,6 +27,7 @@ CTL_SOCKET_PATH = "/var/run/sysSentry/control.sock" SYSSENTRY_CONF_PATH = "/etc/sysSentry" INSPECT_CONF_PATH = "/etc/sysSentry/inspect.conf" TASK_LOG_DIR = "/var/log/sysSentry" +DEFAULT_ALARM_CLEAR_TIME = 15 SENTRY_RUN_DIR_PERM = 0o750 @@ -76,6 +77,9 @@ class InspectTask: self.env_file = "" # start mode self.conflict = "up" + # alarm id + self.alarm_id = -1 + self.alarm_clear_time = DEFAULT_ALARM_CLEAR_TIME def start(self): """ diff --git a/src/python/syssentry/load_mods.py b/src/python/syssentry/load_mods.py index 48d7e66e4674a862653ef89144dce328ff2ac820..78db446b33856a7f5a2533796b495a6ef95d763e 100644 --- a/src/python/syssentry/load_mods.py +++ b/src/python/syssentry/load_mods.py @@ -24,6 +24,7 @@ from .task_map import TasksMap, ONESHOT_TYPE, PERIOD_TYPE from .cron_process import PeriodTask from .mod_status import set_task_status +from xalarm.register_xalarm import MIN_ALARM_ID, MAX_ALARM_ID ONESHOT_CONF = 'oneshot' PERIOD_CONF = 'period' @@ -41,6 +42,8 @@ CONF_TASK_RESTART = 'task_restart' CONF_ONSTART = 'onstart' CONF_ENV_FILE = 'env_file' CONF_CONFLICT = 'conflict' +CONF_ALARM_ID = 'alarm_id' +CONF_ALARM_CLEAR_TIME = 'alarm_clear_time' MOD_FILE_SUFFIX = '.mod' MOD_SUFFIX_LEN = 4 @@ -194,6 +197,18 @@ def parse_mod_conf(mod_name, mod_conf): task.heartbeat_interval = heartbeat_interval task.load_enabled = is_enabled + try: + task.alarm_id = mod_conf.get(CONF_TASK, CONF_ALARM_ID) + except configparser.NoOptionError: + task.alarm_id = None + logging.warning(f"{mod_name} alarm_id not set, alarm_id is None") + + if task.alarm_id is not None: + try: + task.alarm_clear_time = mod_conf.get(CONF_TASK, CONF_ALARM_CLEAR_TIME) + except configparser.NoOptionError: + logging.warning(f"{mod_name} not set alarm_clear_time, use 15s as default") + if CONF_ONSTART in mod_conf.options(CONF_TASK): is_onstart = (mod_conf.get(CONF_TASK, CONF_ONSTART) == 'yes') if task_type == PERIOD_CONF: @@ -327,3 +342,4 @@ def reload_single_mod(mod_name): res, ret = reload_mod_by_name(mod_name) return res, ret + diff --git a/src/python/syssentry/sentry_config.py b/src/python/syssentry/sentry_config.py index 01f3df83530de2cee2495daef23c95aaa54dd5d0..1169887c2e413169839a0a0610b42cc83dcb44d7 100644 --- a/src/python/syssentry/sentry_config.py +++ b/src/python/syssentry/sentry_config.py @@ -21,6 +21,34 @@ import sys DEFAULT_INSPECT_DELAY = 3 INSPECT_CONF_PATH = "/etc/sysSentry/inspect.conf" +CONF_LOG = 'log' +CONF_LOG_LEVEL = 'level' +LogLevel = { + "debug": logging.DEBUG, + "info": logging.INFO, + "warning": logging.WARNING, + "error": logging.ERROR, + "critical": logging.CRITICAL +} + + +def get_log_level(filename=INSPECT_CONF_PATH): + if not os.path.exists(filename): + return logging.INFO + + try: + config = configparser.ConfigParser() + config.read(filename) + if not config.has_option(CONF_LOG, CONF_LOG_LEVEL): + return logging.INFO + log_level = config.get(CONF_LOG, CONF_LOG_LEVEL) + + if log_level.lower() in LogLevel: + return LogLevel.get(log_level.lower()) + return logging.INFO + except configparser.Error: + return logging.INFO + class SentryConfig: """ @@ -103,14 +131,18 @@ class CpuPluginsParamsConfig: """read config file""" config_param_section_args = {} if os.path.exists(self.config_file): - self.config.read(self.config_file) try: + self.config.read(self.config_file) config_param_section_args = dict(self.config[self.param_section_name]) - except (ValueError, KeyError): + except (ValueError, KeyError, configparser.InterpolationSyntaxError): config_param_section_args = {} + logging.error("Failed to parse cpu_sentry.ini!") return config_param_section_args def join_cpu_start_cmd(self, cpu_param_dict: dict) -> str: + if not cpu_param_dict: + return "" + cpu_list = cpu_param_dict.get("cpu_list", "default") if cpu_list == "default": cpu_list = CpuPluginsParamsConfig.get_cpu_info() diff --git a/src/python/syssentry/sentryctl b/src/python/syssentry/sentryctl index e94491f110f12b692ed3d6461935e1f730063046..c2e3cefae7184141d4b64510ab49e13f66e448b2 100644 --- a/src/python/syssentry/sentryctl +++ b/src/python/syssentry/sentryctl @@ -25,6 +25,8 @@ MAX_PARAM_LENGTH = 256 RESULT_MSG_DATA_LEN = 4 CTL_MSG_LEN_LEN = 3 +ALARM_MSG_DATA_LEN = 6 +DEFAULT_ALARM_TIME_RANGE = 10 def status_output_format(res_data): """format output""" @@ -57,6 +59,8 @@ def res_output_handle(res_struct, req_type): status_output_format(res_struct['data']) elif req_type == 'get_result': result_output_format(res_struct['data']) + elif req_type == 'get_alarm': + result_output_format(res_struct['data']) elif res_struct['ret'] == "failed": print(res_struct['data']) @@ -75,6 +79,7 @@ def client_send_and_recv(request_data, data_str_len): print("sentryctl: client creat socket error") return None + # connect to syssentry try: client_socket.connect(CTL_SOCKET_PATH) except OSError: @@ -82,6 +87,7 @@ def client_send_and_recv(request_data, data_str_len): print("sentryctl: client connect error") return None + # msg: CTL{len}{data} req_data_len = len(request_data) request_msg = "CTL" + str(req_data_len).zfill(3) + request_data @@ -94,8 +100,8 @@ def client_send_and_recv(request_data, data_str_len): print("sentryctl: client communicate error") return None + # res: RES{len}{data} res_magic = res_data[:3] - if res_magic != "RES": print("res msg format error") return None @@ -128,6 +134,10 @@ if __name__ == '__main__': parser_status.add_argument('task_name') parser_get_result = subparsers.add_parser('get_result', help='get task result') parser_get_result.add_argument('task_name') + parser_get_alarm = subparsers.add_parser('get_alarm', help='get task alarm') + parser_get_alarm.add_argument('task_name') + parser_get_alarm.add_argument('-s', '--time_range', type=int, default=DEFAULT_ALARM_TIME_RANGE, help='Specified time range') + parser_get_alarm.add_argument('-d', '--detailed', action='store_true', help='Print Detailed Information') parser_list = subparsers.add_parser('list', help='show all loaded task mod') client_args = parser.parse_args() @@ -142,6 +152,17 @@ if __name__ == '__main__': req_msg_struct = {"type": "get_status", "data": client_args.task_name} elif client_args.cmd_type == 'get_result': req_msg_struct = {"type": "get_result", "data": client_args.task_name} + elif client_args.cmd_type == 'get_alarm': + if not isinstance(client_args.time_range, int) or client_args.time_range <= 0: + print(f"time_range is not a positive integer: {client_args.time_range}") + req_msg_struct = { + "type": "get_alarm", + "data": { + 'task_name': client_args.task_name, + 'time_range': client_args.time_range, + 'detailed': client_args.detailed, + } + } elif client_args.cmd_type == 'reload': req_msg_struct = {"type": "reload", "data": client_args.task_name} else: @@ -155,6 +176,8 @@ if __name__ == '__main__': request_message = json.dumps(req_msg_struct) if client_args.cmd_type == 'get_result': result_message = client_send_and_recv(request_message, RESULT_MSG_DATA_LEN) + elif client_args.cmd_type == 'get_alarm': + result_message = client_send_and_recv(request_message, ALARM_MSG_DATA_LEN) else: result_message = client_send_and_recv(request_message, CTL_MSG_LEN_LEN) if not result_message: diff --git a/src/python/syssentry/syssentry.py b/src/python/syssentry/syssentry.py index 32b81e30b5da3255c91be365b3c9dc98c2f5b9a7..ea09095eec29faf16a2eecf62e2fe9252002a951 100644 --- a/src/python/syssentry/syssentry.py +++ b/src/python/syssentry/syssentry.py @@ -23,12 +23,12 @@ import fcntl import select -from .sentry_config import SentryConfig +from .sentry_config import SentryConfig, get_log_level from .task_map import TasksMap from .global_values import SENTRY_RUN_DIR, CTL_SOCKET_PATH, SENTRY_RUN_DIR_PERM from .cron_process import period_tasks_handle -from .callbacks import mod_list_show, task_start, task_get_status, task_stop, task_get_result +from .callbacks import mod_list_show, task_start, task_get_status, task_stop, task_get_result, task_get_alarm from .mod_status import get_task_by_pid, set_runtime_status from .load_mods import load_tasks, reload_single_mod from .heartbeat import (heartbeat_timeout_chk, heartbeat_fd_create, @@ -36,8 +36,18 @@ from .heartbeat import (heartbeat_timeout_chk, heartbeat_fd_create, from .result import RESULT_MSG_HEAD_LEN, RESULT_MSG_MAGIC_LEN, RESULT_MAGIC from .result import RESULT_LEVEL_ERR_MSG_DICT, ResultLevel from .utils import get_current_time_string -from .cpu_alarm import (upload_bmc, check_fixed_param, parser_cpu_alarm_info, - Type, Module, TransTo, MIN_DATA_LEN, MAX_DATA_LEN) +from .alarm import alarm_register + +from xalarm.register_xalarm import xalarm_unregister + +clientId = -1 + +CPU_EXIST = True +try: + from .cpu_alarm import cpu_alarm_recv +except ImportError: + CPU_EXIST = False + INSPECTOR = None @@ -46,6 +56,7 @@ CTL_MSG_MAGIC_LEN = 3 CTL_MSG_LEN_LEN = 3 CTL_MAGIC = "CTL" RES_MAGIC = "RES" +ALARM_MSG_DATA_LEN = 6 CTL_LISTEN_QUEUE_LEN = 5 SERVER_EPOLL_TIMEOUT = 0.3 @@ -56,6 +67,7 @@ type_func = { 'stop': task_stop, 'get_status': task_get_status, 'get_result': task_get_result, + 'get_alarm': task_get_alarm, 'reload': reload_single_mod } @@ -76,45 +88,6 @@ PID_FILE_FLOCK = None RESULT_SOCKET_PATH = "/var/run/sysSentry/result.sock" CPU_ALARM_SOCKET_PATH = "/var/run/sysSentry/report.sock" -PARAM_REP_LEN = 3 -PARAM_TYPE_LEN = 1 -PARAM_MODULE_LEN = 1 -PARAM_TRANS_TO_LEN = 2 -PARAM_DATA_LEN = 3 - - -def cpu_alarm_recv(server_socket: socket.socket): - try: - client_socket, _ = server_socket.accept() - logging.debug("cpu alarm fd listen ok") - - data = client_socket.recv(PARAM_REP_LEN) - check_fixed_param(data, "REP") - - data = client_socket.recv(PARAM_TYPE_LEN) - _type = check_fixed_param(data, Type) - - data = client_socket.recv(PARAM_MODULE_LEN) - module = check_fixed_param(data, Module) - - data = client_socket.recv(PARAM_TRANS_TO_LEN) - trans_to = check_fixed_param(data, TransTo) - - data = client_socket.recv(PARAM_DATA_LEN) - data_len = check_fixed_param(data, (MIN_DATA_LEN, MAX_DATA_LEN)) - - data = client_socket.recv(data_len) - - command, event_type, socket_id, core_id = parser_cpu_alarm_info(data) - except socket.error: - logging.error("socket error") - return - except (ValueError, OSError, UnicodeError, TypeError, NotImplementedError): - logging.error("server recv cpu alarm msg failed!") - client_socket.close() - return - - upload_bmc(_type, module, command, event_type, socket_id, core_id) def msg_data_process(msg_data): @@ -140,11 +113,12 @@ def msg_data_process(msg_data): return "Invaild cmd type" cmd_param = data_struct['data'] - logging.debug("msg_data_process cmd_type:%s cmd_param:%s", cmd_type, cmd_param) + logging.debug("msg_data_process cmd_type:%s cmd_param:%s", cmd_type, str(cmd_param)) if cmd_type in type_func: ret, res_data = type_func[cmd_type](cmd_param) else: ret, res_data = type_func_void[cmd_type]() + logging.debug("msg_data_process res_data:%s",str(res_data)) res_msg_struct = {"ret": ret, "data": res_data} res_msg = json.dumps(res_msg_struct) @@ -283,6 +257,8 @@ def server_recv(server_socket: socket.socket): res_head = RES_MAGIC if cmd_type == "get_result": res_data_len = str(len(res_data)).zfill(RESULT_MSG_HEAD_LEN - RESULT_MSG_MAGIC_LEN) + elif cmd_type == "get_alarm": + res_data_len = str(len(res_data)).zfill(ALARM_MSG_DATA_LEN) else: res_data_len = str(len(res_data)).zfill(CTL_MSG_MAGIC_LEN) res_head += res_data_len @@ -462,6 +438,14 @@ def main_loop(): epoll_fd.register(cpu_alarm_fd.fileno(), select.EPOLLIN) logging.debug("start main loop") + # onstart_tasks_handle() + for task_type in TasksMap.tasks_dict: + for task_name in TasksMap.tasks_dict.get(task_type): + task = TasksMap.tasks_dict.get(task_type).get(task_name) + if not task: + continue + task.onstart_handle() + while True: try: events_list = epoll_fd.poll(SERVER_EPOLL_TIMEOUT) @@ -472,7 +456,7 @@ def main_loop(): server_result_recv(server_result_fd) elif event_fd == heartbeat_fd.fileno(): heartbeat_recv(heartbeat_fd) - elif event_fd == cpu_alarm_fd.fileno(): + elif CPU_EXIST and event_fd == cpu_alarm_fd.fileno(): cpu_alarm_recv(cpu_alarm_fd) else: continue @@ -587,27 +571,35 @@ def main(): if not os.path.exists(SENTRY_RUN_DIR): os.mkdir(SENTRY_RUN_DIR) os.chmod(SENTRY_RUN_DIR, mode=SENTRY_RUN_DIR_PERM) + + log_level = get_log_level() + log_format = "%(asctime)s - %(levelname)s - [%(filename)s:%(lineno)d] - %(message)s" + + logging.basicConfig(filename=SYSSENTRY_LOG_FILE, level=log_level, format=log_format) + os.chmod(SYSSENTRY_LOG_FILE, 0o600) + if not chk_and_set_pidfile(): logging.error("get pid file lock failed, exist") sys.exit(17) - logging.basicConfig(filename=SYSSENTRY_LOG_FILE, level=logging.INFO) - os.chmod(SYSSENTRY_LOG_FILE, 0o600) - try: signal.signal(signal.SIGINT, sig_handler) signal.signal(signal.SIGTERM, sig_handler) signal.signal(signal.SIGHUP, sig_handler) signal.signal(signal.SIGCHLD, sigchld_handler) - logging.debug("finish main parse_args") + logging.info("finish main parse_args") _ = SentryConfig.init_param() TasksMap.init_task_map() load_tasks() + clientId = alarm_register() main_loop() except Exception: logging.error('%s', traceback.format_exc()) finally: + if clientId != -1: + xalarm_unregister(clientId) release_pidfile() + diff --git a/src/python/syssentry/task_map.py b/src/python/syssentry/task_map.py index 70aa19d878b199468e25fc248e3752f149cfa7ac..27e97ff8ddbc74d31769eebdf10e50cbefaad5e1 100644 --- a/src/python/syssentry/task_map.py +++ b/src/python/syssentry/task_map.py @@ -13,16 +13,16 @@ tasks map class and initialize function. """ import logging +from typing import Dict ONESHOT_TYPE = "ONESHOT" PERIOD_TYPE = "PERIOD" TASKS_MAP = None - class TasksMap: """task map class""" - tasks_dict = {} + tasks_dict: Dict[str, Dict] = {} @classmethod def init_task_map(cls): @@ -65,3 +65,4 @@ class TasksMap: logging.debug("getting task by name: %s", res) break return res + diff --git a/src/python/xalarm/register_xalarm.py b/src/python/xalarm/register_xalarm.py new file mode 100644 index 0000000000000000000000000000000000000000..2a6dabf2a6beae73c17d879e1fb3b0ea0f619a5a --- /dev/null +++ b/src/python/xalarm/register_xalarm.py @@ -0,0 +1,196 @@ +import os +import sys +import socket +import logging +import threading +import time +import fcntl +import inspect +from struct import error as StructParseError + +from .xalarm_api import Xalarm, alarm_bin2stu + + +ALARM_REPORT_LEN = 8216 +MAX_NUM_OF_ALARM_ID=128 +MIN_ALARM_ID = 1001 +MAX_ALARM_ID = (MIN_ALARM_ID + MAX_NUM_OF_ALARM_ID - 1) +DIR_XALARM = "/var/run/xalarm" +PATH_REG_ALARM = "/var/run/xalarm/alarm" +PATH_REPORT_ALARM = "/var/run/xalarm/report" +ALARM_DIR_PERMISSION = 0o0750 +ALARM_REG_SOCK_PERMISSION = 0o0700 +ALARM_SOCKET_PERMISSION = 0o0700 +TIME_UNIT_MILLISECONDS = 1000 +ALARM_REGISTER_INFO = None + + +class AlarmRegister: + def __init__(self, id_filter: list, callback: callable): + self.id_filter = id_filter + self.callback = callback + self.socket = self.create_unix_socket() + self.is_registered = True + self.thread = threading.Thread(target=self.alarm_recv) + self.thread_should_stop = False + + def check_params(self) -> bool: + if (len(self.id_filter) != MAX_NUM_OF_ALARM_ID): + sys.stderr.write("check_params: invalid param id_filter\n") + return False + + sig = inspect.signature(self.callback) + if len(sig.parameters) != 1: + sys.stderr.write("check_params: invalid param callback\n") + return False + + if self.socket is None: + sys.stderr.write("check_params: socket create failed\n") + return False + return True + + def set_id_filter(self, id_filter: list) -> bool: + if (len(id_filter) > MAX_NUM_OF_ALARM_ID): + sys.stderr.write("set_id_filter: invalid param id_filter\n") + return False + self.id_filter = id_filter + + def id_is_registered(self, alarm_id) -> bool: + if alarm_id < MIN_ALARM_ID or alarm_id > MAX_ALARM_ID: + return False + return self.id_filter[alarm_id - MIN_ALARM_ID] + + def put_alarm_info(self, alarm_info: Xalarm) -> None: + if not self.callback or not alarm_info: + return + if not self.id_is_registered(alarm_info.alarm_id): + return + self.callback(alarm_info) + + def create_unix_socket(self) -> socket.socket: + try: + sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) + sock.setblocking(False) + + if not os.access(DIR_XALARM, os.F_OK): + os.makedirs(DIR_XALARM) + os.chmod(DIR_XALARM, ALARM_DIR_PERMISSION) + + sock.connect(PATH_REG_ALARM) + return sock + except (IOError, OSError, FileNotFoundError) as e: + sock.close() + sys.stderr.write(f"create_unix_socket: create socket error:{e}\n") + return None + + def alarm_recv(self): + while not self.thread_should_stop: + try: + data = self.socket.recv(ALARM_REPORT_LEN) + if not data: + sys.stderr.write("connection closed by xalarmd, maybe connections reach max num or service stopped.\n") + self.thread_should_stop = True + break + if len(data) != ALARM_REPORT_LEN: + sys.stderr.write(f"server receive report msg length wrong {len(data)}\n") + continue + + alarm_info = alarm_bin2stu(data) + self.put_alarm_info(alarm_info) + except (BlockingIOError) as e: + time.sleep(0.1) + except (ConnectionResetError, ConnectionAbortedError, BrokenPipeError): + sys.stderr.write("Connection closed by the server.\n") + self.thread_should_stop = True + except (ValueError, StructParseError, InterruptedError) as e: + sys.stderr.write(f"{e}\n") + except Exception as e: + sys.stderr.write(f"{e}\n") + self.thread_should_stop = True + + def start_thread(self) -> None: + self.thread.daemon = True + self.thread.start() + + def stop_thread(self) -> None: + self.thread_should_stop = True + self.thread.join() + self.socket.close() + + +def xalarm_register(callback: callable, id_filter: list) -> int: + global ALARM_REGISTER_INFO + + if ALARM_REGISTER_INFO is not None: + sys.stderr.write("xalarm_register: alarm has registered\n") + return -1 + + ALARM_REGISTER_INFO = AlarmRegister(id_filter, callback) + if not ALARM_REGISTER_INFO.check_params(): + return -1 + + ALARM_REGISTER_INFO.start_thread() + + return 0 + + +def xalarm_unregister(clientId: int) -> None: + global ALARM_REGISTER_INFO + if clientId < 0: + sys.stderr.write("xalarm_unregister: invalid client\n") + return + + if ALARM_REGISTER_INFO is None: + sys.stderr.write("xalarm_unregister: alarm has not registered\n") + return + + ALARM_REGISTER_INFO.stop_thread() + ALARM_REGISTER_INFO = None + + +def xalarm_upgrade(id_filter: list, clientId: int) -> bool: + global ALARM_REGISTER_INFO + if clientId < 0: + sys.stderr.write("xalarm_upgrade: invalid client\n") + return False + if ALARM_REGISTER_INFO is None: + sys.stderr.write("xalarm_upgrade: alarm has not registered\n") + return False + if ALARM_REGISTER_INFO.thread_should_stop: + sys.stderr.write("xalarm_upgrade: upgrade failed, alarm thread has stopped\n") + return False + ALARM_REGISTER_INFO.id_filter = id_filter + return True + + +def xalarm_getid(alarm_info: Xalarm) -> int: + if not alarm_info: + return 0 + return alarm_info.alarm_id + + +def xalarm_getlevel(alarm_info: Xalarm) -> int: + if not alarm_info: + return 0 + return alarm_info.alarm_level + + +def xalarm_gettype(alarm_info: Xalarm) -> int: + if not alarm_info: + return 0 + return alarm_info.alarm_type + + +def xalarm_gettime(alarm_info: Xalarm) -> int: + if not alarm_info: + return 0 + return alarm_info.timetamp.tv_sec * TIME_UNIT_MILLISECONDS + alarm_info.timetamp.tv_usec / TIME_UNIT_MILLISECONDS + +def xalarm_getdesc(alarm_info: Xalarm) -> str: + if not alarm_info: + return None + try: + desc_str = alarm_info.msg1.rstrip(b'\x00').decode('utf-8') + except UnicodeError: + desc_str = None + return desc_str diff --git a/src/python/xalarm/sentry_notify.py b/src/python/xalarm/sentry_notify.py new file mode 100644 index 0000000000000000000000000000000000000000..1e3fa76a77d163d8d0084c655725826c36fbe940 --- /dev/null +++ b/src/python/xalarm/sentry_notify.py @@ -0,0 +1,72 @@ +import os +import sys +import time +import socket +import logging +from struct import error as StructParseError + +from .xalarm_api import alarm_stu2bin, Xalarm + +MAX_NUM_OF_ALARM_ID = 128 +MIN_ALARM_ID = 1001 +MAX_ALARM_ID = (MIN_ALARM_ID + MAX_NUM_OF_ALARM_ID - 1) + +MINOR_ALM = 1 +MAJOR_ALM = 2 +CRITICAL_ALM = 3 + +ALARM_TYPE_OCCUR = 1 +ALARM_TYPE_RECOVER = 2 + +MAX_PUC_PARAS_LEN = 8192 + +DIR_XALARM = "/var/run/xalarm" +PATH_REPORT_ALARM = "/var/run/xalarm/report" +ALARM_DIR_PERMISSION = 0o750 +ALARM_SOCKET_PERMISSION = 0o700 + + +def check_params(alarm_id, alarm_level, alarm_type, puc_paras) -> bool: + if not os.path.exists(DIR_XALARM): + logging.error(f"check_params: {DIR_XALARM} not exist, failed") + return False + + if not os.path.exists(PATH_REPORT_ALARM): + logging.error(f"check_params: {PATH_REPORT_ALARM} not exist, failed") + return False + + if (alarm_id < MIN_ALARM_ID or alarm_id > MAX_ALARM_ID or + alarm_level < MINOR_ALM or alarm_level > CRITICAL_ALM or + alarm_type < ALARM_TYPE_OCCUR or alarm_type > ALARM_TYPE_RECOVER): + logging.error("check_params: alarm info invalid") + return False + + if len(puc_paras) >= MAX_PUC_PARAS_LEN: + logging.error(f"check_params: alarm msg should be less than {MAX_PUC_PARAS_LEN}") + return False + + return True + +def xalarm_report(alarm_id, alarm_level, alarm_type, puc_paras) -> bool: + if not check_params(alarm_id, alarm_level, alarm_type, puc_paras): + return False + + try: + sock = socket.socket(socket.AF_UNIX, socket.SOCK_DGRAM) + + current_time = time.time() + current_time_seconds = int(current_time) + current_microseconds = int((current_time - current_time_seconds) * 1_000_000) + alarm_info = Xalarm(alarm_id, alarm_type, alarm_level, + current_time_seconds, current_microseconds, puc_paras) + + sock.sendto(alarm_stu2bin(alarm_info), PATH_REPORT_ALARM) + except (FileNotFoundError, StructParseError, socket.error, OSError, UnicodeError) as e: + logging.error(f"error occurs when sending msg.") + return False + finally: + sock.close() + + return True + + diff --git a/src/python/xalarm/xalarm_api.py b/src/python/xalarm/xalarm_api.py index 94d76382df7cf44c7de7212df687078d118aa180..863bd026804d4b5377115f231b775993301fc806 100644 --- a/src/python/xalarm/xalarm_api.py +++ b/src/python/xalarm/xalarm_api.py @@ -23,6 +23,7 @@ ALARM_LEVELS = (1, 2, 3, 4, 5) ALARM_SOCK_PATH = "/var/run/xalarm/report" MIN_ALARM_ID = 1001 MAX_ALARM_ID = 1128 +MAX_MSG_LEN = 8192 @dataclasses.dataclass @@ -97,15 +98,15 @@ class Xalarm: def msg1(self, msg): """msg1 setter """ - if len(msg) > 512: - raise ValueError("msg1 length must below 255") + if len(msg) > MAX_MSG_LEN: + raise ValueError(f"msg1 length must below {MAX_MSG_LEN}") self._msg1 = msg def alarm_bin2stu(bin_data): """alarm binary to struct """ - struct_data = struct.unpack("@HBBll512s", bin_data) + struct_data = struct.unpack(f"@HBBll{MAX_MSG_LEN}s", bin_data) alarm_info = Xalarm(1001, 2, 1, 0, 0, "") alarm_info.alarm_id = struct_data[0] @@ -116,3 +117,18 @@ def alarm_bin2stu(bin_data): alarm_info.msg1 = struct_data[5] return alarm_info + + +def alarm_stu2bin(alarm_info: Xalarm): + alarm_msg = alarm_info.msg1 + padding_length = MAX_MSG_LEN - len(alarm_msg) + if padding_length > 0: + alarm_msg = alarm_msg + ('\x00' * padding_length) + return struct.pack( + f'@HBBll{MAX_MSG_LEN}s', + alarm_info.alarm_id, + alarm_info.alarm_level, + alarm_info.alarm_type, + alarm_info.timetamp.tv_sec, + alarm_info.timetamp.tv_usec, + alarm_msg.encode('utf-8')) diff --git a/src/python/xalarm/xalarm_config.py b/src/python/xalarm/xalarm_config.py index 8e56d10d90f4e9ec8aa1fb3b91c57bf3bcb077fa..754a81641176ecf8980035243da84ab1db2907b7 100644 --- a/src/python/xalarm/xalarm_config.py +++ b/src/python/xalarm/xalarm_config.py @@ -15,9 +15,10 @@ Create: 2023-11-02 """ import re +import os import dataclasses import logging -from configparser import ConfigParser +import configparser MAIN_CONFIG_PATH = '/etc/sysSentry/xalarm.conf' @@ -27,6 +28,34 @@ MIN_ID_NUMBER = 1001 MAX_ID_NUMBER = 1128 MAX_ID_MASK_CAPACITY = 128 +# log +CONF_LOG = 'log' +CONF_LOG_LEVEL = 'level' +LogLevel = { + "debug": logging.DEBUG, + "info": logging.INFO, + "warning": logging.WARNING, + "error": logging.ERROR, + "critical": logging.CRITICAL +} + + +def get_log_level(filename=MAIN_CONFIG_PATH): + if not os.path.exists(filename): + return logging.INFO + + try: + config = configparser.ConfigParser() + config.read(filename) + if not config.has_option(CONF_LOG, CONF_LOG_LEVEL): + return logging.INFO + log_level = config.get(CONF_LOG, CONF_LOG_LEVEL) + if log_level.lower() in LogLevel: + return LogLevel.get(log_level.lower()) + return logging.INFO + except configparser.Error: + return logging.INFO + @dataclasses.dataclass class AlarmConfig: @@ -106,7 +135,7 @@ def config_init(): """ alarm_config = AlarmConfig() - cfg = ConfigParser() + cfg = configparser.ConfigParser() cfg.read(MAIN_CONFIG_PATH) id_mask = parse_id_mask(cfg) diff --git a/src/python/xalarm/xalarm_daemon.py b/src/python/xalarm/xalarm_daemon.py index 00e8886c69e94b843ae72c14c2b1deba0016eee7..3ab211c829748b8fecd34a2307089f15440ecd72 100644 --- a/src/python/xalarm/xalarm_daemon.py +++ b/src/python/xalarm/xalarm_daemon.py @@ -21,7 +21,7 @@ import signal import fcntl import socket -from .xalarm_config import config_init +from .xalarm_config import config_init, get_log_level from .xalarm_server import server_loop, SOCK_FILE ALARM_DIR = "/var/run/xalarm" @@ -120,9 +120,10 @@ def alarm_process_create(): os.mkdir(ALARM_DIR) os.chmod(ALARM_DIR, ALARM_DIR_PERMISSION) + log_level = get_log_level() + log_format = "%(asctime)s - %(levelname)s - [%(filename)s:%(lineno)d] - %(message)s" - logging.basicConfig(filename=ALARM_LOGFILE, level=logging.INFO, - format='%(asctime)s|%(levelname)s| %(message)s') + logging.basicConfig(filename=ALARM_LOGFILE, level=log_level, format=log_format) signal.signal(signal.SIGTERM, signal_handler) diff --git a/src/python/xalarm/xalarm_server.py b/src/python/xalarm/xalarm_server.py index 84db2733dd0f8986849c6501a613a481b39a7d55..f90a0e2929886cea7be3ec612c5434695800a0e1 100644 --- a/src/python/xalarm/xalarm_server.py +++ b/src/python/xalarm/xalarm_server.py @@ -17,16 +17,25 @@ Create: 2023-11-02 import socket import os import logging +import select +import threading from struct import error as StructParseError from .xalarm_api import alarm_bin2stu -from .xalarm_transfer import check_filter, transmit_alarm +from .xalarm_transfer import ( + check_filter, + transmit_alarm, + wait_for_connection, + peroid_task_to_cleanup_connections +) ALARM_DIR = "/var/run/xalarm" +USER_RECV_SOCK = "/var/run/xalarm/alarm" SOCK_FILE = "/var/run/xalarm/report" -ALARM_REPORT_LEN = 536 +ALARM_REPORT_LEN = 8216 ALARM_DIR_PERMISSION = 0o750 +ALARM_LISTEN_QUEUE_LEN = 5 def clear_sock_path(): @@ -37,6 +46,8 @@ def clear_sock_path(): os.chmod(ALARM_DIR, ALARM_DIR_PERMISSION) if os.path.exists(SOCK_FILE): os.unlink(SOCK_FILE) + if os.path.exists(USER_RECV_SOCK): + os.unlink(USER_RECV_SOCK) def server_loop(alarm_config): @@ -49,6 +60,25 @@ def server_loop(alarm_config): sock.bind(SOCK_FILE) os.chmod(SOCK_FILE, 0o600) + alarm_sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) + alarm_sock.bind(USER_RECV_SOCK) + os.chmod(USER_RECV_SOCK, 0o600) + alarm_sock.listen(ALARM_LISTEN_QUEUE_LEN) + alarm_sock.setblocking(False) + + epoll = select.epoll() + epoll.register(alarm_sock.fileno(), select.EPOLLIN) + fd_to_socket = {alarm_sock.fileno(): alarm_sock,} + thread_should_stop = False + + conn_thread = threading.Thread(target=wait_for_connection, args=(alarm_sock, epoll, fd_to_socket, thread_should_stop)) + conn_thread.daemon = True + conn_thread.start() + + cleanup_thread = threading.Thread(target=peroid_task_to_cleanup_connections, args=(alarm_sock, epoll, fd_to_socket, thread_should_stop)) + cleanup_thread.daemon = True + cleanup_thread.start() + while True: try: data, _ = sock.recvfrom(ALARM_REPORT_LEN) @@ -58,14 +88,22 @@ def server_loop(alarm_config): logging.debug("server receive report msg length wrong %d", len(data)) continue - alarm_info = alarm_bin2stu(data) logging.debug("server bin2stu msg") if not check_filter(alarm_info, alarm_config): continue + transmit_alarm(alarm_sock, epoll, fd_to_socket, data) + except Exception as e: + logging.error(f"Error server:{e}") + + thread_should_stop = True + conn_thread.join() + cleanup_thread.join() - transmit_alarm(data) - except (ValueError, StructParseError): - pass + epoll.unregister(alarm_sock.fileno()) + epoll.close() + alarm_sock.close() + os.unlink(USER_RECV_SOCK) sock.close() + diff --git a/src/python/xalarm/xalarm_transfer.py b/src/python/xalarm/xalarm_transfer.py index b590b4330762691fe22d628ee6186b014be41341..d13509546ab4539d8a1a639be93db783346530a1 100644 --- a/src/python/xalarm/xalarm_transfer.py +++ b/src/python/xalarm/xalarm_transfer.py @@ -16,10 +16,14 @@ Create: 2023-11-02 import socket import logging +import select +from time import sleep -USER_RECV_SOCK = "/var/run/xalarm/alarm" MIN_ID_NUMBER = 1001 MAX_ID_NUMBER = 1128 +MAX_CONNECTION_NUM = 100 +TEST_CONNECT_BUFFER_SIZE = 32 +PEROID_SCANN_TIME = 60 def check_filter(alarm_info, alarm_filter): @@ -35,16 +39,88 @@ def check_filter(alarm_info, alarm_filter): return True -def transmit_alarm(bin_data): - """forward alarm message +def cleanup_closed_connections(server_sock, epoll, fd_to_socket): """ - sock = socket.socket(socket.AF_UNIX, socket.SOCK_DGRAM) - try: - sock.sendto(bin_data, USER_RECV_SOCK) - logging.debug("transfer alarm success") - except ConnectionRefusedError: - logging.debug("transfer sendto failed") - except FileNotFoundError: - logging.debug("transfer sendto failed") - finally: - sock.close() + clean invalid client socket connections saved in 'fd_to_socket' + :param server_sock: server socket instance of alarm + :param epoll: epoll instance, used to unregister invalid client connections + :param fd_to_socket: dict instance, used to hold client connections and server connections + """ + to_remove = [] + for fileno, connection in fd_to_socket.items(): + if connection is server_sock: + continue + try: + # test whether connection still alive, use MSG_DONTWAIT to avoid blocking thread + # use MSG_PEEK to avoid consuming buffer data + data = connection.recv(TEST_CONNECT_BUFFER_SIZE, socket.MSG_DONTWAIT | socket.MSG_PEEK) + if not data: + to_remove.append(fileno) + except BlockingIOError: + pass + except (ConnectionResetError, ConnectionAbortedError, BrokenPipeError): + to_remove.append(fileno) + + for fileno in to_remove: + fd_to_socket[fileno].close() + del fd_to_socket[fileno] + logging.info(f"cleaned up connection {fileno} for client lost connection.") + + +def peroid_task_to_cleanup_connections(server_sock, epoll, fd_to_socket, thread_should_stop): + while not thread_should_stop: + sleep(PEROID_SCANN_TIME) + cleanup_closed_connections(server_sock, epoll, fd_to_socket) + + +def wait_for_connection(server_sock, epoll, fd_to_socket, thread_should_stop): + """ + thread function for catch and save client connection + :param server_sock: server socket instance of alarm + :param epoll: epoll instance, used to unregister invalid client connections + :param fd_to_socket: dict instance, used to hold client connections and server connections + :param thread_should_stop: bool instance + """ + while not thread_should_stop: + try: + events = epoll.poll(1) + + for fileno, event in events: + if fileno == server_sock.fileno(): + connection, client_address = server_sock.accept() + # if reach max connection, cleanup closed connections + if len(fd_to_socket) - 1 >= MAX_CONNECTION_NUM: + cleanup_closed_connections(server_sock, epoll, fd_to_socket) + # if connections still reach max num, close this connection automatically + if len(fd_to_socket) - 1 >= MAX_CONNECTION_NUM: + logging.info(f"connection reach max num of {MAX_CONNECTION_NUM}, closed current connection!") + connection.close() + continue + fd_to_socket[connection.fileno()] = connection + except socket.error as e: + logging.debug(f"socket error, reason is {e}") + break + except (KeyError, OSError, ValueError) as e: + logging.debug(f"wait for connection failed {e}") + + +def transmit_alarm(server_sock, epoll, fd_to_socket, bin_data): + """ + this function is to broadcast alarm data to client, if fail to send data, remove connections held by fd_to_socket + :param server_sock: server socket instance of alarm + :param epoll: epoll instance, used to unregister invalid client connections + :param fd_to_socket: dict instance, used to hold client connections and server connections + :param bin_data: binary instance, alarm info data in C-style struct format defined in xalarm_api.py + """ + to_remove = [] + for fileno, connection in fd_to_socket.items(): + if connection is not server_sock: + try: + connection.sendall(bin_data) + except (BrokenPipeError, ConnectionResetError): + to_remove.append(fileno) + for fileno in to_remove: + fd_to_socket[fileno].close() + del fd_to_socket[fileno] + logging.info(f"cleaned up connection {fileno} for client lost connection.") +