From d5704af0cf0f14dba5b3e265b2a0eabe863faab5 Mon Sep 17 00:00:00 2001 From: jungheil Date: Fri, 11 Oct 2024 16:39:02 +0800 Subject: [PATCH] refactor: config parser --- .../sentryPlugins/ai_block_io/ai_block_io.py | 116 ++++-- .../ai_block_io/config_parser.py | 387 ++++++++++++------ .../sentryPlugins/ai_block_io/data_access.py | 11 +- .../python/sentryPlugins/ai_block_io/utils.py | 3 +- 4 files changed, 347 insertions(+), 170 deletions(-) diff --git a/sysSentry-1.0.2/src/python/sentryPlugins/ai_block_io/ai_block_io.py b/sysSentry-1.0.2/src/python/sentryPlugins/ai_block_io/ai_block_io.py index 77104a9..c0750f9 100644 --- a/sysSentry-1.0.2/src/python/sentryPlugins/ai_block_io/ai_block_io.py +++ b/sysSentry-1.0.2/src/python/sentryPlugins/ai_block_io/ai_block_io.py @@ -43,82 +43,122 @@ class SlowIODetection: self.__init_detector() def __init_detector_name_list(self): - self._disk_list = check_collect_valid(self._config_parser.get_slow_io_detect_frequency()) + self._disk_list = check_collect_valid( + self._config_parser.slow_io_detect_frequency + ) if self._disk_list is None: - Report.report_pass("get available disk error, please check if the collector plug is enable. exiting...") + Report.report_pass( + "get available disk error, please check if the collector plug is enable. exiting..." + ) exit(1) logging.info(f"ai_block_io plug has found disks: {self._disk_list}") - disks_to_detection: list = self._config_parser.get_disks_to_detection() + disks_to_detection = self._config_parser.disks_to_detection # 情况1：None，则启用所有磁盘检测 # 情况2：is not None and len = 0，则不启动任何磁盘检测 # 情况3：len ！= 0，则取交集 if disks_to_detection is None: - logging.warning("you not specify any disk or use default, so ai_block_io will enable all available disk.") + logging.warning( + "you not specify any disk or use default, so ai_block_io will enable all available disk." + ) for disk in self._disk_list: - self._detector_name_list.append(MetricName(disk, "bio", "read", "latency")) - self._detector_name_list.append(MetricName(disk, "bio", "write", "latency")) - elif len(disks_to_detection) == 0: - logging.warning('please attention: conf file not specify any disk to detection, so it will not start ai block io.') + self._detector_name_list.append( + MetricName(disk, "bio", "read", "latency") + ) + self._detector_name_list.append( + MetricName(disk, "bio", "write", "latency") + ) + if len(self._detector_name_list) >= 30: + logging.warning( + "the number of disks to detection is large than 30, so it will be ignored." + ) + break else: for disk_to_detection in disks_to_detection: if disk_to_detection in self._disk_list: - self._detector_name_list.append(MetricName(disk_to_detection, "bio", "read", "latency")) - self._detector_name_list.append(MetricName(disk_to_detection, "bio", "write", "latency")) + self._detector_name_list.append( + MetricName(disk_to_detection, "bio", "read", "latency") + ) + self._detector_name_list.append( + MetricName(disk_to_detection, "bio", "write", "latency") + ) else: - logging.warning(f"disk：[{disk_to_detection}] not in available disk list, so it will be ignored.") - logging.info(f'start to detection follow disk and it\'s metric: {self._detector_name_list}') + logging.warning( + "disk: [%s] not in available disk list, so it will be ignored.", + disk_to_detection, + ) + if len(self._detector_name_list) == 0: + logging.critical( + "the disks to detection is empty, ai_block_io will exit." + ) + Report.report_pass( + "the disks to detection is empty, ai_block_io will exit." + ) + exit(1) + logging.info( + f"start to detection follow disk and it's metric: {self._detector_name_list}" + ) def __init_detector(self): - train_data_duration, train_update_duration = (self._config_parser. - get_train_data_duration_and_train_update_duration()) - slow_io_detection_frequency = self._config_parser.get_slow_io_detect_frequency() - threshold_type = self._config_parser.get_algorithm_type() - data_queue_size, update_size = get_data_queue_size_and_update_size(train_data_duration, - train_update_duration, - slow_io_detection_frequency) - sliding_window_type = self._config_parser.get_sliding_window_type() - window_size, window_threshold = self._config_parser.get_window_size_and_window_minimum_threshold() + train_data_duration, train_update_duration = ( + self._config_parser.get_train_data_duration_and_train_update_duration() + ) + slow_io_detection_frequency = self._config_parser.slow_io_detect_frequency + threshold_type = self._config_parser.algorithm_type + data_queue_size, update_size = get_data_queue_size_and_update_size( + train_data_duration, train_update_duration, slow_io_detection_frequency + ) + sliding_window_type = self._config_parser.sliding_window_type + window_size, window_threshold = ( + self._config_parser.get_window_size_and_window_minimum_threshold() + ) for detector_name in self._detector_name_list: - threshold = ThresholdFactory().get_threshold(threshold_type, - boxplot_parameter=self._config_parser.get_boxplot_parameter(), - n_sigma_paramter=self._config_parser.get_n_sigma_parameter(), - data_queue_size=data_queue_size, - data_queue_update_size=update_size) - sliding_window = SlidingWindowFactory().get_sliding_window(sliding_window_type, queue_length=window_size, - threshold=window_threshold) + threshold = ThresholdFactory().get_threshold( + threshold_type, + boxplot_parameter=self._config_parser.boxplot_parameter, + n_sigma_paramter=self._config_parser.n_sigma_parameter, + data_queue_size=data_queue_size, + data_queue_update_size=update_size, + ) + sliding_window = SlidingWindowFactory().get_sliding_window( + sliding_window_type, + queue_length=window_size, + threshold=window_threshold, + ) detector = Detector(detector_name, threshold, sliding_window) # 绝对阈值的阈值初始化 if isinstance(threshold, AbsoluteThreshold): - threshold.set_threshold(self._config_parser.get_absolute_threshold()) + threshold.set_threshold(self._config_parser.absolute_threshold) self._detectors[detector_name] = detector logging.info(f"add detector: {detector}") def launch(self): while True: - logging.debug('step0. AI threshold slow io event detection is looping.') + logging.debug("step0. AI threshold slow io event detection is looping.") # Step1：获取IO数据 io_data_dict_with_disk_name = get_io_data_from_collect_plug( - self._config_parser.get_slow_io_detect_frequency(), self._disk_list + self._config_parser.slow_io_detect_frequency, self._disk_list ) - logging.debug(f'step1. Get io data: {str(io_data_dict_with_disk_name)}') + logging.debug(f"step1. Get io data: {str(io_data_dict_with_disk_name)}") if io_data_dict_with_disk_name is None: - Report.report_pass("get io data error, please check if the collector plug is enable. exitting...") + Report.report_pass( + "get io data error, please check if the collector plug is enable. exitting..." + ) exit(1) # Step2：慢IO检测 - logging.debug('step2. Start to detection slow io event.') + logging.debug("step2. Start to detection slow io event.") slow_io_event_list = [] for metric_name, detector in self._detectors.items(): result = detector.is_slow_io_event(io_data_dict_with_disk_name) if result[0]: slow_io_event_list.append((detector.get_metric_name(), result)) - logging.debug('step2. End to detection slow io event.') + logging.debug("step2. End to detection slow io event.") # Step3：慢IO事件上报 - logging.debug('step3. Report slow io event to sysSentry.') + logging.debug("step3. Report slow io event to sysSentry.") for slow_io_event in slow_io_event_list: metric_name: MetricName = slow_io_event[0] result = slow_io_event[1] @@ -135,8 +175,8 @@ class SlowIODetection: logging.warning(alarm_content) # Step4：等待检测时间 - logging.debug('step4. Wait to start next slow io event detection loop.') - time.sleep(self._config_parser.get_slow_io_detect_frequency()) + logging.debug("step4. Wait to start next slow io event detection loop.") + time.sleep(self._config_parser.slow_io_detect_frequency) def main(): diff --git a/sysSentry-1.0.2/src/python/sentryPlugins/ai_block_io/config_parser.py b/sysSentry-1.0.2/src/python/sentryPlugins/ai_block_io/config_parser.py index 354c122..1fa2fa5 100644 --- a/sysSentry-1.0.2/src/python/sentryPlugins/ai_block_io/config_parser.py +++ b/sysSentry-1.0.2/src/python/sentryPlugins/ai_block_io/config_parser.py @@ -9,8 +9,10 @@ # PURPOSE. # See the Mulan PSL v2 for more details. +import os import configparser import logging +from .alarm_report import Report from .threshold import ThresholdType from .utils import get_threshold_type_enum, get_sliding_window_type_enum, get_log_level @@ -21,32 +23,38 @@ LOG_FORMAT = "%(asctime)s - %(levelname)s - [%(filename)s:%(lineno)d] - %(messag def init_log_format(log_level: str): logging.basicConfig(level=get_log_level(log_level.lower()), format=LOG_FORMAT) - if log_level.lower() not in ('info', 'warning', 'error', 'debug'): - logging.warning(f'the log_level: {log_level} you set is invalid, use default value: info.') + if log_level.lower() not in ("info", "warning", "error", "debug"): + logging.warning( + f"the log_level: {log_level} you set is invalid, use default value: info." + ) class ConfigParser: DEFAULT_ABSOLUTE_THRESHOLD = 40 DEFAULT_SLOW_IO_DETECTION_FREQUENCY = 1 - DEFAULT_LOG_LEVEL = 'info' + DEFAULT_LOG_LEVEL = "info" - DEFAULT_ALGORITHM_TYPE = 'boxplot' + DEFAULT_ALGORITHM_TYPE = "boxplot" DEFAULT_TRAIN_DATA_DURATION = 24 DEFAULT_TRAIN_UPDATE_DURATION = 2 DEFAULT_BOXPLOT_PARAMETER = 1.5 DEFAULT_N_SIGMA_PARAMETER = 3 - DEFAULT_SLIDING_WINDOW_TYPE = 'not_continuous' + DEFAULT_SLIDING_WINDOW_TYPE = "not_continuous" DEFAULT_WINDOW_SIZE = 30 DEFAULT_WINDOW_MINIMUM_THRESHOLD = 6 def __init__(self, config_file_name): self.__absolute_threshold = ConfigParser.DEFAULT_ABSOLUTE_THRESHOLD - self.__slow_io_detect_frequency = ConfigParser.DEFAULT_SLOW_IO_DETECTION_FREQUENCY + self.__slow_io_detect_frequency = ( + ConfigParser.DEFAULT_SLOW_IO_DETECTION_FREQUENCY + ) self.__log_level = ConfigParser.DEFAULT_LOG_LEVEL self.__disks_to_detection = None - self.__algorithm_type = ConfigParser.DEFAULT_ALGORITHM_TYPE + self.__algorithm_type = get_threshold_type_enum( + ConfigParser.DEFAULT_ALGORITHM_TYPE + ) self.__train_data_duration = ConfigParser.DEFAULT_TRAIN_UPDATE_DURATION self.__train_update_duration = ConfigParser.DEFAULT_TRAIN_UPDATE_DURATION self.__boxplot_parameter = ConfigParser.DEFAULT_BOXPLOT_PARAMETER @@ -58,199 +66,326 @@ class ConfigParser: self.__config_file_name = config_file_name - def __read_absolute_threshold(self, items_common: dict): + def _get_config_value( + self, + config_items: dict, + key: str, + value_type, + default_value=None, + gt=None, + ge=None, + lt=None, + le=None, + ): + value = config_items.get(key) + if value is None: + logging.warning( + "config of %s not found, the default value %s will be used.", + key, + default_value, + ) + value = default_value + if not value: + logging.critical( + "the value of %s is empty, ai_block_io plug will exit.", key + ) + Report.report_pass( + f"the value of {key} is empty, ai_block_io plug will exit." + ) + exit(1) try: - self.__absolute_threshold = float(items_common.get('absolute_threshold', - ConfigParser.DEFAULT_ABSOLUTE_THRESHOLD)) - if self.__absolute_threshold <= 0: - logging.warning( - f'the_absolute_threshold: {self.__absolute_threshold} you set is invalid, use default value: {ConfigParser.DEFAULT_ABSOLUTE_THRESHOLD}.') - self.__absolute_threshold = ConfigParser.DEFAULT_ABSOLUTE_THRESHOLD + value = value_type(value) except ValueError: - self.__absolute_threshold = ConfigParser.DEFAULT_ABSOLUTE_THRESHOLD - logging.warning( - f'the_absolute_threshold type conversion has error, use default value: {self.__absolute_threshold}.') + logging.critical( + "the value of %s is not a valid %s, ai_block_io plug will exit.", + key, + value_type, + ) + Report.report_pass( + f"the value of {key} is not a valid {value_type}, ai_block_io plug will exit." + ) + exit(1) + if gt is not None and value <= gt: + logging.critical( + "the value of %s is not greater than %s, ai_block_io plug will exit.", + key, + gt, + ) + Report.report_pass( + f"the value of {key} is not greater than {gt}, ai_block_io plug will exit." + ) + exit(1) + if ge is not None and value < ge: + logging.critical( + "the value of %s is not greater than or equal to %s, ai_block_io plug will exit.", + key, + ge, + ) + Report.report_pass( + f"the value of {key} is not greater than or equal to {ge}, ai_block_io plug will exit." + ) + exit(1) + if lt is not None and value >= lt: + logging.critical( + "the value of %s is not less than %s, ai_block_io plug will exit.", + key, + lt, + ) + Report.report_pass( + f"the value of {key} is not less than {lt}, ai_block_io plug will exit." + ) + exit(1) + if le is not None and value > le: + logging.critical( + "the value of %s is not less than or equal to %s, ai_block_io plug will exit.", + key, + le, + ) + Report.report_pass( + f"the value of {key} is not less than or equal to {le}, ai_block_io plug will exit." + ) + exit(1) + + return value + + def __read_absolute_threshold(self, items_common: dict): + self.__absolute_threshold = self._get_config_value( + items_common, + "absolute_threshold", + float, + ConfigParser.DEFAULT_ABSOLUTE_THRESHOLD, + gt=0, + ) def __read__slow_io_detect_frequency(self, items_common: dict): - try: - self.__slow_io_detect_frequency = int(items_common.get('slow_io_detect_frequency', - ConfigParser.DEFAULT_SLOW_IO_DETECTION_FREQUENCY)) - if self.__slow_io_detect_frequency < 1 or self.__slow_io_detect_frequency > 10: - logging.warning( - f'the slow_io_detect_frequency: {self.__slow_io_detect_frequency} you set is invalid, use default value: {ConfigParser.DEFAULT_SLOW_IO_DETECTION_FREQUENCY}.') - self.__slow_io_detect_frequency = ConfigParser.DEFAULT_SLOW_IO_DETECTION_FREQUENCY - except ValueError: - self.__slow_io_detect_frequency = ConfigParser.DEFAULT_SLOW_IO_DETECTION_FREQUENCY - logging.warning(f'slow_io_detect_frequency type conversion has error, use default value: {self.__slow_io_detect_frequency}.') + self.__slow_io_detect_frequency = self._get_config_value( + items_common, + "slow_io_detect_frequency", + int, + ConfigParser.DEFAULT_SLOW_IO_DETECTION_FREQUENCY, + gt=0, + le=300, + ) def __read__disks_to_detect(self, items_common: dict): - disks_to_detection = items_common.get('disk') + disks_to_detection = items_common.get("disk") if disks_to_detection is None: - logging.warning(f'config of disk not found, the default value will be used.') + logging.warning("config of disk not found, the default value will be used.") self.__disks_to_detection = None return - disk_list = disks_to_detection.split(',') - if len(disk_list) == 0 or (len(disk_list) == 1 and disk_list[0] == ''): - logging.warning("you don't specify any disk.") - self.__disks_to_detection = [] - return - if len(disk_list) == 1 and disk_list[0] == 'default': + disks_to_detection = disks_to_detection.strip() + if not disks_to_detection: + logging.critical("the value of disk is empty, ai_block_io plug will exit.") + Report.report_pass( + "the value of disk is empty, ai_block_io plug will exit." + ) + exit(1) + disk_list = disks_to_detection.split(",") + if len(disk_list) == 1 and disk_list[0] == "default": self.__disks_to_detection = None return self.__disks_to_detection = disk_list def __read__train_data_duration(self, items_algorithm: dict): - try: - self.__train_data_duration = float(items_algorithm.get('train_data_duration', - ConfigParser.DEFAULT_TRAIN_DATA_DURATION)) - if self.__train_data_duration <= 0 or self.__train_data_duration > 720: - logging.warning( - f'the train_data_duration: {self.__train_data_duration} you set is invalid, use default value: {ConfigParser.DEFAULT_TRAIN_DATA_DURATION}.') - self.__train_data_duration = ConfigParser.DEFAULT_TRAIN_DATA_DURATION - except ValueError: - self.__train_data_duration = ConfigParser.DEFAULT_TRAIN_DATA_DURATION - logging.warning(f'the train_data_duration type conversion has error, use default value: {self.__train_data_duration}.') + self.__train_data_duration = self._get_config_value( + items_algorithm, + "train_data_duration", + float, + ConfigParser.DEFAULT_TRAIN_DATA_DURATION, + gt=0, + le=720, + ) def __read__train_update_duration(self, items_algorithm: dict): default_train_update_duration = ConfigParser.DEFAULT_TRAIN_UPDATE_DURATION if default_train_update_duration > self.__train_data_duration: default_train_update_duration = self.__train_data_duration / 2 - - try: - self.__train_update_duration = float(items_algorithm.get('train_update_duration', - ConfigParser.DEFAULT_TRAIN_UPDATE_DURATION)) - if self.__train_update_duration <= 0 or self.__train_update_duration > self.__train_data_duration: - logging.warning( - f'the train_update_duration: {self.__train_update_duration} you set is invalid, use default value: {default_train_update_duration}.') - self.__train_update_duration = default_train_update_duration - except ValueError: - self.__train_update_duration = default_train_update_duration - logging.warning(f'the train_update_duration type conversion has error, use default value: {self.__train_update_duration}.') + self.__train_update_duration = self._get_config_value( + items_algorithm, + "train_update_duration", + float, + default_train_update_duration, + gt=0, + le=self.__train_data_duration, + ) def __read__algorithm_type_and_parameter(self, items_algorithm: dict): - algorithm_type = items_algorithm.get('algorithm_type', ConfigParser.DEFAULT_ALGORITHM_TYPE) + algorithm_type = items_algorithm.get( + "algorithm_type", ConfigParser.DEFAULT_ALGORITHM_TYPE + ) self.__algorithm_type = get_threshold_type_enum(algorithm_type) + if self.__algorithm_type is None: + logging.critical( + "the algorithm_type: %s you set is invalid. ai_block_io plug will exit.", + algorithm_type, + ) + Report.report_pass( + f"the algorithm_type: {algorithm_type} you set is invalid. ai_block_io plug will exit." + ) + exit(1) if self.__algorithm_type == ThresholdType.NSigmaThreshold: - try: - self.__n_sigma_parameter = float(items_algorithm.get('n_sigma_parameter', - ConfigParser.DEFAULT_N_SIGMA_PARAMETER)) - if self.__n_sigma_parameter <= 0 or self.__n_sigma_parameter > 10: - logging.warning( - f'the n_sigma_parameter: {self.__n_sigma_parameter} you set is invalid, use default value: {ConfigParser.DEFAULT_N_SIGMA_PARAMETER}.') - self.__n_sigma_parameter = ConfigParser.DEFAULT_N_SIGMA_PARAMETER - except ValueError: - self.__n_sigma_parameter = ConfigParser.DEFAULT_N_SIGMA_PARAMETER - logging.warning(f'the n_sigma_parameter type conversion has error, use default value: {self.__n_sigma_parameter}.') + self.__n_sigma_parameter = self._get_config_value( + items_algorithm, + "n_sigma_parameter", + float, + ConfigParser.DEFAULT_N_SIGMA_PARAMETER, + gt=0, + le=10, + ) elif self.__algorithm_type == ThresholdType.BoxplotThreshold: - try: - self.__boxplot_parameter = float(items_algorithm.get('boxplot_parameter', - ConfigParser.DEFAULT_BOXPLOT_PARAMETER)) - if self.__boxplot_parameter <= 0 or self.__boxplot_parameter > 10: - logging.warning( - f'the boxplot_parameter: {self.__boxplot_parameter} you set is invalid, use default value: {ConfigParser.DEFAULT_BOXPLOT_PARAMETER}.') - self.__n_sigma_parameter = ConfigParser.DEFAULT_BOXPLOT_PARAMETER - except ValueError: - self.__boxplot_parameter = ConfigParser.DEFAULT_BOXPLOT_PARAMETER - logging.warning(f'the boxplot_parameter type conversion has error, use default value: {self.__boxplot_parameter}.') + self.__boxplot_parameter = self._get_config_value( + items_algorithm, + "boxplot_parameter", + float, + ConfigParser.DEFAULT_BOXPLOT_PARAMETER, + gt=0, + le=10, + ) def __read__window_size(self, items_sliding_window: dict): - try: - self.__window_size = int(items_sliding_window.get('window_size', - ConfigParser.DEFAULT_WINDOW_SIZE)) - if self.__window_size < 1 or self.__window_size > 3600: - logging.warning( - f'the window_size: {self.__window_size} you set is invalid, use default value: {ConfigParser.DEFAULT_WINDOW_SIZE}.') - self.__window_size = ConfigParser.DEFAULT_WINDOW_SIZE - except ValueError: - self.__window_size = ConfigParser.DEFAULT_WINDOW_SIZE - logging.warning(f'window_size type conversion has error, use default value: {self.__window_size}.') + self.__window_size = self._get_config_value( + items_sliding_window, + "window_size", + int, + ConfigParser.DEFAULT_WINDOW_SIZE, + gt=0, + le=3600, + ) def __read__window_minimum_threshold(self, items_sliding_window: dict): default_window_minimum_threshold = ConfigParser.DEFAULT_WINDOW_MINIMUM_THRESHOLD if default_window_minimum_threshold > self.__window_size: default_window_minimum_threshold = self.__window_size / 2 - try: - self.__window_minimum_threshold = ( - int(items_sliding_window.get('window_minimum_threshold', - ConfigParser.DEFAULT_WINDOW_MINIMUM_THRESHOLD))) - if self.__window_minimum_threshold < 1 or self.__window_minimum_threshold > self.__window_size: - logging.warning( - f'the window_minimum_threshold: {self.__window_minimum_threshold} you set is invalid, use default value: {default_window_minimum_threshold}.') - self.__window_minimum_threshold = default_window_minimum_threshold - except ValueError: - self.__window_minimum_threshold = default_window_minimum_threshold - logging.warning(f'window_minimum_threshold type conversion has error, use default value: {self.__window_minimum_threshold}.') + self.__window_minimum_threshold = self._get_config_value( + items_sliding_window, + "window_minimum_threshold", + int, + default_window_minimum_threshold, + gt=0, + le=self.__window_size, + ) def read_config_from_file(self): + if not os.path.exists(self.__config_file_name): + init_log_format(self.__log_level) + logging.critical( + "config file %s not found, ai_block_io plug will exit.", + self.__config_file_name, + ) + Report.report_pass( + f"config file {self.__config_file_name} not found, ai_block_io plug will exit." + ) + exit(1) + con = configparser.ConfigParser() try: - con.read(self.__config_file_name, encoding='utf-8') + con.read(self.__config_file_name, encoding="utf-8") except configparser.Error as e: init_log_format(self.__log_level) - logging.critical(f'config file read error: {e}, ai_block_io plug will exit.') + logging.critical( + f"config file read error: %s, ai_block_io plug will exit.", e + ) + Report.report_pass( + f"config file read error: {e}, ai_block_io plug will exit." + ) exit(1) - if con.has_section('common'): - items_common = dict(con.items('common')) - self.__log_level = items_common.get('log_level', ConfigParser.DEFAULT_LOG_LEVEL) + if con.has_section("common"): + items_common = dict(con.items("common")) + self.__log_level = items_common.get( + "log_level", ConfigParser.DEFAULT_LOG_LEVEL + ) init_log_format(self.__log_level) self.__read_absolute_threshold(items_common) self.__read__slow_io_detect_frequency(items_common) self.__read__disks_to_detect(items_common) else: init_log_format(self.__log_level) - logging.warning("common section parameter not found, it will be set to default value.") + logging.warning( + "common section parameter not found, it will be set to default value." + ) - if con.has_section('algorithm'): - items_algorithm = dict(con.items('algorithm')) + if con.has_section("algorithm"): + items_algorithm = dict(con.items("algorithm")) self.__read__train_data_duration(items_algorithm) self.__read__train_update_duration(items_algorithm) self.__read__algorithm_type_and_parameter(items_algorithm) else: - logging.warning("algorithm section parameter not found, it will be set to default value.") - - if con.has_section('sliding_window'): - items_sliding_window = dict(con.items('sliding_window')) - sliding_window_type = items_sliding_window.get('sliding_window_type', - ConfigParser.DEFAULT_SLIDING_WINDOW_TYPE) - self.__sliding_window_type = get_sliding_window_type_enum(sliding_window_type) + logging.warning( + "algorithm section parameter not found, it will be set to default value." + ) + + if con.has_section("sliding_window"): + items_sliding_window = dict(con.items("sliding_window")) + sliding_window_type = items_sliding_window.get( + "sliding_window_type", ConfigParser.DEFAULT_SLIDING_WINDOW_TYPE + ) + self.__sliding_window_type = get_sliding_window_type_enum( + sliding_window_type + ) self.__read__window_size(items_sliding_window) self.__read__window_minimum_threshold(items_sliding_window) else: - logging.warning("sliding_window section parameter not found, it will be set to default value.") + logging.warning( + "sliding_window section parameter not found, it will be set to default value." + ) self.__print_all_config_value() def __print_all_config_value(self): pass - def get_slow_io_detect_frequency(self): + def get_train_data_duration_and_train_update_duration(self): + return self.__train_data_duration, self.__train_update_duration + + def get_window_size_and_window_minimum_threshold(self): + return self.__window_size, self.__window_minimum_threshold + + @property + def slow_io_detect_frequency(self): return self.__slow_io_detect_frequency - def get_algorithm_type(self): + @property + def algorithm_type(self): return self.__algorithm_type - def get_sliding_window_type(self): + @property + def sliding_window_type(self): return self.__sliding_window_type - def get_train_data_duration_and_train_update_duration(self): - return self.__train_data_duration, self.__train_update_duration + @property + def train_data_duration(self): + return self.__train_data_duration - def get_window_size_and_window_minimum_threshold(self): - return self.__window_size, self.__window_minimum_threshold + @property + def train_update_duration(self): + return self.__train_update_duration + + @property + def window_size(self): + return self.__window_size + + @property + def window_minimum_threshold(self): + return self.__window_minimum_threshold - def get_absolute_threshold(self): + @property + def absolute_threshold(self): return self.__absolute_threshold - def get_log_level(self): + @property + def log_level(self): return self.__log_level - def get_disks_to_detection(self): + @property + def disks_to_detection(self): return self.__disks_to_detection - def get_boxplot_parameter(self): + @property + def boxplot_parameter(self): return self.__boxplot_parameter - def get_n_sigma_parameter(self): + @property + def n_sigma_parameter(self): return self.__n_sigma_parameter diff --git a/sysSentry-1.0.2/src/python/sentryPlugins/ai_block_io/data_access.py b/sysSentry-1.0.2/src/python/sentryPlugins/ai_block_io/data_access.py index c7679cd..bdc484a 100644 --- a/sysSentry-1.0.2/src/python/sentryPlugins/ai_block_io/data_access.py +++ b/sysSentry-1.0.2/src/python/sentryPlugins/ai_block_io/data_access.py @@ -41,11 +41,14 @@ def check_collect_valid(period): try: data = json.loads(data_raw["message"]) except Exception as e: - logging.warning(f"get io data failed, {e}") + logging.warning(f"get valid devices failed, occur exception: {e}") + return None + if data.empty(): + logging.warning(f"get valid devices failed, return {data_raw}") return None return [k for k in data.keys()] else: - logging.warning(f"get io data failed, return {data_raw}") + logging.warning(f"get valid devices failed, return {data_raw}") return None @@ -60,7 +63,7 @@ def _get_raw_data(period, disk_list): def _get_io_stage_data(data): io_stage_data = IOStageData() - for data_type in ('read', 'write', 'flush', 'discard'): + for data_type in ("read", "write", "flush", "discard"): if data_type in data: getattr(io_stage_data, data_type).latency = data[data_type][0] getattr(io_stage_data, data_type).io_dump = data[data_type][1] @@ -87,7 +90,7 @@ def get_io_data_from_collect_plug(period, disk_list): getattr(disk_ret, k) setattr(disk_ret, k, _get_io_stage_data(v)) except AttributeError: - logging.debug(f'no attr {k}') + logging.debug(f"no attr {k}") continue ret[disk] = disk_ret return ret diff --git a/sysSentry-1.0.2/src/python/sentryPlugins/ai_block_io/utils.py b/sysSentry-1.0.2/src/python/sentryPlugins/ai_block_io/utils.py index 8dbba06..0ed37b9 100644 --- a/sysSentry-1.0.2/src/python/sentryPlugins/ai_block_io/utils.py +++ b/sysSentry-1.0.2/src/python/sentryPlugins/ai_block_io/utils.py @@ -25,8 +25,7 @@ def get_threshold_type_enum(algorithm_type: str): return ThresholdType.BoxplotThreshold if algorithm_type.lower() == 'n_sigma': return ThresholdType.NSigmaThreshold - logging.warning(f"the algorithm type: {algorithm_type} you set is invalid, use default value: boxplot") - return ThresholdType.BoxplotThreshold + return None def get_sliding_window_type_enum(sliding_window_type: str): -- Gitee