diff --git a/README-add-data-collecting-output-description.patch b/README-add-data-collecting-output-description.patch new file mode 100644 index 0000000000000000000000000000000000000000..277c72f8073b19be9f0a96c191ae9170700babf2 --- /dev/null +++ b/README-add-data-collecting-output-description.patch @@ -0,0 +1,47 @@ +From c8d0e4c3030808fe0a575e595d8bd9a51d30124a Mon Sep 17 00:00:00 2001 +From: HuBin95 +Date: Mon, 7 Mar 2022 02:10:24 +0000 +Subject: [PATCH 02/11] README: add data collecting output description + +--- + README.en.md | 4 +++- + README.md | 4 +++- + 2 files changed, 6 insertions(+), 2 deletions(-) + +diff --git a/README.en.md b/README.en.md +index 2f11270..8a54fa4 100644 +--- a/README.en.md ++++ b/README.en.md +@@ -52,9 +52,11 @@ Table 1 **collect_data.json** file + | sample_num | Sample number to be collected. | Integer | > 0 | + | interval | Interval for collecting data, in seconds. | Integer | > 0 | + | output_dir | Path for storing collected data. | Character string | - | +-| workload_type | Application load type of the collection environment. The default value is **default**. | Character string | - | ++| workload_type | Application load type of the collection environment, used as output file name. The default value is **default**. | Character string | - | + | collection_items | Table 2 lists the system parameters to be collected. | List | - | + ++When data collecting is finished, the data will be saved as: `${output_dir}/${workload_type}-${finish_timestamp}.csv` ++ + Table 2 Description of the **collection_items** configuration + + | Parameter | Description | Type | Value Range | +diff --git a/README.md b/README.md +index 36d5954..c121c8d 100644 +--- a/README.md ++++ b/README.md +@@ -53,9 +53,11 @@ python3 collect_data.py [OPTIONS] + | sample_num | 待采集的次数 | 整型 | >0 | + | interval | 待采集的间隔时间，单位为秒 | 整型 | >0 | + | output_dir | 采集完后数据存储的文件路径 | 字符串 | - | +-| workload_type | 采集环境的应用负载类型，默认为default | 字符串 | - | ++| workload_type | 采集环境的应用负载类型，用作输出文件名，默认为default | 字符串 | - | + | collection_items | 需要采集的系统参数项，参见表2 | 列表 | - | + ++最终采集完后，数据将保存为: `${output_dir}/${workload_type}-${finish_timestamp}.csv` ++ + 表2 collection_items项配置说明 + + | **配置名称** | **配置说明** | **参数类型** | **取值范围** | +-- +2.27.0 + diff --git a/Ubuntu-20.04.3-LTS.patch b/Ubuntu-20.04.3-LTS.patch new file mode 100644 index 0000000000000000000000000000000000000000..e5b1f4dbbe2871f674b22f946ad6937905e742b9 --- /dev/null +++ b/Ubuntu-20.04.3-LTS.patch @@ -0,0 +1,39 @@ +From 2fd1148dcec21a8a99fbb59240ba2e2abfe7c31c Mon Sep 17 00:00:00 2001 +From: Masterwater-y +Date: Fri, 28 Jan 2022 18:07:52 +0800 +Subject: [PATCH 01/11] adapt Ubuntu 20.04.3 LTS + +--- + atune_collector/plugin/monitor/common.py | 2 +- + atune_collector/plugin/monitor/memory/topo.py | 2 ++ + 2 files changed, 3 insertions(+), 1 deletion(-) + +diff --git a/atune_collector/plugin/monitor/common.py b/atune_collector/plugin/monitor/common.py +index a646b32..12a07f2 100755 +--- a/atune_collector/plugin/monitor/common.py ++++ b/atune_collector/plugin/monitor/common.py +@@ -230,7 +230,7 @@ class Monitor(object): + def walk_class_type(father, class_type, desc, datas): + """get key field""" + if "class" in father and father["class"] == class_type: +- if "description" in father and (desc is None or father["description"] == desc): ++ if "description" in father and (desc is None or father["description"].lower() == desc.lower()): + datas.append(father) + return + if "children" in father: +diff --git a/atune_collector/plugin/monitor/memory/topo.py b/atune_collector/plugin/monitor/memory/topo.py +index 7e828f3..2c0c39e 100755 +--- a/atune_collector/plugin/monitor/memory/topo.py ++++ b/atune_collector/plugin/monitor/memory/topo.py +@@ -95,6 +95,8 @@ class MemTopo(Monitor): + cmd=self.__cmd).split(), stderr=no_print) + info = o_json.decode() + json_content = json.loads(info) ++ if (isinstance(json_content, list)): ++ json_content = {'children': json_content} + + dict_datas = get_class_type(json_content, "memory", "System Memory") + if fmt == "json": +-- +2.27.0 + diff --git a/add-new-dims.patch b/add-new-dims.patch new file mode 100644 index 0000000000000000000000000000000000000000..54825a2933deea6141bb038092e4c86f2e4a8574 --- /dev/null +++ b/add-new-dims.patch @@ -0,0 +1,427 @@ +From 2d6269e360a654f11370f31ea4faf6ee4012d73f Mon Sep 17 00:00:00 2001 +From: hamster +Date: Mon, 14 Nov 2022 03:24:49 +0000 +Subject: [PATCH 04/11] add new dimensions + +From: @hujing2 +Reviewed-by: @gaoruoshu +Signed-off-by: @gaoruoshu +--- + README.md | 16 +++ + atune_collector/collect_data.json | 14 ++ + atune_collector/collect_data.py | 97 ++++++++------ + atune_collector/plugin/monitor/__init__.py | 1 + + atune_collector/plugin/monitor/common.py | 1 + + .../plugin/monitor/process/__init__.py | 20 +++ + .../plugin/monitor/process/sched.py | 124 ++++++++++++++++++ + 7 files changed, 233 insertions(+), 40 deletions(-) + create mode 100644 atune_collector/plugin/monitor/process/__init__.py + create mode 100644 atune_collector/plugin/monitor/process/sched.py + +diff --git a/README.md b/README.md +index c121c8d..242f345 100644 +--- a/README.md ++++ b/README.md +@@ -50,12 +50,14 @@ python3 collect_data.py [OPTIONS] + | ---------------- | ------------------------------------- | ------------ | ------------ | + | network | 待采集的指定网卡 | 字符串 | - | + | block | 待采集的指定磁盘 | 字符串 | - | ++| application | 需要采集的应用进程 | 字符串 | - | + | sample_num | 待采集的次数 | 整型 | >0 | + | interval | 待采集的间隔时间，单位为秒 | 整型 | >0 | + | output_dir | 采集完后数据存储的文件路径 | 字符串 | - | + | workload_type | 采集环境的应用负载类型，用作输出文件名，默认为default | 字符串 | - | + | collection_items | 需要采集的系统参数项，参见表2 | 列表 | - | + ++ + 最终采集完后，数据将保存为: `${output_dir}/${workload_type}-${finish_timestamp}.csv` + + 表2 collection_items项配置说明 +@@ -76,6 +78,7 @@ collect_data.json文件配置示例： + { + "network": "eth0", + "block": "sda", ++ "application": "mysqld", + "sample_num": 20, + "interval": 5, + "output_dir": "/var/atuned/collect_data", +@@ -203,6 +206,19 @@ collect_data.json文件配置示例： + "metrics": [ + "fd-util" + ] ++ }, ++ { ++ "name": "process", ++ "module": "PROCESS", ++ "purpose": "SCHED", ++ "metrics": [ ++ "exec_start", ++ "vruntime", ++ "sum_exec_runtime", ++ "switches", ++ "voluntary_switches", ++ "involuntary_switches" ++ ] + } + ] + } +diff --git a/atune_collector/collect_data.json b/atune_collector/collect_data.json +index db96501..af286bd 100755 +--- a/atune_collector/collect_data.json ++++ b/atune_collector/collect_data.json +@@ -1,6 +1,7 @@ + { + "network": "eth0", + "block": "sda", ++ "application": "firewalld,dockerd", + "sample_num": 20, + "interval": 5, + "output_dir": "/var/atuned/collect_data", +@@ -140,6 +141,19 @@ + "metrics": [ + "fd-util" + ] ++ }, ++ { ++ "name": "process", ++ "module": "PROCESS", ++ "purpose": "SCHED", ++ "metrics": [ ++ "exec_start", ++ "vruntime", ++ "sum_exec_runtime", ++ "switches", ++ "voluntary_switches", ++ "involuntary_switches" ++ ] + } + ] + } +\ No newline at end of file +diff --git a/atune_collector/collect_data.py b/atune_collector/collect_data.py +index 1764304..3593db6 100755 +--- a/atune_collector/collect_data.py ++++ b/atune_collector/collect_data.py +@@ -18,6 +18,7 @@ import argparse + import json + import os + import time ++import csv + + from plugin.plugin import MPI + +@@ -30,66 +31,80 @@ class Collector: + self.field_name = [] + self.support_multi_block = ['storage'] + self.support_multi_nic = ['network', 'network-err'] ++ self.support_multi_app = ['process'] + + def parse_json(self): + """parse json data""" + monitors = [] + for item in self.data["collection_items"]: +- parameters = ["--interval=%s;" % self.data["interval"]] +- for metric in item["metrics"]: +- nics = self.data["network"].split(',') +- blocks = self.data["block"].split(',') +- if item["name"] in self.support_multi_nic and len(nics) > 1: +- for net in nics: ++ if item["name"] in self.support_multi_app and ('application' not in self.data or ++ self.data["application"] == ""): ++ continue ++ if item["name"] in self.support_multi_app: ++ applications = self.data["application"].split(',') ++ parameters = ["--interval=%s --app=%s;" %(self.data["interval"], self.data["application"])] ++ for application in applications: ++ for metric in item["metrics"]: + self.field_name.append( +- "%s.%s.%s#%s" % (item["module"], item["purpose"], metric, net)) +- elif item["name"] in self.support_multi_block and len(blocks) > 1: +- for block in blocks: +- self.field_name.append( +- "%s.%s.%s#%s" % (item["module"], item["purpose"], metric, block)) +- else: +- self.field_name.append("%s.%s.%s" % (item["module"], item["purpose"], metric)) +- parameters.append("--fields=%s" % metric) +- if "threshold" in item: +- parameters.append("--threshold=%s" % item["threshold"]) ++ "%s.%s.%s#%s" % (item["module"], item["purpose"], metric, application)) ++ parameters.append("--fields=%s" % metric) ++ else: ++ parameters = ["--interval=%s;" % self.data["interval"]] ++ for metric in item["metrics"]: ++ nics = self.data["network"].split(',') ++ blocks = self.data["block"].split(',') ++ ++ if item["name"] in self.support_multi_nic and len(nics) > 1: ++ for net in nics: ++ self.field_name.append( ++ "%s.%s.%s#%s" % (item["module"], item["purpose"], metric, net)) ++ elif item["name"] in self.support_multi_block and len(blocks) > 1: ++ for block in blocks: ++ self.field_name.append( ++ "%s.%s.%s#%s" % (item["module"], item["purpose"], metric, block)) ++ else: ++ self.field_name.append("%s.%s.%s" % (item["module"], item["purpose"], metric)) ++ parameters.append("--fields=%s" % metric) ++ if "threshold" in item: ++ parameters.append("--threshold=%s" % item["threshold"]) ++ + parameters.append("--nic=%s" % self.data["network"]) + parameters.append("--device=%s" % self.data["block"]) + monitors.append([item["module"], item["purpose"], " ".join(parameters)]) + return monitors + +- def save_csv(self, field_data): +- """save data to csv file""" ++ def collect_data(self): ++ """collect data""" ++ collect_num = self.data["sample_num"] ++ if int(collect_num) < 1: ++ os.abort("sample_num must be greater than 0") ++ ++ mpi = MPI() ++ monitors = self.parse_json() + path = self.data["output_dir"] + if not os.path.exists(path): + os.makedirs(path, 0o750) + file_name = "{}-{}.csv".format(self.data.get("workload_type", "default"), + int(round(time.time() * 1000))) +- import csv ++ ++ print("start to collect data, csv path is %s" % os.path.join(path, file_name)) ++ print(" ".join(self.field_name)) + with open(os.path.join(path, file_name), "w") as csvfile: + writer = csv.writer(csvfile) + self.field_name.insert(0, "TimeStamp") + writer.writerow(self.field_name) +- writer.writerows(field_data) ++ csvfile.flush() ++ for _ in range(collect_num): ++ raw_data = mpi.get_monitors_data(monitors) ++ float_data = [float(num) for num in raw_data] ++ str_data = [str(round(value, 3)) for value in float_data] ++ print(" ".join(str_data)) ++ float_data.insert(0, time.strftime("%H:%M:%S")) ++ # field_data.append(float_data) ++ writer.writerow(float_data) ++ csvfile.flush() + print("finish to collect data, csv path is %s" % os.path.join(path, file_name)) + +- def collect_data(self): +- """collect data""" +- collect_num = self.data["sample_num"] +- if int(collect_num) < 1: +- os.abort("sample_num must be greater than 0") +- field_data = [] +- mpi = MPI() +- monitors = self.parse_json() +- print(" ".join(self.field_name)) +- for _ in range(collect_num): +- raw_data = mpi.get_monitors_data(monitors) +- float_data = [float(num) for num in raw_data] +- str_data = [str(round(value, 3)) for value in float_data] +- print(" ".join(str_data)) +- float_data.insert(0, time.strftime("%H:%M:%S")) +- field_data.append(float_data) +- return field_data +- + + if __name__ == "__main__": + default_json_path = "/etc/atune_collector/collect_data.json" +@@ -100,5 +115,7 @@ if __name__ == "__main__": + with open(ARGS.config, 'r') as file: + json_data = json.load(file) + collector = Collector(json_data) +- dataset = collector.collect_data() +- collector.save_csv(dataset) ++ try: ++ collector.collect_data() ++ except KeyboardInterrupt: ++ print("user stop collect data") +\ No newline at end of file +diff --git a/atune_collector/plugin/monitor/__init__.py b/atune_collector/plugin/monitor/__init__.py +index 6291401..9292071 100755 +--- a/atune_collector/plugin/monitor/__init__.py ++++ b/atune_collector/plugin/monitor/__init__.py +@@ -19,6 +19,7 @@ __all__ = [ + "memory", + "network", + "performance", ++ "process", + "processor", + "storage", + "common", +diff --git a/atune_collector/plugin/monitor/common.py b/atune_collector/plugin/monitor/common.py +index 12a07f2..d0aa60c 100755 +--- a/atune_collector/plugin/monitor/common.py ++++ b/atune_collector/plugin/monitor/common.py +@@ -134,6 +134,7 @@ class Monitor(object): + "--interval=" to specify period of time + "--cpu=" to select which cpu + "--event=" to select which event ++ "--app" to select which applications + :returns value: Success, collected info string + :raises Exceptions: Fail, with info + """ +diff --git a/atune_collector/plugin/monitor/process/__init__.py b/atune_collector/plugin/monitor/process/__init__.py +new file mode 100644 +index 0000000..4c4ceb3 +--- /dev/null ++++ b/atune_collector/plugin/monitor/process/__init__.py +@@ -0,0 +1,20 @@ ++#!/usr/bin/python3 ++# -*- coding: utf-8 -*- ++# Copyright (c) 2022 Huawei Technologies Co., Ltd. ++# A-Tune is licensed under the Mulan PSL v2. ++# You can use this software according to the terms and conditions of the Mulan PSL v2. ++# You may obtain a copy of Mulan PSL v2 at: ++# http://license.coscl.org.cn/MulanPSL2 ++# THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR ++# IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR ++# PURPOSE. ++# See the Mulan PSL v2 for more details. ++# Create: 2022-10-14 ++ ++""" ++Init file. ++""" ++ ++__all__ = ["sched"] ++ ++from . import * +\ No newline at end of file +diff --git a/atune_collector/plugin/monitor/process/sched.py b/atune_collector/plugin/monitor/process/sched.py +new file mode 100644 +index 0000000..5289d84 +--- /dev/null ++++ b/atune_collector/plugin/monitor/process/sched.py +@@ -0,0 +1,124 @@ ++#!/usr/bin/python3 ++# -*- coding: utf-8 -*- ++# Copyright (c) 2022 Huawei Technologies Co., Ltd. ++# A-Tune is licensed under the Mulan PSL v2. ++# You can use this software according to the terms and conditions of the Mulan PSL v2. ++# You may obtain a copy of Mulan PSL v2 at: ++# http://license.coscl.org.cn/MulanPSL2 ++# THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR ++# IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR ++# PURPOSE. ++# See the Mulan PSL v2 for more details. ++# Create: 2022-10-14 ++ ++""" ++The sub class of the monitor, used to collect the process sched info ++""" ++import inspect ++import logging ++import subprocess ++import getopt ++import re ++from ..common import Monitor ++ ++LOGGER = logging.getLogger(__name__) ++ ++ ++class ProcSched(Monitor): ++ """To collect the process sched info""" ++ _module = "PROCESS" ++ _purpose = "SCHED" ++ _option = "/proc/{}/sched" ++ ++ def __init__(self, user=None): ++ Monitor.__init__(self, user) ++ self.__cmd = "cat" ++ self.__interval = 1 ++ self.__applications = [] ++ self.__pids = [] ++ ++ def _get(self, para=None): ++ output = "" ++ pids = [] ++ if para is not None: ++ opts, _ = getopt.getopt(para.split(), None, ['interval=', 'app=']) ++ for opt, val in opts: ++ if opt in '--interval': ++ if val.isdigit(): ++ self.__interval = int(val) ++ else: ++ err = ValueError( ++ "Invalid parameter: {opt}={val}".format( ++ opt=opt, val=val)) ++ LOGGER.error("%s.%s: %s", self.__class__.__name__, ++ inspect.stack()[0][3], str(err)) ++ raise err ++ continue ++ elif opt in '--app': ++ if val is not None: ++ self.__applications = val.split(',') ++ else: ++ err = ValueError( ++ "{opt} parameter is none".format( ++ opt=opt)) ++ LOGGER.error("%s.%s: %s", self.__class__.__name__, ++ inspect.stack()[0][3], str(err)) ++ raise err ++ ++ for app in self.__applications: ++ pid = subprocess.getoutput( ++ "ps -A | grep {} | awk '{{print $1}}'".format(app)).split()[0] ++ pids.append(pid) ++ self.__pids = pids ++ ++ for pid in self.__pids: ++ out = subprocess.check_output( ++ "{cmd} {opt}".format( ++ cmd=self.__cmd, ++ opt=self._option.format(pid)).split()) ++ output = output + "" + out.decode() ++ return output ++ ++ def decode(self, info, para): ++ """ ++ decode the result of the operation ++ :param info: content that needs to be decoded ++ :param para: command line argument ++ :returns ret: operation result ++ """ ++ ++ if para is None: ++ return info ++ ++ start = 0 ++ keys = [] ++ ret = "" ++ ++ opts, _ = getopt.getopt(para.split(), None, ['nic=', 'fields=', 'device=']) ++ for opt, val in opts: ++ if opt in '--fields': ++ keys.append(val) ++ continue ++ ++ pattern = re.compile( ++ r"(\w+)\ {1,}\:\ {1,}(\d+.\d+)", ++ re.I | re.UNICODE | re.MULTILINE) ++ search_obj = pattern.findall(info) ++ search_list = [] ++ for obj in search_obj: ++ if obj[0][:3] == "nr_": ++ search_list.append(obj[0][3:]) ++ else: ++ search_list.append(obj[0]) ++ search_list.append(obj[1]) ++ if len(search_obj) == 0: ++ err = LookupError("Fail to find data") ++ LOGGER.error("%s.%s: %s", self.__class__.__name__, ++ inspect.stack()[0][3], str(err)) ++ raise err ++ ++ while start <= len(self.__applications) * len(keys): ++ for key in keys: ++ ret = ret + " " + search_list[search_list.index(key, start)+1] ++ start = search_list.index(key, start) + 1 ++ return ret +\ No newline at end of file +-- +2.27.0 + diff --git a/atune-collector.spec b/atune-collector.spec index c178b8e180837a73cceb35428781bc35800eb8a3..cc98480b285ad6df3a4f22eaf75922b7f05edfe4 100644 --- a/atune-collector.spec +++ b/atune-collector.spec @@ -2,13 +2,23 @@ Name: atune-collector Version: 1.1.0 -Release: 2 +Release: 3 Summary: A-Tune-Collector is used to collect various system resources. License: Mulan PSL v2 URL: https://gitee.com/openeuler/A-Tune-Collector Source: https://gitee.com/openeuler/A-Tune-Collector/repository/archive/v%{version}.tar.gz -Patch0: 0001-fix-bug-list-index-out-of-range-in-virtualbox.patch +Patch1: Ubuntu-20.04.3-LTS.patch +Patch2: README-add-data-collecting-output-description.patch +Patch3: fix-bug-list-index-out-of-range-in-virtualbox.patch +Patch4: add-new-dims.patch +Patch5: fix-CWE-23.patch +Patch6: memory-bandwidth-dimm-slot-atuned.patch +Patch7: feature-set-mysql.patch +Patch8: feature-set-redis-value.patch +Patch9: feature-set-nginx.patch +Patch10: bugfix-create-file-only-if-setting-params.patch +Patch11: change-import-behavior.patch BuildRequires: python3-setuptools Requires: python3-dict2xml @@ -33,6 +43,9 @@ The A-Tune-Collector is used to collect various system resources and can also be %attr(0600,root,root) %{_sysconfdir}/atune_collector/* %changelog +* Tue Aug 01 2023 gaoruoshu - 1.1.0-3 +- feature: enable application config + * Thu Mar 09 2023 zhangjian - 1.1.0-2 - fix bug: list index out of range in virtualbox diff --git a/bugfix-create-file-only-if-setting-params.patch b/bugfix-create-file-only-if-setting-params.patch new file mode 100644 index 0000000000000000000000000000000000000000..d927561217df93c62a2224a181b308447afb9632 --- /dev/null +++ b/bugfix-create-file-only-if-setting-params.patch @@ -0,0 +1,113 @@ +From 1c513970f8f5e81268e998cc33ab8a93802bb722 Mon Sep 17 00:00:00 2001 +From: gaoruoshu +Date: Fri, 14 Jul 2023 18:47:25 +0800 +Subject: [PATCH 10/11] bugfix: create file only if setting params + +--- + atune_collector/plugin/configurator/mysql/mysql.py | 9 ++++++--- + atune_collector/plugin/configurator/nginx/nginx.py | 7 ++++--- + atune_collector/plugin/configurator/redis/redis.py | 7 +++++-- + 3 files changed, 15 insertions(+), 8 deletions(-) + +diff --git a/atune_collector/plugin/configurator/mysql/mysql.py b/atune_collector/plugin/configurator/mysql/mysql.py +index cfc533d..e472a1c 100644 +--- a/atune_collector/plugin/configurator/mysql/mysql.py ++++ b/atune_collector/plugin/configurator/mysql/mysql.py +@@ -31,8 +31,10 @@ class Mysql(Configurator): + self.__cmd = "" + self.__file_path = "/etc/my.cnf" + self.__mysqld_ind = -1 ++ ++ def _init_file(self): + if not os.path.isfile(self.__file_path): +- with open(self.__file_path, 'w'): ++ with open(self.__file_path, 'w', 0o644): + pass + os.chmod(self.__file_path, 0o644) + self.__check_mysqld() +@@ -61,7 +63,8 @@ class Mysql(Configurator): + return False, ind + + def _set(self, key, value): +- with open(self.__file_path, 'r') as f: ++ self._init_file() ++ with open(self.__file_path, 'r', 0o400) as f: + lines = f.readlines() + + key_exist, ind = self.__check_file_exists(lines, key) +@@ -71,7 +74,7 @@ class Mysql(Configurator): + else: + lines[ind] = new_line + +- with open(self.__file_path, 'w') as f: ++ with open(self.__file_path, 'w', 0o644) as f: + f.writelines(lines) + return 0 + +diff --git a/atune_collector/plugin/configurator/nginx/nginx.py b/atune_collector/plugin/configurator/nginx/nginx.py +index 7ab619d..9d38dd0 100644 +--- a/atune_collector/plugin/configurator/nginx/nginx.py ++++ b/atune_collector/plugin/configurator/nginx/nginx.py +@@ -42,18 +42,18 @@ class Nginx(Configurator): + os.mkdir(self.__file_dir) + os.chmod(self.__file_dir, 0o755) + if not os.path.isfile(self.__file_path): +- with open(self.__file_path, 'w'): ++ with open(self.__file_path, 'w', 0o644): + pass + os.chmod(self.__file_path, 0o644) + self._set_index() + + def _get_lines(self): +- with open(self.__file_path, 'r') as f: ++ with open(self.__file_path, 'r', 0o444) as f: + lines = f.readlines() + return lines + + def _set_lines(self, lines): +- with open(self.__file_path, 'w') as f: ++ with open(self.__file_path, 'w', 0o644) as f: + f.writelines(lines) + + def _set_index(self): +@@ -209,3 +209,4 @@ def rewrite_value(value): + raise SetConfigError("Failed to get cpu number") + return output.stdout.decode().count("\n") + ++ +diff --git a/atune_collector/plugin/configurator/redis/redis.py b/atune_collector/plugin/configurator/redis/redis.py +index c56f38e..aa14bcd 100644 +--- a/atune_collector/plugin/configurator/redis/redis.py ++++ b/atune_collector/plugin/configurator/redis/redis.py +@@ -34,15 +34,18 @@ class Redis(Configurator): + self.__re = r"^#\?\s*{key}\s* " + self.__file_dir = "/etc/redis/" + self.__file_path = self.__file_dir + "redis.conf" ++ ++ def _init_file(self): + if not os.path.isdir(self.__file_dir): + os.mkdir(self.__file_dir) + os.chmod(self.__file_dir, 0o755) + if not os.path.isfile(self.__file_path): +- with open(self.__file_path, 'w', 0o200): ++ with open(self.__file_path, 'w', 0o600): + pass + os.chmod(self.__file_path, 0o644) + + def _set(self, key, value): ++ self._init_file() + re_cmd = self.__re.format(key=key) + grep_cmd = [r"grep", re_cmd, self.__file_path] + out, err = self.execute_cmd(grep_cmd) +@@ -51,7 +54,7 @@ class Redis(Configurator): + num_lines = out.count("\n") + new_line = self.__lines.format(key=key, value=value) + if num_lines == 0: +- with open(self.__file_path, 'a', 0o600) as f: ++ with open(self.__file_path, 'a', 0o644) as f: + f.write(new_line + '\n') + elif num_lines == 1: + sed_cmd = [r"sed", "-i", r"s/{}.*$/{}/g".format(re_cmd, new_line), self.__file_path] +-- +2.27.0 + diff --git a/change-import-behavior.patch b/change-import-behavior.patch new file mode 100644 index 0000000000000000000000000000000000000000..af2dd988a5a5c5611392b82c855d1360fd035407 --- /dev/null +++ b/change-import-behavior.patch @@ -0,0 +1,355 @@ +From c02c86fbd6927a3d1968d5a1e3030058bbb943a3 Mon Sep 17 00:00:00 2001 +From: gaoruoshu +Date: Sat, 15 Jul 2023 15:50:06 +0800 +Subject: [PATCH 11/11] change import behavior + +--- + .../plugin/configurator/__init__.py | 47 +++++++++++-------- + .../plugin/configurator/affinity/__init__.py | 2 +- + .../plugin/configurator/bios/__init__.py | 2 +- + .../configurator/bootloader/__init__.py | 2 +- + atune_collector/plugin/configurator/common.py | 2 +- + .../configurator/file_config/__init__.py | 2 +- + .../configurator/kernel_config/__init__.py | 2 +- + .../plugin/configurator/mysql/__init__.py | 3 +- + .../plugin/configurator/nginx/__init__.py | 3 +- + .../plugin/configurator/redis/__init__.py | 3 +- + .../plugin/configurator/script/__init__.py | 2 +- + .../plugin/configurator/sysctl/__init__.py | 2 +- + .../plugin/configurator/sysfs/__init__.py | 2 +- + .../plugin/configurator/systemctl/__init__.py | 2 +- + .../plugin/configurator/ulimit/__init__.py | 2 +- + atune_collector/plugin/monitor/__init__.py | 11 ++++- + .../plugin/monitor/memory/__init__.py | 2 +- + .../plugin/monitor/network/__init__.py | 2 +- + .../plugin/monitor/performance/__init__.py | 2 +- + .../plugin/monitor/process/__init__.py | 2 +- + .../plugin/monitor/processor/__init__.py | 2 +- + .../plugin/monitor/storage/__init__.py | 2 +- + .../plugin/monitor/system/__init__.py | 2 +- + atune_collector/plugin/plugin.py | 4 +- + 24 files changed, 60 insertions(+), 47 deletions(-) + +diff --git a/atune_collector/plugin/configurator/__init__.py b/atune_collector/plugin/configurator/__init__.py +index 31520f1..fd7ea63 100755 +--- a/atune_collector/plugin/configurator/__init__.py ++++ b/atune_collector/plugin/configurator/__init__.py +@@ -17,24 +17,33 @@ Init file. + import sys + import os + +-sys.path.append(os.path.dirname(__file__)) ++from .affinity import * ++from .bios import * ++from .bootloader import * ++from .file_config import * ++from .kernel_config import * ++from .script import * ++from .sysctl import * ++from .sysfs import * ++from .systemctl import * ++from .ulimit import * ++from .mysql import * ++from .redis import * ++from .nginx import * + +-__all__ = [ +- "exceptions", +- "affinity", +- "bios", +- "bootloader", +- "file_config", +- "kernel_config", +- "script", +- "sysctl", +- "sysfs", +- "systemctl", +- "ulimit", +- "mysql", +- "redis", +- "nginx", +- "common"] +- +-from . import * ++__all__ = ["exceptions", ++ "affinity", ++ "bios", ++ "bootloader", ++ "file_config", ++ "kernel_config", ++ "script", ++ "sysctl", ++ "sysfs", ++ "systemctl", ++ "ulimit", ++ "mysql", ++ "redis", ++ "nginx", ++ "common"] + +diff --git a/atune_collector/plugin/configurator/affinity/__init__.py b/atune_collector/plugin/configurator/affinity/__init__.py +index e29377f..1610cc3 100755 +--- a/atune_collector/plugin/configurator/affinity/__init__.py ++++ b/atune_collector/plugin/configurator/affinity/__init__.py +@@ -19,6 +19,6 @@ import os + + __all__ = ["irq", "task", "processid"] + +-from . import * ++from . import irq, task, processid + + sys.path.insert(0, os.path.dirname(__file__)) +diff --git a/atune_collector/plugin/configurator/bios/__init__.py b/atune_collector/plugin/configurator/bios/__init__.py +index 1e193cf..1f9df24 100755 +--- a/atune_collector/plugin/configurator/bios/__init__.py ++++ b/atune_collector/plugin/configurator/bios/__init__.py +@@ -17,4 +17,4 @@ Init file. + + __all__ = ["bios"] + +-from . import * ++from . import bios +diff --git a/atune_collector/plugin/configurator/bootloader/__init__.py b/atune_collector/plugin/configurator/bootloader/__init__.py +index 2412c54..d6cf156 100755 +--- a/atune_collector/plugin/configurator/bootloader/__init__.py ++++ b/atune_collector/plugin/configurator/bootloader/__init__.py +@@ -20,6 +20,6 @@ import os + + __all__ = ["grub2", "cmdline"] + +-from . import * ++from . import grub2, cmdline + + sys.path.insert(0, os.path.dirname(__file__)) +diff --git a/atune_collector/plugin/configurator/common.py b/atune_collector/plugin/configurator/common.py +index 0fbc2f3..4ab459a 100755 +--- a/atune_collector/plugin/configurator/common.py ++++ b/atune_collector/plugin/configurator/common.py +@@ -20,7 +20,7 @@ import logging + import json + from functools import wraps + +-from exceptions import SetConfigError ++from .exceptions import SetConfigError + + LOGGER = logging.getLogger(__name__) + +diff --git a/atune_collector/plugin/configurator/file_config/__init__.py b/atune_collector/plugin/configurator/file_config/__init__.py +index 48aeb30..023de44 100755 +--- a/atune_collector/plugin/configurator/file_config/__init__.py ++++ b/atune_collector/plugin/configurator/file_config/__init__.py +@@ -17,4 +17,4 @@ Init file. + + __all__ = ["fconfig"] + +-from . import * ++from . import fconfig +diff --git a/atune_collector/plugin/configurator/kernel_config/__init__.py b/atune_collector/plugin/configurator/kernel_config/__init__.py +index 5905327..58dd2b0 100755 +--- a/atune_collector/plugin/configurator/kernel_config/__init__.py ++++ b/atune_collector/plugin/configurator/kernel_config/__init__.py +@@ -17,4 +17,4 @@ Init file. + + __all__ = ["kconfig"] + +-from . import * ++from . import kconfig +diff --git a/atune_collector/plugin/configurator/mysql/__init__.py b/atune_collector/plugin/configurator/mysql/__init__.py +index 52bb8c3..bfdb807 100644 +--- a/atune_collector/plugin/configurator/mysql/__init__.py ++++ b/atune_collector/plugin/configurator/mysql/__init__.py +@@ -17,5 +17,4 @@ Init file. + + __all__ = ["mysql"] + +-from . import * +- ++from . import mysql +diff --git a/atune_collector/plugin/configurator/nginx/__init__.py b/atune_collector/plugin/configurator/nginx/__init__.py +index e28a310..07c82b0 100644 +--- a/atune_collector/plugin/configurator/nginx/__init__.py ++++ b/atune_collector/plugin/configurator/nginx/__init__.py +@@ -17,5 +17,4 @@ Init file. + + __all__ = ["nginx"] + +-from . import * +- ++from . import nginx +diff --git a/atune_collector/plugin/configurator/redis/__init__.py b/atune_collector/plugin/configurator/redis/__init__.py +index d386f1f..7d21825 100644 +--- a/atune_collector/plugin/configurator/redis/__init__.py ++++ b/atune_collector/plugin/configurator/redis/__init__.py +@@ -17,5 +17,4 @@ Init file. + + __all__ = ["redis"] + +-from . import * +- ++from . import redis +diff --git a/atune_collector/plugin/configurator/script/__init__.py b/atune_collector/plugin/configurator/script/__init__.py +index f743a20..ec6aa17 100755 +--- a/atune_collector/plugin/configurator/script/__init__.py ++++ b/atune_collector/plugin/configurator/script/__init__.py +@@ -17,4 +17,4 @@ Init file. + + __all__ = ["script"] + +-from . import * ++from . import script +diff --git a/atune_collector/plugin/configurator/sysctl/__init__.py b/atune_collector/plugin/configurator/sysctl/__init__.py +index d6b3f61..3d28df7 100755 +--- a/atune_collector/plugin/configurator/sysctl/__init__.py ++++ b/atune_collector/plugin/configurator/sysctl/__init__.py +@@ -17,4 +17,4 @@ Init file. + + __all__ = ["sysctl"] + +-from . import * ++from . import sysctl +diff --git a/atune_collector/plugin/configurator/sysfs/__init__.py b/atune_collector/plugin/configurator/sysfs/__init__.py +index d220f40..9d5afb6 100755 +--- a/atune_collector/plugin/configurator/sysfs/__init__.py ++++ b/atune_collector/plugin/configurator/sysfs/__init__.py +@@ -17,4 +17,4 @@ Init file. + + __all__ = ["sysfs"] + +-from . import * ++from . import sysfs +diff --git a/atune_collector/plugin/configurator/systemctl/__init__.py b/atune_collector/plugin/configurator/systemctl/__init__.py +index 94762b6..c51a07f 100755 +--- a/atune_collector/plugin/configurator/systemctl/__init__.py ++++ b/atune_collector/plugin/configurator/systemctl/__init__.py +@@ -17,4 +17,4 @@ Init file. + + __all__ = ["systemctl"] + +-from . import * ++from . import systemctl +diff --git a/atune_collector/plugin/configurator/ulimit/__init__.py b/atune_collector/plugin/configurator/ulimit/__init__.py +index 2582b7c..c28a601 100755 +--- a/atune_collector/plugin/configurator/ulimit/__init__.py ++++ b/atune_collector/plugin/configurator/ulimit/__init__.py +@@ -17,4 +17,4 @@ Init file. + + __all__ = ["ulimit"] + +-from . import * ++from . import ulimit +diff --git a/atune_collector/plugin/monitor/__init__.py b/atune_collector/plugin/monitor/__init__.py +index 9292071..948f869 100755 +--- a/atune_collector/plugin/monitor/__init__.py ++++ b/atune_collector/plugin/monitor/__init__.py +@@ -15,6 +15,15 @@ + Init file. + """ + ++from .memory import * ++from .network import * ++from .performance import * ++from .process import * ++from .processor import * ++from .storage import * ++from .system import * ++from . import common ++ + __all__ = [ + "memory", + "network", +@@ -24,5 +33,3 @@ __all__ = [ + "storage", + "common", + "system"] +- +-from . import * +diff --git a/atune_collector/plugin/monitor/memory/__init__.py b/atune_collector/plugin/monitor/memory/__init__.py +index 0a5a356..5002d6e 100755 +--- a/atune_collector/plugin/monitor/memory/__init__.py ++++ b/atune_collector/plugin/monitor/memory/__init__.py +@@ -17,4 +17,4 @@ Init file. + + __all__ = ["numainfo", "topo", "bandwidth", "vmstat", "utilstat", "meminfo"] + +-from . import * ++from . import numainfo, topo, bandwidth, vmstat, utilstat, meminfo +diff --git a/atune_collector/plugin/monitor/network/__init__.py b/atune_collector/plugin/monitor/network/__init__.py +index 5d40ab8..388611a 100755 +--- a/atune_collector/plugin/monitor/network/__init__.py ++++ b/atune_collector/plugin/monitor/network/__init__.py +@@ -17,4 +17,4 @@ Init file. + + __all__ = ["info", "netstat", "netestat", "topo"] + +-from . import * ++from . import info, netstat, netestat, topo +diff --git a/atune_collector/plugin/monitor/performance/__init__.py b/atune_collector/plugin/monitor/performance/__init__.py +index 707f911..93fc6c4 100755 +--- a/atune_collector/plugin/monitor/performance/__init__.py ++++ b/atune_collector/plugin/monitor/performance/__init__.py +@@ -17,4 +17,4 @@ Init file. + + __all__ = ["stat", "top"] + +-from . import * ++from . import stat, top +diff --git a/atune_collector/plugin/monitor/process/__init__.py b/atune_collector/plugin/monitor/process/__init__.py +index 4c4ceb3..5a34c10 100644 +--- a/atune_collector/plugin/monitor/process/__init__.py ++++ b/atune_collector/plugin/monitor/process/__init__.py +@@ -17,4 +17,4 @@ Init file. + + __all__ = ["sched"] + +-from . import * +\ No newline at end of file ++from . import sched +diff --git a/atune_collector/plugin/monitor/processor/__init__.py b/atune_collector/plugin/monitor/processor/__init__.py +index fc88e1b..f46d8fd 100755 +--- a/atune_collector/plugin/monitor/processor/__init__.py ++++ b/atune_collector/plugin/monitor/processor/__init__.py +@@ -17,4 +17,4 @@ Init file. + + __all__ = ["info", "stat", "topo"] + +-from . import * ++from . import info, stat, topo +diff --git a/atune_collector/plugin/monitor/storage/__init__.py b/atune_collector/plugin/monitor/storage/__init__.py +index 670294d..9e5e152 100755 +--- a/atune_collector/plugin/monitor/storage/__init__.py ++++ b/atune_collector/plugin/monitor/storage/__init__.py +@@ -17,4 +17,4 @@ Init file. + + __all__ = ["iostat", "topo"] + +-from . import * ++from . import iostat, topo +diff --git a/atune_collector/plugin/monitor/system/__init__.py b/atune_collector/plugin/monitor/system/__init__.py +index 5c6a00c..f66c1ef 100755 +--- a/atune_collector/plugin/monitor/system/__init__.py ++++ b/atune_collector/plugin/monitor/system/__init__.py +@@ -17,4 +17,4 @@ The import content of the package. + + __all__ = ["bios", "ldavg", "tasks", "filed", "interrupts"] + +-from . import * ++from . import bios, ldavg, tasks, filed, interrupts +diff --git a/atune_collector/plugin/plugin.py b/atune_collector/plugin/plugin.py +index b83d3aa..3f84f2f 100755 +--- a/atune_collector/plugin/plugin.py ++++ b/atune_collector/plugin/plugin.py +@@ -19,9 +19,9 @@ import logging + import threading + import time + +-from configurator.common import Configurator ++from .configurator.common import Configurator + +-from monitor.common import Monitor ++from .monitor.common import Monitor + + LOGGER = logging.getLogger(__name__) + +-- +2.27.0 + diff --git a/feature-set-mysql.patch b/feature-set-mysql.patch new file mode 100644 index 0000000000000000000000000000000000000000..41cac5cae9e6b3e3ef86175760138d768abea08c --- /dev/null +++ b/feature-set-mysql.patch @@ -0,0 +1,145 @@ +From 16ef5dc208cd79b9f9d4c7c9d66ce46dc9d013a1 Mon Sep 17 00:00:00 2001 +From: gaoruoshu +Date: Tue, 11 Jul 2023 20:53:28 +0800 +Subject: [PATCH 07/11] feature: set mysql + +--- + .../plugin/configurator/__init__.py | 1 + + .../plugin/configurator/mysql/__init__.py | 21 +++++ + .../plugin/configurator/mysql/mysql.py | 84 +++++++++++++++++++ + 3 files changed, 106 insertions(+) + create mode 100644 atune_collector/plugin/configurator/mysql/__init__.py + create mode 100644 atune_collector/plugin/configurator/mysql/mysql.py + +diff --git a/atune_collector/plugin/configurator/__init__.py b/atune_collector/plugin/configurator/__init__.py +index 29c674a..677d208 100755 +--- a/atune_collector/plugin/configurator/__init__.py ++++ b/atune_collector/plugin/configurator/__init__.py +@@ -31,6 +31,7 @@ __all__ = [ + "sysfs", + "systemctl", + "ulimit", ++ "mysql", + "common"] + + from . import * +diff --git a/atune_collector/plugin/configurator/mysql/__init__.py b/atune_collector/plugin/configurator/mysql/__init__.py +new file mode 100644 +index 0000000..52bb8c3 +--- /dev/null ++++ b/atune_collector/plugin/configurator/mysql/__init__.py +@@ -0,0 +1,21 @@ ++#!/usr/bin/python3 ++# -*- coding: utf-8 -*- ++# Copyright (c) 2023 Huawei Technologies Co., Ltd. ++# A-Tune is licensed under the Mulan PSL v2. ++# You can use this software according to the terms and conditions of the Mulan PSL v2. ++# You may obtain a copy of Mulan PSL v2 at: ++# http://license.coscl.org.cn/MulanPSL2 ++# THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR ++# IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR ++# PURPOSE. ++# See the Mulan PSL v2 for more details. ++# Create: 2023-07-11 ++ ++""" ++Init file. ++""" ++ ++__all__ = ["mysql"] ++ ++from . import * ++ +diff --git a/atune_collector/plugin/configurator/mysql/mysql.py b/atune_collector/plugin/configurator/mysql/mysql.py +new file mode 100644 +index 0000000..cfc533d +--- /dev/null ++++ b/atune_collector/plugin/configurator/mysql/mysql.py +@@ -0,0 +1,84 @@ ++#!/usr/bin/python3 ++# -*- coding: utf-8 -*- ++# Copyright (c) 2023 Huawei Technologies Co., Ltd. ++# A-Tune is licensed under the Mulan PSL v2. ++# You can use this software according to the terms and conditions of the Mulan PSL v2. ++# You may obtain a copy of Mulan PSL v2 at: ++# http://license.coscl.org.cn/MulanPSL2 ++# THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR ++# IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR ++# PURPOSE. ++# See the Mulan PSL v2 for more details. ++# Create: 2023-07-11 ++ ++""" ++The sub class of the Configurator, used to change the /etc/my.cnf config. ++""" ++ ++ ++import logging ++import os ++from ..common import Configurator ++ ++LOGGER = logging.getLogger(__name__) ++ ++class Mysql(Configurator): ++ _module = "MYSQL" ++ _submod = "MYSQL" ++ ++ def __init__(self, user=None): ++ Configurator.__init__(self, user) ++ self.__cmd = "" ++ self.__file_path = "/etc/my.cnf" ++ self.__mysqld_ind = -1 ++ if not os.path.isfile(self.__file_path): ++ with open(self.__file_path, 'w'): ++ pass ++ os.chmod(self.__file_path, 0o644) ++ self.__check_mysqld() ++ ++ def __check_mysqld(self): ++ with open(self.__file_path, 'r') as f: ++ lines = f.readlines() ++ ++ for line in lines: ++ self.__mysqld_ind = self.__mysqld_ind + 1 ++ if line[:-1] == '[mysqld]': ++ self.__mysqld_ind = self.__mysqld_ind + 1 ++ return ++ ++ lines.insert(self.__mysqld_ind, "[mysqld]\n") ++ self.__mysqld_ind = self.__mysqld_ind + 1 ++ with open(self.__file_path, 'w') as f: ++ f.writelines(lines) ++ ++ def __check_file_exists(self, lines, start_with): ++ ind = -1 ++ for line in lines: ++ ind = ind + 1 ++ if line.split('=')[0].rstrip() == start_with: ++ return True, ind ++ return False, ind ++ ++ def _set(self, key, value): ++ with open(self.__file_path, 'r') as f: ++ lines = f.readlines() ++ ++ key_exist, ind = self.__check_file_exists(lines, key) ++ new_line = key + " = " + value + "\n" ++ if not key_exist: ++ lines.insert(self.__mysqld_ind, new_line) ++ else: ++ lines[ind] = new_line ++ ++ with open(self.__file_path, 'w') as f: ++ f.writelines(lines) ++ return 0 ++ ++ def _get(self, key, _): ++ pass ++ ++ @staticmethod ++ def check(config1, config2): ++ return True ++ +-- +2.27.0 + diff --git a/feature-set-nginx.patch b/feature-set-nginx.patch new file mode 100644 index 0000000000000000000000000000000000000000..2cf24749df3a6a6a0edb1c4e2972a5229c352041 --- /dev/null +++ b/feature-set-nginx.patch @@ -0,0 +1,272 @@ +From b723f4cc584e6df195fbd8b8d0213a3b53aeceea Mon Sep 17 00:00:00 2001 +From: gaoruoshu +Date: Fri, 14 Jul 2023 16:50:53 +0800 +Subject: [PATCH 09/11] feature: set nginx + +--- + .../plugin/configurator/__init__.py | 1 + + .../plugin/configurator/nginx/__init__.py | 21 ++ + .../plugin/configurator/nginx/nginx.py | 211 ++++++++++++++++++ + 3 files changed, 233 insertions(+) + create mode 100644 atune_collector/plugin/configurator/nginx/__init__.py + create mode 100644 atune_collector/plugin/configurator/nginx/nginx.py + +diff --git a/atune_collector/plugin/configurator/__init__.py b/atune_collector/plugin/configurator/__init__.py +index 9597692..31520f1 100755 +--- a/atune_collector/plugin/configurator/__init__.py ++++ b/atune_collector/plugin/configurator/__init__.py +@@ -33,6 +33,7 @@ __all__ = [ + "ulimit", + "mysql", + "redis", ++ "nginx", + "common"] + + from . import * +diff --git a/atune_collector/plugin/configurator/nginx/__init__.py b/atune_collector/plugin/configurator/nginx/__init__.py +new file mode 100644 +index 0000000..e28a310 +--- /dev/null ++++ b/atune_collector/plugin/configurator/nginx/__init__.py +@@ -0,0 +1,21 @@ ++#!/usr/bin/python3 ++# -*- coding: utf-8 -*- ++# Copyright (c) 2019 Huawei Technologies Co., Ltd. ++# A-Tune is licensed under the Mulan PSL v2. ++# You can use this software according to the terms and conditions of the Mulan PSL v2. ++# You may obtain a copy of Mulan PSL v2 at: ++# http://license.coscl.org.cn/MulanPSL2 ++# THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR ++# IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR ++# PURPOSE. ++# See the Mulan PSL v2 for more details. ++# Create: 2023-07-14 ++ ++""" ++Init file. ++""" ++ ++__all__ = ["nginx"] ++ ++from . import * ++ +diff --git a/atune_collector/plugin/configurator/nginx/nginx.py b/atune_collector/plugin/configurator/nginx/nginx.py +new file mode 100644 +index 0000000..7ab619d +--- /dev/null ++++ b/atune_collector/plugin/configurator/nginx/nginx.py +@@ -0,0 +1,211 @@ ++#!/usr/bin/python3 ++# -*- coding: utf-8 -*- ++# Copyright (c) 2019 Huawei Technologies Co., Ltd. ++# A-Tune is licensed under the Mulan PSL v2. ++# You can use this software according to the terms and conditions of the Mulan PSL v2. ++# You may obtain a copy of Mulan PSL v2 at: ++# http://license.coscl.org.cn/MulanPSL2 ++# THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR ++# IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR ++# PURPOSE. ++# See the Mulan PSL v2 for more details. ++# Create: 2023-07-14 ++ ++""" ++The sub class of the Configurator, used to extend script for /etc/nginx/nginx.conf for CPI. ++""" ++ ++import logging ++import os ++import subprocess ++ ++from ..exceptions import SetConfigError ++from ..common import Configurator ++ ++LOGGER = logging.getLogger(__name__) ++ ++ ++class Nginx(Configurator): ++ _module = "NGINX" ++ _submod = "NGINX" ++ ++ def __init__(self, user=None): ++ Configurator.__init__(self, user) ++ self.__file_dir = "/etc/nginx/" ++ self.__file_path = self.__file_dir + "nginx.conf" ++ self.__line = "{key} {value};\n" ++ self.__key_match = r"^([a-zA-Z]+(_[a-zA-Z]+)*.?[a-zA-Z]+)*$" ++ self.__ind = {"main": [0, 0]} ++ ++ def _init_file(self): ++ if not os.path.isdir(self.__file_dir): ++ os.mkdir(self.__file_dir) ++ os.chmod(self.__file_dir, 0o755) ++ if not os.path.isfile(self.__file_path): ++ with open(self.__file_path, 'w'): ++ pass ++ os.chmod(self.__file_path, 0o644) ++ self._set_index() ++ ++ def _get_lines(self): ++ with open(self.__file_path, 'r') as f: ++ lines = f.readlines() ++ return lines ++ ++ def _set_lines(self, lines): ++ with open(self.__file_path, 'w') as f: ++ f.writelines(lines) ++ ++ def _set_index(self): ++ lines = self._get_lines() ++ filo = [] ++ ind = [] ++ filo.append("start") ++ ind.append(0) ++ is_https_server = False ++ for i, line in enumerate(lines): ++ if i == len(lines) - 1: ++ self.__ind["main"] = [ind.pop(), i] ++ del filo, ind ++ if line.strip().startswith("}"): ++ close = filo.pop() ++ close_ind = ind.pop() ++ if close.startswith("events"): ++ self.__ind["events"] = [close_ind, i] ++ if close.startswith("http"): ++ self.__ind["http"] = [close_ind, i] ++ if close.startswith("server"): ++ if is_https_server: ++ self.__ind["https_server"] = [close_ind, i] ++ else: ++ self.__ind["http_server"] = [close_ind, i] ++ is_https_server = False ++ if "{" in line.strip().split("#")[0]: ++ filo.append(line.strip()) ++ ind.append(i) ++ if line.strip().startswith("ssl_"): ++ is_https_server = True ++ ++ def _check_key(self, key): ++ if len(key) == 0: ++ return False, 0 ++ count = 1 ++ if not ('a' <= key[0] <= 'z' or 'A' <= key[0] <= 'Z'): ++ return False, count ++ for i in range(1, len(key)): ++ if key[i] == '_' or key[i] == '.': ++ if not ('a' <= key[i - 1] <= 'z' or 'A' <= key[i - 1] <= 'Z'): ++ return False, count ++ if i == len(key) - 1: ++ return False, count ++ if key[i] == '.': ++ count = count + 1 ++ elif 'a' <= key[i] <= 'z' or 'A' <= key[i] <= 'Z': ++ continue ++ else: ++ return False, count ++ return True, count ++ ++ def _update_index(self, after_ind): ++ for key in self.__ind: ++ for i, val in enumerate(self.__ind[key]): ++ if val > after_ind: ++ self.__ind[key][i] = val + 1 ++ ++ def _append_lines(self, section, key, value): ++ lines = self._get_lines() ++ filo = [] ++ for i in range(self.__ind[section][0], self.__ind[section][1]): ++ curr_line = lines[i].strip() ++ if "{" in curr_line.split("#")[0]: ++ filo.append(lines[i].strip()) ++ elif curr_line.startswith("}"): ++ filo.pop() ++ elif section == 'main' and len(filo) > 0: ++ continue ++ elif len(filo) > 1: ++ continue ++ elif curr_line.startswith(key + " ") or curr_line.startswith(key + "\t"): ++ new_line = lines[i].split(key)[0] ++ new_line = new_line + self.__line.format(key=key, value=value) ++ lines[i] = new_line ++ return lines ++ new_line = "" ++ if section != "main" and not "server" in section: ++ new_line = lines[self.__ind[section][0]].split(section)[0] + " " ++ elif "server" in section: ++ new_line = lines[self.__ind[section][0]].split("server")[0] + " " ++ new_line = new_line + self.__line.format(key=key, value=value) ++ lines.insert(self.__ind[section][0] + 1, new_line) ++ self._update_index(self.__ind[section][0]) ++ return lines ++ ++ def _set_main_section(self, key, value): ++ return self._append_lines("main", key, value) ++ ++ def _set_event_section(self, key, value): ++ if "events" not in self.__ind: ++ raise SetConfigError("No events section") ++ return self._append_lines("events", key, value) ++ ++ def _set_http_section(self, key, value): ++ if "http" not in self.__ind: ++ raise SetConfigError("No http section") ++ return self._append_lines("http", key, value) ++ ++ def _set_http_server(self, key, value): ++ if "http_server" not in self.__ind: ++ raise SetConfigError("No http server in http section") ++ return self._append_lines("http_server", key, value) ++ ++ def _set_https_server(self, key, value): ++ if "https_server" not in self.__ind: ++ raise SetConfigError("No https server in http section") ++ return self._append_lines("https_server", key, value) ++ ++ def _set(self, key, value): ++ self._init_file() ++ valid, count = self._check_key(key) ++ if not valid or count < 1 or count > 3: ++ raise SetConfigError("Invalid value {}".format(key)) ++ if value == "!CPU_CORE!": ++ value = rewrite_value(value) ++ lines = [] ++ try: ++ if count == 1: ++ lines = self._set_main_section(key, value) ++ elif count == 2 and key.split('.')[0] == "events": ++ lines = self._set_event_section(key.split('.')[1], value) ++ elif key.split('.')[0] == "http": ++ if count == 2: ++ lines = self._set_http_section(key.split('.')[1], value) ++ elif count == 3: ++ if key.split('.')[1] == "http": ++ lines = self._set_http_server(key.split('.')[2], value) ++ elif key.split('.')[1] == "https": ++ lines = self._set_https_server(key.split('.')[2], value) ++ else: ++ raise SetConfigError("Invalid value {}".format(key)) ++ else: ++ raise SetConfigError("Invalid value {}".format(key)) ++ else: ++ raise SetConfigError("Invalid value {}".format(key)) ++ except SetConfigError as err: ++ raise err ++ self._set_lines(lines) ++ return 0 ++ ++ def _get(self, key, _): ++ pass ++ ++ @staticmethod ++ def check(config1, config2): ++ return True ++ ++def rewrite_value(value): ++ command = ["grep", "processor", "/proc/cpuinfo"] ++ output = subprocess.run(command, shell=False, stdout=subprocess.PIPE, stderr=subprocess.PIPE) ++ if output.returncode != 0: ++ raise SetConfigError("Failed to get cpu number") ++ return output.stdout.decode().count("\n") ++ +-- +2.27.0 + diff --git a/feature-set-redis-value.patch b/feature-set-redis-value.patch new file mode 100644 index 0000000000000000000000000000000000000000..421a46a4f02da9689216510570a320a13ef6b031 --- /dev/null +++ b/feature-set-redis-value.patch @@ -0,0 +1,139 @@ +From 70a1f8bc884b6a1b2772fd7199e103fa5351ca3d Mon Sep 17 00:00:00 2001 +From: gaoruoshu +Date: Thu, 13 Jul 2023 14:40:57 +0800 +Subject: [PATCH 08/11] feature: set redis value + +--- + .../plugin/configurator/__init__.py | 2 + + .../plugin/configurator/redis/__init__.py | 21 +++++ + .../plugin/configurator/redis/redis.py | 77 +++++++++++++++++++ + 3 files changed, 100 insertions(+) + create mode 100644 atune_collector/plugin/configurator/redis/__init__.py + create mode 100644 atune_collector/plugin/configurator/redis/redis.py + +diff --git a/atune_collector/plugin/configurator/__init__.py b/atune_collector/plugin/configurator/__init__.py +index 677d208..9597692 100755 +--- a/atune_collector/plugin/configurator/__init__.py ++++ b/atune_collector/plugin/configurator/__init__.py +@@ -32,6 +32,8 @@ __all__ = [ + "systemctl", + "ulimit", + "mysql", ++ "redis", + "common"] + + from . import * ++ +diff --git a/atune_collector/plugin/configurator/redis/__init__.py b/atune_collector/plugin/configurator/redis/__init__.py +new file mode 100644 +index 0000000..d386f1f +--- /dev/null ++++ b/atune_collector/plugin/configurator/redis/__init__.py +@@ -0,0 +1,21 @@ ++#!/usr/bin/python3 ++# -*- coding: utf-8 -*- ++# Copyright (c) 2023 Huawei Technologies Co., Ltd. ++# A-Tune is licensed under the Mulan PSL v2. ++# You can use this software according to the terms and conditions of the Mulan PSL v2. ++# You may obtain a copy of Mulan PSL v2 at: ++# http://license.coscl.org.cn/MulanPSL2 ++# THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR ++# IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR ++# PURPOSE. ++# See the Mulan PSL v2 for more details. ++# Create: 2023-07-13 ++ ++""" ++Init file. ++""" ++ ++__all__ = ["redis"] ++ ++from . import * ++ +diff --git a/atune_collector/plugin/configurator/redis/redis.py b/atune_collector/plugin/configurator/redis/redis.py +new file mode 100644 +index 0000000..c56f38e +--- /dev/null ++++ b/atune_collector/plugin/configurator/redis/redis.py +@@ -0,0 +1,77 @@ ++#!/usr/bin/python3 ++# -*- coding: utf-8 -*- ++# Copyright (c) 2023 Huawei Technologies Co., Ltd. ++# A-Tune is licensed under the Mulan PSL v2. ++# You can use this software according to the terms and conditions of the Mulan PSL v2. ++# You may obtain a copy of Mulan PSL v2 at: ++# http://license.coscl.org.cn/MulanPSL2 ++# THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR ++# IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR ++# PURPOSE. ++# See the Mulan PSL v2 for more details. ++# Create: 2023-07-13 ++ ++""" ++The sub class of the Configurator, used to change the redis config. ++""" ++ ++import logging ++import os ++import subprocess ++from ..exceptions import GetConfigError, SetConfigError ++from ..common import Configurator ++ ++LOGGER = logging.getLogger(__name__) ++ ++ ++class Redis(Configurator): ++ _module = "REDIS" ++ _submod = "REDIS" ++ ++ def __init__(self, user=None): ++ Configurator.__init__(self, user) ++ self.__lines = "{key} {value}" ++ self.__re = r"^#\?\s*{key}\s* " ++ self.__file_dir = "/etc/redis/" ++ self.__file_path = self.__file_dir + "redis.conf" ++ if not os.path.isdir(self.__file_dir): ++ os.mkdir(self.__file_dir) ++ os.chmod(self.__file_dir, 0o755) ++ if not os.path.isfile(self.__file_path): ++ with open(self.__file_path, 'w', 0o200): ++ pass ++ os.chmod(self.__file_path, 0o644) ++ ++ def _set(self, key, value): ++ re_cmd = self.__re.format(key=key) ++ grep_cmd = [r"grep", re_cmd, self.__file_path] ++ out, err = self.execute_cmd(grep_cmd) ++ if len(err) != 0: ++ raise SetConfigError("Failed to set {}: {}".format(key, err)) ++ num_lines = out.count("\n") ++ new_line = self.__lines.format(key=key, value=value) ++ if num_lines == 0: ++ with open(self.__file_path, 'a', 0o600) as f: ++ f.write(new_line + '\n') ++ elif num_lines == 1: ++ sed_cmd = [r"sed", "-i", r"s/{}.*$/{}/g".format(re_cmd, new_line), self.__file_path] ++ _, err_rep = self.execute_cmd(sed_cmd) ++ if len(err_rep) > 0: ++ raise SetConfigError("Failed to set {}: {}".format(key, err_rep)) ++ else: ++ raise SetConfigError("Failed to set {}: more than 1 key has same name".format(key)) ++ return 0 ++ ++ def _get(self, key, _): ++ pass ++ ++ @staticmethod ++ def check(config1, config2): ++ return True ++ ++ @staticmethod ++ def execute_cmd(cmd): ++ output = subprocess.run(cmd, shell=False, stdout=subprocess.PIPE, stderr=subprocess.PIPE) ++ return output.stdout.decode(), output.stderr.decode()[:-1] ++ ++ +-- +2.27.0 + diff --git a/fix-CWE-23.patch b/fix-CWE-23.patch new file mode 100644 index 0000000000000000000000000000000000000000..4afb3b4aa943af49f73e63414ee4316208fef3c6 --- /dev/null +++ b/fix-CWE-23.patch @@ -0,0 +1,33 @@ +From 121a1bcbd68ef9b18ec0c0cdcc8ca0748fe08bdd Mon Sep 17 00:00:00 2001 +From: unknown +Date: Wed, 26 Apr 2023 21:44:16 +0800 +Subject: [PATCH 05/11] fix CWE-23 + +collect_data.py直接使用命令行参数作为文件路径，可以被攻击者输入../访问上级目录，从而获取系统的敏感信息，或者写入任意文件（使用保存路径遍历文件名的恶意zip存档）。为了修复这一漏洞，我们采用了werzeug库中的secure_filename函数，这一函数会过滤掉文件路径中的所有危险字符，防范这一攻击方式。 +--- + atune_collector/collect_data.py | 2 ++ + 1 file changed, 2 insertions(+) + +diff --git a/atune_collector/collect_data.py b/atune_collector/collect_data.py +index 3593db6..167141e 100755 +--- a/atune_collector/collect_data.py ++++ b/atune_collector/collect_data.py +@@ -21,6 +21,7 @@ import time + import csv + + from plugin.plugin import MPI ++from werkzeug.utils import secure_filename + + + class Collector: +@@ -112,6 +113,7 @@ if __name__ == "__main__": + ARG_PARSER.add_argument('-c', '--config', metavar='json', + default=default_json_path, help='input json path') + ARGS = ARG_PARSER.parse_args() ++ filename=secure_filename(ARGS.config) + with open(ARGS.config, 'r') as file: + json_data = json.load(file) + collector = Collector(json_data) +-- +2.27.0 + diff --git a/0001-fix-bug-list-index-out-of-range-in-virtualbox.patch b/fix-bug-list-index-out-of-range-in-virtualbox.patch similarity index 92% rename from 0001-fix-bug-list-index-out-of-range-in-virtualbox.patch rename to fix-bug-list-index-out-of-range-in-virtualbox.patch index fb41c6a054aaded75f38e3c725bd0be2de8e24a4..9ed762a0340e0d5aebcc41afa6b90d362c62e49a 100644 --- a/0001-fix-bug-list-index-out-of-range-in-virtualbox.patch +++ b/fix-bug-list-index-out-of-range-in-virtualbox.patch @@ -1,7 +1,7 @@ From 4cb49adb623f36590f254b7e11c8aa5b0777bd00 Mon Sep 17 00:00:00 2001 From: gaoruoshu Date: Tue, 26 Apr 2022 02:01:27 +0000 -Subject: [PATCH] fix bug: list index out of range in virtualbox +Subject: [PATCH 03/11] fix bug: list index out of range in virtualbox --- atune_collector/plugin/monitor/memory/bandwidth.py | 2 +- @@ -21,5 +21,5 @@ index 593a4a8..f44925f 100755 for dimm in info["memorys"][0]["children"]: if dimm.get("size") is None: -- -2.33.0 +2.27.0 diff --git a/memory-bandwidth-dimm-slot-atuned.patch b/memory-bandwidth-dimm-slot-atuned.patch new file mode 100644 index 0000000000000000000000000000000000000000..c9845e0d16ae9aa9e7c09c33300f5a056733b1f9 --- /dev/null +++ b/memory-bandwidth-dimm-slot-atuned.patch @@ -0,0 +1,37 @@ +From 7c21a2da2c931089e4ec2b86ae35a510f2f7793d Mon Sep 17 00:00:00 2001 +From: SWWBF <1152719547@qq.com> +Date: Mon, 10 Jul 2023 05:52:54 +0800 +Subject: [PATCH 06/11] some kvm has no slot which may cause panic of +atune + +--- + atune_collector/plugin/monitor/memory/bandwidth.py | 13 +++++++------ + 1 file changed, 7 insertions(+), 6 deletions(-) + +diff --git a/atune_collector/plugin/monitor/memory/bandwidth.py b/atune_collector/plugin/monitor/memory/bandwidth.py +index f44925f..6d59ab6 100755 +--- a/atune_collector/plugin/monitor/memory/bandwidth.py ++++ b/atune_collector/plugin/monitor/memory/bandwidth.py +@@ -158,12 +158,13 @@ class MemBandwidth(Monitor): + for dimm in info["memorys"][0]["children"]: + if dimm.get("size") is None: + continue +- locator = memtopo.table_get_locator(dimm["slot"]) +- if locator is None: +- continue +- if dimms[locator[0]][locator[1]] == 0: +- dimms[locator[0]][locator[1]] = dimm["width"] * \ +- memtopo.table_get_freq(dimm["description"]) / 8 ++ if "slot" in dimm: ++ locator = memtopo.table_get_locator(dimm["slot"]) ++ if locator is None: ++ continue ++ if dimms[locator[0]][locator[1]] == 0: ++ dimms[locator[0]][locator[1]] = dimm["width"] * \ ++ memtopo.table_get_freq(dimm["description"]) / 8 + ret = 0 + for channel in dimms[socket]: + ret += channel +-- +2.27.0 +