From 3c7a21a632d0d673ea729c57972fe8f762393aa4 Mon Sep 17 00:00:00 2001 From: Ke Li Date: Mon, 14 Apr 2025 16:37:29 +0800 Subject: [PATCH] python lint --- CI/script/gtest_collect_output.py | 80 +- .../logs_handler/do_compress_and_archive.py | 41 +- common/script/logs_handler/logs_handler.py | 55 +- common/script/logs_handler/logs_tool/log.py | 10 +- .../logs_handler/logs_tool/log_config.py | 4 +- .../service/cantian_exporter/config/config.py | 4 +- .../cantian_exporter/exporter/execute.py | 11 +- .../exporter/get_certificate_status.py | 8 +- .../cantian_exporter/exporter/get_info.py | 560 ++-- .../service/cantian_exporter/exporter/log.py | 10 +- .../cantian_exporter/exporter/save_file.py | 16 +- .../service/cantian_exporter/exporter/tool.py | 39 +- .../query_storage_info/get_dr_info.py | 66 +- .../query_storage_info/response_parse.py | 21 +- .../query_storage_info/rest_client.py | 92 +- .../query_storage_info/rest_constant.py | 180 +- ct_om/service/ctcli/display_as_table.py | 18 +- ct_om/service/ctcli/handle_info.py | 112 +- ct_om/service/ctcli/main.py | 2 +- .../ctcli/params_factory/param_config.py | 24 +- ct_om/service/ctcli/params_factory/tools.py | 14 +- ct_om/service/ctcli/uds_client.py | 2 +- .../ctmgr/checker/collection_logs_checker.py | 10 +- ct_om/service/ctmgr/checkers.py | 40 +- ct_om/service/ctmgr/common/common_tool.py | 2 +- ct_om/service/ctmgr/common/output_tool.py | 11 +- ct_om/service/ctmgr/log_tool/om_log.py | 56 +- ct_om/service/ctmgr/log_tool/om_log_config.py | 10 +- .../service/ctmgr/logs_collection/execute.py | 29 +- .../ctmgr/logs_collection/logs_collection.py | 205 +- ct_om/service/ctmgr/logs_collection/tools.py | 22 +- ct_om/service/ctmgr/task.py | 182 +- ct_om/service/ctmgr/task_obj.py | 140 +- ct_om/service/ctmgr/tasks/audit_task.py | 36 +- .../ctmgr/tasks/inspection/inspection_task.py | 150 +- .../service/ctmgr/tasks/log_progress_query.py | 12 +- ct_om/service/ctmgr/tasks/log_query.py | 6 +- .../ctmgr/tasks/logs_collections_task.py | 65 +- ct_om/service/ctmgr/uds_client.py | 29 +- ct_om/service/ctmgr/uds_server.py | 68 +- pkg/admin/scripts/fetch_cls_stat.py | 35 +- pkg/deploy/action/cantian/Common.py | 18 +- pkg/deploy/action/cantian/bind_cpu_config.py | 157 +- pkg/deploy/action/cantian/cantian_funclib.py | 398 +-- pkg/deploy/action/cantian/cantian_install.py | 1916 +++++++++----- .../action/cantian/cantian_post_upgrade.py | 2 +- pkg/deploy/action/cantian/cantian_start.py | 2 +- .../action/cantian/cantian_uninstall.py | 434 ++-- pkg/deploy/action/cantian/exception.py | 2 +- pkg/deploy/action/cantian/get_config_info.py | 90 +- pkg/deploy/action/cantian/log.py | 42 +- pkg/deploy/action/cantian/options.py | 2 +- .../action/cantian_common/crypte_adapter.py | 14 +- pkg/deploy/action/cantian_common/exec_sql.py | 34 +- .../action/cantian_common/get_config_info.py | 2 +- .../action/cantian_common/mysql_shell.py | 58 +- pkg/deploy/action/check_pwd.py | 32 +- pkg/deploy/action/clear_history_version.py | 15 +- pkg/deploy/action/clear_upgrade_backup.py | 18 +- pkg/deploy/action/cms/cms_node0_stop.py | 20 +- pkg/deploy/action/cms/cmsctl.py | 943 ++++--- pkg/deploy/action/cms/get_config_info.py | 9 +- pkg/deploy/action/cms/log.py | 42 +- pkg/deploy/action/dbstor/dbstor_backup.py | 52 +- pkg/deploy/action/dbstor/dbstor_install.py | 401 ++- pkg/deploy/action/dbstor/dbstor_uninstall.py | 55 +- pkg/deploy/action/dbstor/init_unify_config.py | 116 +- pkg/deploy/action/dbstor/kmc_adapter.py | 144 +- .../action/dbstor/update_dbstor_config.py | 210 +- pkg/deploy/action/do_snapshot.py | 50 +- pkg/deploy/action/docker/cantian_numa.py | 127 +- .../action/docker/delete_unready_pod.py | 62 +- .../action/docker/docker_common/file_utils.py | 17 +- .../docker_common/kubernetes_service.py | 58 +- pkg/deploy/action/docker/dr_deploy.py | 127 +- pkg/deploy/action/docker/get_config_info.py | 2 +- pkg/deploy/action/docker/pod_record.py | 25 +- pkg/deploy/action/docker/resolve_pwd.py | 33 +- pkg/deploy/action/docker/update_config.py | 11 +- .../action/docker/update_policy_params.py | 18 +- .../action/docker/upgrade_version_check.py | 48 +- pkg/deploy/action/dss/config.py | 10 +- pkg/deploy/action/dss/dssctl.py | 142 +- pkg/deploy/action/fetch_cls_stat.py | 35 +- pkg/deploy/action/find_next_page.py | 99 +- pkg/deploy/action/get_config_info.py | 16 +- pkg/deploy/action/get_source_version.py | 6 +- .../certificate_update_and_revocation.py | 30 +- .../action/implement/check_deploy_param.py | 23 +- .../check_nfs4_mandatory_lock_switch.py | 8 +- pkg/deploy/action/implement/check_pwd.py | 20 +- pkg/deploy/action/implement/config_opt.py | 18 +- .../action/implement/get_source_version.py | 7 +- .../action/implement/update_cantian_passwd.py | 12 +- .../action/implement/upgrade_version_check.py | 72 +- pkg/deploy/action/inspection/declear_env.py | 4 +- .../inspection/generate_html_results.py | 114 +- .../action/inspection/inspection_cantian.py | 86 +- .../action/inspection/inspection_mysql.py | 32 +- .../inspection_scripts/cms/cms_res_check.py | 47 +- .../inspection_scripts/cms/cms_stat_check.py | 40 +- .../cms/cms_version_check.py | 21 +- .../inspection_scripts/cms/common_func.py | 11 +- .../ct_om/cantian_om_check.py | 128 +- .../ct_om/cantian_om_logicrep_check.py | 73 +- .../ct_om/cantian_om_ntp_check.py | 48 +- .../ct_om/om_common_function.py | 10 +- .../kernal/check_archive_status.py | 66 +- .../kernal/check_db_version.py | 28 +- .../kernal/check_drc_res_ratio.py | 39 +- .../kernal/check_redundant_links.py | 48 +- .../kernal/check_session.py | 58 +- .../kernal/check_transaction.py | 32 +- .../inspection_scripts/kernal/ct_check.py | 310 ++- .../inspection_scripts/kernal/gs_check.py | 354 +-- .../inspection_scripts/mysql/common_func.py | 11 +- .../mysql/mysql_connection_check.py | 24 +- .../mysql/mysql_diskspace_check.py | 37 +- .../mysql/mysql_file_check.py | 52 +- .../mysql_transaction_isolation_check.py | 36 +- .../action/inspection/inspection_task.py | 197 +- pkg/deploy/action/inspection/log_tool.py | 16 +- pkg/deploy/action/logic/common_func.py | 20 +- pkg/deploy/action/logic/storage_operate.py | 141 +- pkg/deploy/action/logicrep/logicrep_ctl.py | 171 +- pkg/deploy/action/modify_env.py | 36 +- pkg/deploy/action/mysql/mysqlctl.py | 12 +- pkg/deploy/action/obtains_lsid.py | 30 +- pkg/deploy/action/om_log.py | 38 +- pkg/deploy/action/om_log_config.py | 4 +- pkg/deploy/action/pre_install.py | 495 ++-- pkg/deploy/action/pre_upgrade.py | 32 +- pkg/deploy/action/rest_client.py | 235 +- .../storage_operate/create_file_system.py | 119 +- .../action/storage_operate/do_snapshot.py | 119 +- .../dr_deploy_operate/dr_deploy.py | 990 ++++--- .../dr_deploy_operate/dr_deploy_common.py | 387 +-- .../dr_deploy_operate/dr_deploy_full_sync.py | 269 +- .../dr_deploy_operate/dr_deploy_pre_check.py | 499 +++- .../dr_deploy_progress_query.py | 176 +- .../dr_deploy_operate/dr_deploy_switchover.py | 285 +- .../dr_deploy_operate/dr_undeploy.py | 156 +- .../dr_deploy_operate/install_mysql.py | 57 +- .../dr_deploy_operate/update_dr_params.py | 24 +- .../storage_operate/dr_operate_interface.py | 38 +- .../storage_operate/migrate_file_system.py | 109 +- .../storage_operate/split_dbstore_fs.py | 65 +- pkg/deploy/action/update_config.py | 82 +- pkg/deploy/action/upgrade_version_check.py | 72 +- .../action/utils/client/response_parse.py | 21 +- pkg/deploy/action/utils/client/rest_client.py | 104 +- pkg/deploy/action/utils/client/ssh_client.py | 76 +- .../action/utils/config/rest_constant.py | 206 +- pkg/deploy/action/write_config.py | 6 +- pkg/install/Common.py | 18 +- pkg/install/funclib.py | 437 ++-- pkg/install/install.py | 2286 +++++++++++------ pkg/install/sql_process.py | 1628 ++++++------ pkg/install/uninstall.py | 355 ++- pkg/test/om_test/python_LLT.py | 14 +- .../test_dr_deploy_pre_check.py | 53 +- 161 files changed, 13173 insertions(+), 7395 deletions(-) diff --git a/CI/script/gtest_collect_output.py b/CI/script/gtest_collect_output.py index 80028bb0..9f378da1 100644 --- a/CI/script/gtest_collect_output.py +++ b/CI/script/gtest_collect_output.py @@ -1,5 +1,5 @@ -#!/usr/bin/env python -#-*- coding: UTF-8 -*- +#!/usr/bin/env python +# -*- coding: UTF-8 -*- import os import sys @@ -15,62 +15,68 @@ G_ERRORS = 0 G_TIME = 0 G_NAME = "AllTests" -if len(sys.argv) > 1 : +if len(sys.argv) > 1: G_OUTPUT_DIR = sys.argv[1] - -if len(sys.argv) > 2 : + +if len(sys.argv) > 2: G_OUTPUT_FILE = sys.argv[2] -if not os.path.isdir(G_OUTPUT_DIR) : - print("ERROR:%s not exist." % (G_OUTPUT_DIR)) +if not os.path.isdir(G_OUTPUT_DIR): + print("ERROR:%s not exist." % (G_OUTPUT_DIR)) exit(1) -#逐个读取gtest_output目录的xml文件，最终汇总成一个文件 -#需要考虑testsuites字段属性数字累加，其他字段直接拼接合并 +# 逐个读取gtest_output目录的xml文件，最终汇总成一个文件 +# 需要考虑testsuites字段属性数字累加，其他字段直接拼接合并 impl = xml.dom.getDOMImplementation() -output_dom = impl.createDocument(None, None, None) +output_dom = impl.createDocument(None, None, None) output_root = output_dom.createElement("testsuites") -for FILE_IDX in os.listdir(G_OUTPUT_DIR) : +for FILE_IDX in os.listdir(G_OUTPUT_DIR): print("analyse %s ......" % FILE_IDX) dom = xml.dom.minidom.parse("%s/%s" % (G_OUTPUT_DIR, FILE_IDX)) root = dom.documentElement - #L_TESTSUITES = root.getElementsByTagName("testsuites")[0] - - L_TEST = root.getAttribute('tests') + # L_TESTSUITES = root.getElementsByTagName("testsuites")[0] + + L_TEST = root.getAttribute("tests") G_TEST += int(L_TEST) - - L_FAILURES = root.getAttribute('failures') + + L_FAILURES = root.getAttribute("failures") G_FAILURES += int(L_FAILURES) - - L_DISABLED = root.getAttribute('disabled') + + L_DISABLED = root.getAttribute("disabled") G_DISABLED += int(L_DISABLED) - - L_ERRORS = root.getAttribute('errors') + + L_ERRORS = root.getAttribute("errors") G_ERRORS += int(L_ERRORS) - - L_TIME = root.getAttribute('time') + + L_TIME = root.getAttribute("time") G_TIME += float(L_TIME) - + L_TESTSUITE = root.getElementsByTagName("testsuite") - for TESTSUITE_IDX in L_TESTSUITE : + for TESTSUITE_IDX in L_TESTSUITE: output_root.appendChild(TESTSUITE_IDX) - - print(" tests=%s, failures=%s, disabled=%s, errors=%s, time=%s" % (L_TEST, L_FAILURES, L_DISABLED, L_ERRORS, L_TIME)) - print("all tests=%d, failures=%d, disabled=%d, errors=%d, time=%0.3f" % (G_TEST, G_FAILURES, G_DISABLED, G_ERRORS, G_TIME)) + + print( + " tests=%s, failures=%s, disabled=%s, errors=%s, time=%s" + % (L_TEST, L_FAILURES, L_DISABLED, L_ERRORS, L_TIME) + ) + print( + "all tests=%d, failures=%d, disabled=%d, errors=%d, time=%0.3f" + % (G_TEST, G_FAILURES, G_DISABLED, G_ERRORS, G_TIME) + ) print() -#写入xml文件 +# 写入xml文件 output_dom.appendChild(output_root) -output_root.setAttribute('tests', str(G_TEST)) -output_root.setAttribute('failures', str(G_FAILURES)) -output_root.setAttribute('disabled', str(G_DISABLED)) -output_root.setAttribute('errors', str(G_ERRORS)) -output_root.setAttribute('time', str(G_TIME)) -output_root.setAttribute('name', str(G_NAME)) +output_root.setAttribute("tests", str(G_TEST)) +output_root.setAttribute("failures", str(G_FAILURES)) +output_root.setAttribute("disabled", str(G_DISABLED)) +output_root.setAttribute("errors", str(G_ERRORS)) +output_root.setAttribute("time", str(G_TIME)) +output_root.setAttribute("name", str(G_NAME)) -output_file = open(G_OUTPUT_FILE, mode = 'w') -output_dom.writexml(output_file, '', ' ', '\n', 'utf-8') -output_file.close() \ No newline at end of file +output_file = open(G_OUTPUT_FILE, mode="w") +output_dom.writexml(output_file, "", " ", "\n", "utf-8") +output_file.close() diff --git a/common/script/logs_handler/do_compress_and_archive.py b/common/script/logs_handler/do_compress_and_archive.py index 0c4fb688..dba3a996 100644 --- a/common/script/logs_handler/do_compress_and_archive.py +++ b/common/script/logs_handler/do_compress_and_archive.py @@ -9,7 +9,7 @@ CUR_PATH, _ = os.path.split(os.path.abspath(__file__)) def file_reader(data_path): - with open(data_path, 'r', encoding='utf-8') as file: + with open(data_path, "r", encoding="utf-8") as file: info = file.read() return json.loads(info) @@ -25,7 +25,7 @@ def compress_bak_files(main_path, file_names): # 切换工作目录，确保仅压缩日志文件本身 cwd = os.getcwd() os.chdir(main_path) - with tarfile.open(f'{ori_log_file}.tar.gz', 'w:gz') as tar: + with tarfile.open(f"{ori_log_file}.tar.gz", "w:gz") as tar: tar.add(file_name) os.chdir(cwd) os.remove(ori_log_file) @@ -33,10 +33,12 @@ def compress_bak_files(main_path, file_names): def reg_handler(log_name_prefix, log_name_tail, log_name, match_string): """匹配出待压缩的文件""" - match_condition = 'tar.gz' in match_string or \ - len(match_string) <= len(log_name) \ - or match_string.endswith("swp") or \ - match_string.endswith("swo") + match_condition = ( + "tar.gz" in match_string + or len(match_string) <= len(log_name) + or match_string.endswith("swp") + or match_string.endswith("swo") + ) if match_condition: return False if log_name_prefix in match_string and log_name_tail in match_string: @@ -48,7 +50,7 @@ def change_file_mod(bak_path, file_names, user_name): """改变日志定时清理进程创建的压缩文件权限""" uid, gid = getpwnam(user_name)[2:4] for file_name, _ in file_names: - tar_file_name = f'{str(Path(bak_path, file_name))}.tar.gz' + tar_file_name = f"{str(Path(bak_path, file_name))}.tar.gz" os.chmod(tar_file_name, 0o660) os.chown(tar_file_name, uid=uid, gid=gid) @@ -69,9 +71,14 @@ def delete_old_bak_logs(main_path, log_name, max_log_val): return # 统计main_path目录下所有以log_name的压缩文件 - tar_files = [(str(Path(main_path, file_item)), get_file_creation_time(str(Path(main_path, file_item)))) - for file_item in os.listdir(main_path) - if file_item.startswith(log_name) and file_item.endswith('tar.gz')] + tar_files = [ + ( + str(Path(main_path, file_item)), + get_file_creation_time(str(Path(main_path, file_item))), + ) + for file_item in os.listdir(main_path) + if file_item.startswith(log_name) and file_item.endswith("tar.gz") + ] tar_files.sort(key=lambda x: x[1], reverse=False) for tar_file, _ in tar_files: @@ -88,18 +95,24 @@ def bak_logs_handler(log_content, log_name, ori_bak_files, max_log_vol, user_nam def main(log_content, log_name, max_log_vol, user_name): - log_name_prefix, log_name_tail = log_name.split('.')[0], log_name.split('.')[1] + log_name_prefix, log_name_tail = log_name.split(".")[0], log_name.split(".")[1] ori_bak_files = [] for name in os.listdir(log_content): if reg_handler(log_name_prefix, log_name_tail, log_name, name): - ori_bak_files.append((str(Path(log_content, f'{name}')), - get_file_creation_time(str(Path(log_content, f'{name}'))))) + ori_bak_files.append( + ( + str(Path(log_content, f"{name}")), + get_file_creation_time(str(Path(log_content, f"{name}"))), + ) + ) if not ori_bak_files: return ori_bak_files.sort(key=lambda x: x[1], reverse=False) - bak_logs_handler(log_content, log_name_prefix, ori_bak_files, max_log_vol, user_name) + bak_logs_handler( + log_content, log_name_prefix, ori_bak_files, max_log_vol, user_name + ) if __name__ == "__main__": diff --git a/common/script/logs_handler/logs_handler.py b/common/script/logs_handler/logs_handler.py index d3deec9c..ea43fada 100644 --- a/common/script/logs_handler/logs_handler.py +++ b/common/script/logs_handler/logs_handler.py @@ -12,13 +12,13 @@ ENV_FILE = "/opt/cantian/action/env.sh" def file_reader(data_path): - with open(data_path, 'r', encoding='utf-8') as file: + with open(data_path, "r", encoding="utf-8") as file: info = file.read() return json.loads(info) def get_param_value(param): - with open(ENV_FILE, 'r', encoding='utf-8') as file: + with open(ENV_FILE, "r", encoding="utf-8") as file: env_config = file.readlines() if param == "deploy_user": for line in env_config: @@ -41,11 +41,11 @@ def close_child_process(proc): os.killpg(proc.pid, signal.SIGTERM) except ProcessLookupError as err: _ = err - return 'success' + return "success" except Exception as err: return str(err) - return 'success' + return "success" def shell_task(exec_cmd): @@ -55,8 +55,14 @@ def shell_task(exec_cmd): return: status code, standard output, error output """ bash_cmd = ["bash"] - pobj = subprocess.Popen(bash_cmd, shell=False, stdin=subprocess.PIPE, - stdout=subprocess.PIPE, stderr=subprocess.PIPE, preexec_fn=os.setsid) + pobj = subprocess.Popen( + bash_cmd, + shell=False, + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + preexec_fn=os.setsid, + ) pobj.stdin.write(exec_cmd.encode()) pobj.stdin.write(os.linesep.encode()) try: @@ -78,16 +84,21 @@ def shell_task(exec_cmd): class LogsHandler: def __init__(self): - self.config_params = file_reader(str(Path(CUR_PATH, 'config.json'))) + self.config_params = file_reader(str(Path(CUR_PATH, "config.json"))) self.deploy_user = get_param_value("deploy_user") self.user_name = None def execute(self): for item in self.config_params: - user = item.get('userandgroup') if \ - item.get('userandgroup') != "deploy_user" else self.deploy_user - self.user_name = user.split(':')[0] - log_file_dir, max_log_vol = item.get('log_file_dir'), int(item.get('max_log_vol')) + user = ( + item.get("userandgroup") + if item.get("userandgroup") != "deploy_user" + else self.deploy_user + ) + self.user_name = user.split(":")[0] + log_file_dir, max_log_vol = item.get("log_file_dir"), int( + item.get("max_log_vol") + ) if os.path.exists("/.dockerenv"): max_log_vol //= 2 # 分离日志目录和日志名 @@ -97,18 +108,26 @@ class LogsHandler: files_names = os.listdir(log_content) log_name_pre = log_name.split(".")[0] for file_name in files_names: - conditions = log_name_pre in file_name and not file_name.endswith("tar.gz") \ - and not file_name.endswith("swp") and not file_name.endswith("swo") \ - and len(file_name) > len(log_name) + conditions = ( + log_name_pre in file_name + and not file_name.endswith("tar.gz") + and not file_name.endswith("swp") + and not file_name.endswith("swo") + and len(file_name) > len(log_name) + ) if conditions: # 判断当前是否有新产生的归档日志，有就退出循环进行打包操作 break else: continue - exec_cmd = f"su - {self.user_name} -s /bin/bash -c 'python3 {CUR_PATH}/do_compress_and_archive.py " \ - f"{log_content} {log_name} {max_log_vol} {self.user_name}' " + exec_cmd = ( + f"su - {self.user_name} -s /bin/bash -c 'python3 {CUR_PATH}/do_compress_and_archive.py " + f"{log_content} {log_name} {max_log_vol} {self.user_name}' " + ) return_code, stdout, stderr = shell_task(exec_cmd) if return_code or stderr: - LOG.error(f'failed to execute log cleanup of {log_content}, ' - f'return_code: {return_code}, stderr: {stderr}') + LOG.error( + f"failed to execute log cleanup of {log_content}, " + f"return_code: {return_code}, stderr: {stderr}" + ) diff --git a/common/script/logs_handler/logs_tool/log.py b/common/script/logs_handler/logs_tool/log.py index c1d5ae75..e5709ec0 100644 --- a/common/script/logs_handler/logs_tool/log.py +++ b/common/script/logs_handler/logs_tool/log.py @@ -42,15 +42,19 @@ def setup(project_name): log_path = _get_log_file_path(project_name) if log_path: file_log = handlers.RotatingFileHandler( - log_path, maxBytes=log_config.get("log_file_max_size"), - backupCount=log_config.get("log_file_backup_count")) + log_path, + maxBytes=log_config.get("log_file_max_size"), + backupCount=log_config.get("log_file_backup_count"), + ) log_root.addHandler(file_log) for handler in log_root.handlers: handler.setFormatter( logging.Formatter( fmt=log_config.get("logging_context_format_string"), - datefmt=log_config.get("log_date_format"))) + datefmt=log_config.get("log_date_format"), + ) + ) if log_config.get("debug"): log_root.setLevel(logging.DEBUG) diff --git a/common/script/logs_handler/logs_tool/log_config.py b/common/script/logs_handler/logs_tool/log_config.py index 887d4d3e..7b19f7dc 100644 --- a/common/script/logs_handler/logs_tool/log_config.py +++ b/common/script/logs_handler/logs_tool/log_config.py @@ -7,8 +7,8 @@ CONSOLE_CONF = { "log_file_backup_count": 5, "log_date_format": "%Y-%m-%d %H:%M:%S", "logging_default_format_string": "%(asctime)s console %(levelname)s [pid:%(process)d] [%(threadName)s] " - "[tid:%(thread)d] [%(filename)s:%(lineno)d %(funcName)s] %(message)s", + "[tid:%(thread)d] [%(filename)s:%(lineno)d %(funcName)s] %(message)s", "logging_context_format_string": "%(asctime)s console %(levelname)s [pid:%(process)d] [%(threadName)s] " - "[tid:%(thread)d] [%(filename)s:%(lineno)d %(funcName)s] %(message)s" + "[tid:%(thread)d] [%(filename)s:%(lineno)d %(funcName)s] %(message)s", } } diff --git a/ct_om/service/cantian_exporter/config/config.py b/ct_om/service/cantian_exporter/config/config.py index 75cc174f..ec75fc5e 100644 --- a/ct_om/service/cantian_exporter/config/config.py +++ b/ct_om/service/cantian_exporter/config/config.py @@ -7,8 +7,8 @@ CONSOLE_CONF = { "log_file_backup_count": 5, "log_date_format": "%Y-%m-%d %H:%M:%S", "logging_default_format_string": "%(asctime)s console %(levelname)s [pid:%(process)d] [%(threadName)s] " - "[tid:%(thread)d] [%(filename)s:%(lineno)d %(funcName)s] %(message)s", + "[tid:%(thread)d] [%(filename)s:%(lineno)d %(funcName)s] %(message)s", "logging_context_format_string": "%(asctime)s console %(levelname)s [pid:%(process)d] [%(threadName)s] " - "[tid:%(thread)d] [%(filename)s:%(lineno)d %(funcName)s] %(message)s" + "[tid:%(thread)d] [%(filename)s:%(lineno)d %(funcName)s] %(message)s", } } diff --git a/ct_om/service/cantian_exporter/exporter/execute.py b/ct_om/service/cantian_exporter/exporter/execute.py index 39bd4d0c..94257238 100644 --- a/ct_om/service/cantian_exporter/exporter/execute.py +++ b/ct_om/service/cantian_exporter/exporter/execute.py @@ -18,7 +18,10 @@ def main(): save_file = SaveFile() while True: - cms_nodes_info, dbstor_info = get_node_info.execute(), get_dbstor_info.get_dbstor_info() + cms_nodes_info, dbstor_info = ( + get_node_info.execute(), + get_dbstor_info.get_dbstor_info(), + ) cms_nodes_info.update(dbstor_info) try: dr_status_info = get_dr_info.execute() @@ -29,7 +32,11 @@ def main(): try: save_file.create_files(cms_nodes_info) except Exception as err: - LOG.error("[result] Fail to record report data in json file, [err_msg] {}".format(str(err))) + LOG.error( + "[result] Fail to record report data in json file, [err_msg] {}".format( + str(err) + ) + ) time.sleep(20) diff --git a/ct_om/service/cantian_exporter/exporter/get_certificate_status.py b/ct_om/service/cantian_exporter/exporter/get_certificate_status.py index 54bc6d8c..28f8a603 100644 --- a/ct_om/service/cantian_exporter/exporter/get_certificate_status.py +++ b/ct_om/service/cantian_exporter/exporter/get_certificate_status.py @@ -6,7 +6,7 @@ from cryptography.hazmat.backends import default_backend def file_reader(file_path): - with open(file_path, 'r') as file: + with open(file_path, "r") as file: return file.read() @@ -28,7 +28,11 @@ def get_certificate_status(): cert_status = "revoked" not_before = cert.not_valid_before not_after = cert.not_valid_after - if not not_before.replace(tzinfo=timezone.utc) <= current_time <= not_after.replace(tzinfo=timezone.utc): + if ( + not not_before.replace(tzinfo=timezone.utc) + <= current_time + <= not_after.replace(tzinfo=timezone.utc) + ): cert_status = "expired" return crl_status, cert_status diff --git a/ct_om/service/cantian_exporter/exporter/get_info.py b/ct_om/service/cantian_exporter/exporter/get_info.py index a6652620..c840dbb6 100644 --- a/ct_om/service/cantian_exporter/exporter/get_info.py +++ b/ct_om/service/cantian_exporter/exporter/get_info.py @@ -15,16 +15,17 @@ from datetime import datetime from exporter.log import EXPORTER_LOG as LOG from exporter.tool import SimpleSql from exporter.tool import _exec_popen -sys.path.append('/opt/cantian/action/dbstor') + +sys.path.append("/opt/cantian/action/dbstor") from kmc_adapter import CApiWrapper cur_abs_path, _ = os.path.split(os.path.abspath(__file__)) -OLD_CANTIAND_DATA_SAVE_PATH = Path(cur_abs_path, 'cantiand_report_data_saves.json') -DEPLOY_PARAM_PATH = '/opt/cantian/config/deploy_param.json' -INSTALL_CONFIG_PATH = '/opt/cantian/action/cantian/install_config.json' -CANTIAND_INI_PATH = '/mnt/dbdata/local/cantian/tmp/data/cfg/cantiand.ini' -CANTIAND_LOG_PATH = '/opt/cantian/log/cantian/run/cantiand.rlog' -CTSQL_INI_PATH = '/mnt/dbdata/local/cantian/tmp/data/cfg/*sql.ini' +OLD_CANTIAND_DATA_SAVE_PATH = Path(cur_abs_path, "cantiand_report_data_saves.json") +DEPLOY_PARAM_PATH = "/opt/cantian/config/deploy_param.json" +INSTALL_CONFIG_PATH = "/opt/cantian/action/cantian/install_config.json" +CANTIAND_INI_PATH = "/mnt/dbdata/local/cantian/tmp/data/cfg/cantiand.ini" +CANTIAND_LOG_PATH = "/opt/cantian/log/cantian/run/cantiand.rlog" +CTSQL_INI_PATH = "/mnt/dbdata/local/cantian/tmp/data/cfg/*sql.ini" PRIMARY_KEYSTORE = "/opt/cantian/common/config/primary_keystore_bak.ks" STANDBY_KEYSTORE = "/opt/cantian/common/config/standby_keystore_bak.ks" LOGICREP_START_TIME_PATH = "/opt/software/tools/logicrep/log/start_time" @@ -34,12 +35,12 @@ CONVERT_DICT = { "M": 1024 * 1024, "G": 1024 * 1024 * 1024, "T": 1000 * 1024 * 1024 * 1024, - "P": 1000 * 1000 * 1024 * 1024 * 1024 + "P": 1000 * 1000 * 1024 * 1024 * 1024, } def file_reader(file_path): - with open(file_path, 'r') as file: + with open(file_path, "r") as file: return file.read() @@ -47,28 +48,56 @@ def file_writer(file_path, data): modes = stat.S_IWRITE | stat.S_IRUSR flags = os.O_WRONLY | os.O_TRUNC | os.O_CREAT if data: - with os.fdopen(os.open(file_path, flags, modes), 'w', encoding='utf-8') as file: + with os.fdopen(os.open(file_path, flags, modes), "w", encoding="utf-8") as file: file.write(json.dumps(data)) else: - with os.fdopen(os.open(file_path, flags, modes), 'w', encoding='utf-8') as file: + with os.fdopen(os.open(file_path, flags, modes), "w", encoding="utf-8") as file: file.truncate() class GetNodesInfo: def __init__(self): - self.std_output = {'node_id': '', 'stat': '', 'work_stat': 0, 'cluster_name': '', 'cms_ip': '', - 'cantian_vlan_ip': '', 'storage_vlan_ip': '', 'share_logic_ip': '', - 'storage_share_fs': '', 'storage_archive_fs': '', 'storage_metadata_fs': '', - 'data_buffer_size': '', 'log_buffer_size': '', 'log_buffer_count': '', - 'cluster_stat': '', 'cms_port': '', 'cms_connected_domain': '', 'disk_iostat': '', - 'mem_total': '', 'mem_free': '', 'mem_used': '', 'cpu_us': '', 'cpu_sy': '', 'cpu_id': '', - 'sys_backup_sets': {}, 'checkpoint_pages': {}, 'checkpoint_period': {}, 'global_lock': {}, - 'local_lock': {}, 'local_txn': {}, 'global_txn': {}, "dv_lrpl_detail": {}, - 'pitr_warning': '', 'logicrep': '' - } + self.std_output = { + "node_id": "", + "stat": "", + "work_stat": 0, + "cluster_name": "", + "cms_ip": "", + "cantian_vlan_ip": "", + "storage_vlan_ip": "", + "share_logic_ip": "", + "storage_share_fs": "", + "storage_archive_fs": "", + "storage_metadata_fs": "", + "data_buffer_size": "", + "log_buffer_size": "", + "log_buffer_count": "", + "cluster_stat": "", + "cms_port": "", + "cms_connected_domain": "", + "disk_iostat": "", + "mem_total": "", + "mem_free": "", + "mem_used": "", + "cpu_us": "", + "cpu_sy": "", + "cpu_id": "", + "sys_backup_sets": {}, + "checkpoint_pages": {}, + "checkpoint_period": {}, + "global_lock": {}, + "local_lock": {}, + "local_txn": {}, + "global_txn": {}, + "dv_lrpl_detail": {}, + "pitr_warning": "", + "logicrep": "", + } self.sql = SimpleSql() - self.kmc_decrypt = CApiWrapper(primary_keystore=PRIMARY_KEYSTORE, standby_keystore=STANDBY_KEYSTORE) + self.kmc_decrypt = CApiWrapper( + primary_keystore=PRIMARY_KEYSTORE, standby_keystore=STANDBY_KEYSTORE + ) self.kmc_decrypt.initialize() self.deploy_param = None self.mes_ssl_switch = False @@ -78,15 +107,18 @@ class GetNodesInfo: self.storage_archive_fs = None self.dm_pwd = None - self.sh_cmd = {'top -bn 1 -i': self.update_cpu_mem_info, - 'source ~/.bashrc&&cms stat': self.update_cms_status_info, - 'source ~/.bashrc&&cms node -list': self.update_cms_port_info, - 'source ~/.bashrc&&cms node -connected': self.update_cms_node_connected, - 'source ~/.bashrc&&cms diskiostat': self.update_cms_diskiostat - } + self.sh_cmd = { + "top -bn 1 -i": self.update_cpu_mem_info, + "source ~/.bashrc&&cms stat": self.update_cms_status_info, + "source ~/.bashrc&&cms node -list": self.update_cms_port_info, + "source ~/.bashrc&&cms node -connected": self.update_cms_node_connected, + "source ~/.bashrc&&cms diskiostat": self.update_cms_diskiostat, + } self.sql_file = os.path.join(cur_abs_path, "../config/get_ctsql_info.sql") - self.logicrep_sql_file = os.path.join(cur_abs_path, "../config/get_logicrep_info.sql") - self.reg_string = r'invalid argument' + self.logicrep_sql_file = os.path.join( + cur_abs_path, "../config/get_logicrep_info.sql" + ) + self.reg_string = r"invalid argument" @staticmethod def ctsql_result_parse(result: str) -> zip: @@ -133,8 +165,10 @@ class GetNodesInfo: Return： 返回一个字典，键同dict_data，值为cantian进程当前生效的值 """ - cmd = "ps -ef | grep -v grep | grep cantiand | grep -w '\-D " \ - "/mnt/dbdata/local/cantian/tmp/data' | awk '{print $2}'" + cmd = ( + "ps -ef | grep -v grep | grep cantiand | grep -w '\-D " + "/mnt/dbdata/local/cantian/tmp/data' | awk '{print $2}'" + ) install_config = json.loads(file_reader(INSTALL_CONFIG_PATH)) is_single = install_config.get("M_RUNING_MODE") if is_single == "cantiand_with_mysql_in_cluster": @@ -145,16 +179,18 @@ class GetNodesInfo: return {} if not os.path.exists(OLD_CANTIAND_DATA_SAVE_PATH): - record_data = {'report_data': dict_data, 'cantian_pid': pidof_cantiand} + record_data = {"report_data": dict_data, "cantian_pid": pidof_cantiand} file_writer(OLD_CANTIAND_DATA_SAVE_PATH, record_data) return dict_data old_report_data = json.loads(file_reader(OLD_CANTIAND_DATA_SAVE_PATH)) - old_data, old_pidof_cantiand = old_report_data.get('report_data'), old_report_data.get('cantian_pid') - init_record_data = {'report_data': old_data, 'cantian_pid': pidof_cantiand} + old_data, old_pidof_cantiand = old_report_data.get( + "report_data" + ), old_report_data.get("cantian_pid") + init_record_data = {"report_data": old_data, "cantian_pid": pidof_cantiand} if old_pidof_cantiand != pidof_cantiand: if old_data != dict_data: - init_record_data['report_data'] = dict_data + init_record_data["report_data"] = dict_data file_writer(OLD_CANTIAND_DATA_SAVE_PATH, init_record_data) return dict_data @@ -168,27 +204,31 @@ class GetNodesInfo: res: 上层函数传递进来的字典类型数据，用于记录当前函数获取的上报指标 """ # 找到最后一次出现ntp时间误差的行数 - exist_cmd = f"grep -onE '\[NTP_TIME_WARN\] .* us.*' {CANTIAND_LOG_PATH} " \ - "| grep -v ignored" \ - f"| tail -n 1 | awk -F: '{{print $1}}'" - ignored_exist_cmd = f"grep -onE '\[NTP_TIME_WARN\] .+ ignored.' {CANTIAND_LOG_PATH}" \ - f" | tail -n 1 | awk -F: '{{print $1}}'" + exist_cmd = ( + f"grep -onE '\[NTP_TIME_WARN\] .* us.*' {CANTIAND_LOG_PATH} " + "| grep -v ignored" + f"| tail -n 1 | awk -F: '{{print $1}}'" + ) + ignored_exist_cmd = ( + f"grep -onE '\[NTP_TIME_WARN\] .+ ignored.' {CANTIAND_LOG_PATH}" + f" | tail -n 1 | awk -F: '{{print $1}}'" + ) _, exist_res, _ = _exec_popen(exist_cmd) # 不存在ntp时间误差 if not exist_res: - res.update({'pitr_warning': 'False'}) + res.update({"pitr_warning": "False"}) return _, ignored_res, _ = _exec_popen(ignored_exist_cmd) # 存在ntp时间误差 if not ignored_res: - res.update({'pitr_warning': 'True'}) + res.update({"pitr_warning": "True"}) return ignored_res, exist_res = int(ignored_res), int(exist_res) - pitr_flag = 'False' if ignored_res > exist_res else 'True' - res.update({'pitr_warning': pitr_flag}) + pitr_flag = "False" if ignored_res > exist_res else "True" + res.update({"pitr_warning": pitr_flag}) @staticmethod def get_cms_lock_failed_info(res): @@ -201,20 +241,23 @@ class GetNodesInfo: "read failed" "write failed" """ - check_cmd = "zgrep -E \"(cms_disk_lock timeout.|read failed|write failed)\" " \ - "/opt/cantian/log/cms/run/* | grep -oE \"[0-9]{4}-[0-9]{2}-[0-9]{2} [0-9:]+\" | sort" + check_cmd = ( + 'zgrep -E "(cms_disk_lock timeout.|read failed|write failed)" ' + '/opt/cantian/log/cms/run/* | grep -oE "[0-9]{4}-[0-9]{2}-[0-9]{2} [0-9:]+" | sort' + ) _, output, _ = _exec_popen(check_cmd) # 存在加解锁失败问题 if output: lock_failed_happen_time = output.split("\n")[-1] - datetime_object = datetime.strptime(lock_failed_happen_time, '%Y-%m-%d %H:%M:%S') + datetime_object = datetime.strptime( + lock_failed_happen_time, "%Y-%m-%d %H:%M:%S" + ) happen_times = datetime_object.timestamp() current_time = time.time() if int(current_time) - int(happen_times) < 10: - res.update({'cms_lock_status': 'abnormal'}) + res.update({"cms_lock_status": "abnormal"}) return - res.update({'cms_lock_status': 'normal'}) - + res.update({"cms_lock_status": "normal"}) @staticmethod def close_child_process(proc): @@ -230,7 +273,7 @@ class GetNodesInfo: except Exception as err: return str(err), ABNORMAL_STATE - return 'success', NORMAL_STATE + return "success", NORMAL_STATE @staticmethod def get_logicrep_running_info(max_archive_size, sql_info): @@ -240,18 +283,30 @@ class GetNodesInfo: """ process_speed = sql_info.get("logicrep_progress", {}).get("PROCESS_SPEED", "0") process_speed = float(process_speed) * CONVERT_DICT.get("M") - redo_gen_speed = sql_info.get("logicrep_progress", {}).get("REDO_GEN_SPEED", "0") + redo_gen_speed = sql_info.get("logicrep_progress", {}).get( + "REDO_GEN_SPEED", "0" + ) redo_gen_speed = float(redo_gen_speed) * CONVERT_DICT.get("M") - speed_update_time = sql_info.get("logicrep_progress", {}).get("SPEED_UPDATE_TIME", - "1970-01-01 00:00:00").split(".")[0] - speed_update_time = int(datetime.strptime(speed_update_time, '%Y-%m-%d %H:%M:%S').timestamp()) + speed_update_time = ( + sql_info.get("logicrep_progress", {}) + .get("SPEED_UPDATE_TIME", "1970-01-01 00:00:00") + .split(".")[0] + ) + speed_update_time = int( + datetime.strptime(speed_update_time, "%Y-%m-%d %H:%M:%S").timestamp() + ) current_time = int(time.time()) # 如果入湖进程刷新时间（speed_update_time）与当前时间（current_time）间隔不大于30s， # 当前入湖实际速度为日志刷盘速度（redo_gen_speed） - 工具处理速度（process_speed） # 否则入湖实际速度为（redo_gen_speed） - real_process_speed = redo_gen_speed - process_speed if \ - current_time - speed_update_time < 30 else redo_gen_speed - arch_clean_upper_limit = sql_info.get("arch_clean_upper_limit", {}).get("RUNTIME_VALUE", 85) + real_process_speed = ( + redo_gen_speed - process_speed + if current_time - speed_update_time < 30 + else redo_gen_speed + ) + arch_clean_upper_limit = sql_info.get("arch_clean_upper_limit", {}).get( + "RUNTIME_VALUE", 85 + ) # 归档清理上限（arch_clean_upper_size） arch_clean_upper_size = max_archive_size * int(arch_clean_upper_limit) / 100 return arch_clean_upper_size, real_process_speed @@ -263,7 +318,7 @@ class GetNodesInfo: logicrep_cmd = "ps -ef | grep ZLogCatcherMain | grep -v grep" if os.path.exists(LOGICREP_START_TIME_PATH): - with open(LOGICREP_START_TIME_PATH, 'r') as f: + with open(LOGICREP_START_TIME_PATH, "r") as f: start_time = f.readline().strip() if not start_time: start_time = "null" @@ -272,15 +327,15 @@ class GetNodesInfo: _, process_info, _ = _exec_popen(logicrep_cmd) if process_info: - res.update({'logicrep': 'Online', 'logicrep_start_time': start_time}) + res.update({"logicrep": "Online", "logicrep_start_time": start_time}) res.update(self.get_logicrep_info_from_sql(res)) return logicrep_path = "/opt/software/tools/logicrep/" if os.path.exists(logicrep_path): - res.update({'logicrep': 'Offline', 'logicrep_start_time': start_time}) + res.update({"logicrep": "Offline", "logicrep_start_time": start_time}) res.update(self.get_logicrep_info_from_sql(res)) return - res.update({'logicrep': 'None'}) + res.update({"logicrep": "None"}) def get_certificate_status(self, res): """ @@ -292,15 +347,9 @@ class GetNodesInfo: output, err_state = self.shell_task(cmd) if not err_state and output: crl_status, crt_status = re.findall(r"'([^']*)'", output) - res.update({ - "crl_status": crl_status, - "crt_status": crt_status - }) + res.update({"crl_status": crl_status, "crt_status": crt_status}) else: - res.update({ - "crl_status": None, - "crt_status": None - }) + res.update({"crl_status": None, "crt_status": None}) def shell_task(self, exec_cmd): """公共方法，用于执行shell命令 @@ -309,37 +358,57 @@ class GetNodesInfo: exec_cmd: 具体的某个shell命令 """ try: - proc = subprocess.Popen(exec_cmd, stdout=subprocess.PIPE, shell=True, preexec_fn=os.setsid) + proc = subprocess.Popen( + exec_cmd, stdout=subprocess.PIPE, shell=True, preexec_fn=os.setsid + ) except Exception as err: - LOG.error("[shell task] node {} execute '{}' failed, err: {}".format(self.node_id, exec_cmd, str(err))) + LOG.error( + "[shell task] node {} execute '{}' failed, err: {}".format( + self.node_id, exec_cmd, str(err) + ) + ) _, close_state = self.close_child_process(proc) if close_state: - LOG.error("[shell task] after node {} executes cmd '{}', " - "it fails to kill the forked process ".format(self.node_id, exec_cmd)) + LOG.error( + "[shell task] after node {} executes cmd '{}', " + "it fails to kill the forked process ".format( + self.node_id, exec_cmd + ) + ) return str(err), ABNORMAL_STATE try: output, err_state = proc.communicate(timeout=TIME_OUT) except Exception as err: - LOG.error("[shell task] node {} execute cmd '{}' failed, err: {}".format(self.node_id, exec_cmd, str(err))) + LOG.error( + "[shell task] node {} execute cmd '{}' failed, err: {}".format( + self.node_id, exec_cmd, str(err) + ) + ) return str(err), ABNORMAL_STATE finally: close_res, close_state = self.close_child_process(proc) if close_state: - LOG.error("[shell task] after node {} executes cmd '{}', " - "it fails to kill the forked process ".format(self.node_id, exec_cmd)) + LOG.error( + "[shell task] after node {} executes cmd '{}', " + "it fails to kill the forked process ".format(self.node_id, exec_cmd) + ) return close_res, close_state if err_state or not output: - LOG.error("[shell task] node {} execute cmd '{}' failed, output: {}, " - "err_state: {}".format(self.node_id, exec_cmd, str(output), err_state)) + LOG.error( + "[shell task] node {} execute cmd '{}' failed, output: {}, " + "err_state: {}".format(self.node_id, exec_cmd, str(output), err_state) + ) return output, ABNORMAL_STATE - output = output.decode('utf-8') + output = output.decode("utf-8") if re.findall(self.reg_string, output): - LOG.error("the execution result of command '{}' matched the regular pattern '{}', " - "and the execution failed".format(exec_cmd, self.reg_string)) + LOG.error( + "the execution result of command '{}' matched the regular pattern '{}', " + "and the execution failed".format(exec_cmd, self.reg_string) + ) return output, ABNORMAL_STATE return output, err_state @@ -350,21 +419,37 @@ class GetNodesInfo: Args: res: 上层函数传递进来的字典类型数据，用于记录当前函数获取的上报指标 """ - deploy_key_list = ['cluster_name', 'cantian_vlan_ip', 'storage_vlan_ip', 'cms_ip', 'share_logic_ip', - 'storage_archive_fs', 'storage_share_fs', 'storage_metadata_fs'] - res.update({name: self.deploy_param.get(name, '') for name in deploy_key_list}) + deploy_key_list = [ + "cluster_name", + "cantian_vlan_ip", + "storage_vlan_ip", + "cms_ip", + "share_logic_ip", + "storage_archive_fs", + "storage_share_fs", + "storage_metadata_fs", + ] + res.update({name: self.deploy_param.get(name, "") for name in deploy_key_list}) - cantiand_key_list = ['data_buffer_size', 'log_buffer_size', 'log_buffer_count'] + cantiand_key_list = ["data_buffer_size", "log_buffer_size", "log_buffer_count"] try: cantiand_data = file_reader(CANTIAND_INI_PATH) except Exception as err: - LOG.error("[file read task] node {} read '{}' from {} failed, " - "err_details: {}".format(self.node_id, cantiand_key_list, CANTIAND_INI_PATH, str(err))) + LOG.error( + "[file read task] node {} read '{}' from {} failed, " + "err_details: {}".format( + self.node_id, cantiand_key_list, CANTIAND_INI_PATH, str(err) + ) + ) else: - processed_data = [data for data in cantiand_data.split('\n') if data] - reg_string = r'DATA_BUFFER_SIZE|LOG_BUFFER_SIZE|LOG_BUFFER_COUNT' - report_data = [item for item in processed_data if re.findall(reg_string, item)] - cantian_report_data = {item.split(' ')[0].lower(): item.split(' ')[-1] for item in report_data} + processed_data = [data for data in cantiand_data.split("\n") if data] + reg_string = r"DATA_BUFFER_SIZE|LOG_BUFFER_SIZE|LOG_BUFFER_COUNT" + report_data = [ + item for item in processed_data if re.findall(reg_string, item) + ] + cantian_report_data = { + item.split(" ")[0].lower(): item.split(" ")[-1] for item in report_data + } res.update(self.cantiand_report_handler(cantian_report_data)) def get_info_from_sql(self, res): @@ -374,7 +459,7 @@ class GetNodesInfo: res: 上层函数传递进来的字典类型数据，用于记录当前函数获取的上报指标 """ # 参天进程异常，性能上报不采集ctsql数据库中的指标，防止和参天进程竞争ctsql - if res.get('stat') != 'ONLINE' or str(res.get('work_stat')) != '1': + if res.get("stat") != "ONLINE" or str(res.get("work_stat")) != "1": return res.update(self.sql_info_query()) @@ -395,21 +480,27 @@ class GetNodesInfo: def modify_logicrep_sql_file(self): logicrep_sql = file_reader(self.logicrep_sql_file) - logicrep_sql = logicrep_sql.replace('LOGICREP0', 'LOGICREP1') + logicrep_sql = logicrep_sql.replace("LOGICREP0", "LOGICREP1") modes = stat.S_IWRITE | stat.S_IRUSR flags = os.O_WRONLY | os.O_TRUNC | os.O_CREAT - with os.fdopen(os.open(self.logicrep_sql_file, flags, modes), 'w', encoding='utf-8') as file: + with os.fdopen( + os.open(self.logicrep_sql_file, flags, modes), "w", encoding="utf-8" + ) as file: file.write(logicrep_sql) def get_logicrep_info_from_sql(self, res): logicrep_process_info = { "unrep_archive_count": "null", "unrep_archive_percent": "null", - "estimate_full_time": "null" + "estimate_full_time": "null", } - if res.get('stat') != 'ONLINE' or str(res.get('work_stat')) != '1' or not self.decrypt_pwd \ - or not self.storage_archive_fs: + if ( + res.get("stat") != "ONLINE" + or str(res.get("work_stat")) != "1" + or not self.decrypt_pwd + or not self.storage_archive_fs + ): return logicrep_process_info if self.node_id == 1: self.modify_logicrep_sql_file() @@ -419,18 +510,29 @@ class GetNodesInfo: tmp_archive_count = self.get_tmp_archive_count(res) max_sequence = sql_info.get("max(sequence)", {}).get("MAX(SEQUENCE#)") - max_archive_count = tmp_archive_count if not max_sequence else tmp_archive_count + int(max_sequence) + max_archive_count = ( + tmp_archive_count + if not max_sequence + else tmp_archive_count + int(max_sequence) + ) if not max_archive_count: return logicrep_process_info - undo_archive_size, max_archive_size, sub_logicrep_process_info = self.get_logicrep_undo_count_and_percent( - sql_info, res, max_archive_count) + undo_archive_size, max_archive_size, sub_logicrep_process_info = ( + self.get_logicrep_undo_count_and_percent(sql_info, res, max_archive_count) + ) arch_clean_upper_size, real_process_speed = self.get_logicrep_running_info( - max_archive_size, sql_info) + max_archive_size, sql_info + ) # 日志预计堆满时间 = (归档上限空间（arch_clean_upper_size） - 未梳理日志空间（undo_archive_size）) / 日志刷盘速度（real_process_speed） - full_remaining_time = (arch_clean_upper_size - undo_archive_size) / real_process_speed \ - if real_process_speed != 0 else "null" - logicrep_process_info.update({"estimate_full_time": "{:.2f}s".format(full_remaining_time)}) + full_remaining_time = ( + (arch_clean_upper_size - undo_archive_size) / real_process_speed + if real_process_speed != 0 + else "null" + ) + logicrep_process_info.update( + {"estimate_full_time": "{:.2f}s".format(full_remaining_time)} + ) logicrep_process_info.update(sub_logicrep_process_info) return logicrep_process_info @@ -438,33 +540,63 @@ class GetNodesInfo: """ 获取入湖未归档文件数量（unrep_archive_count）和未归档百分比（unrep_archive_percent） """ - max_arch_files_size = sql_info.get("max_arch_files_size", {}).get("RUNTIME_VALUE") + max_arch_files_size = sql_info.get("max_arch_files_size", {}).get( + "RUNTIME_VALUE" + ) units = max_arch_files_size[-1] max_archive_size = int(max_arch_files_size[:-1]) * CONVERT_DICT.get(units) arch_file_size = sql_info.get("arch_file_size", {}).get("RUNTIME_VALUE") - arch_clean_upper_limit = int(sql_info.get("arch_clean_upper_limit", {}).get("RUNTIME_VALUE", "85")) + arch_clean_upper_limit = int( + sql_info.get("arch_clean_upper_limit", {}).get("RUNTIME_VALUE", "85") + ) units = arch_file_size[-1] single_archive_size = int(arch_file_size[:-1]) * CONVERT_DICT.get(units) logic_point = sql_info.get("logicrep_progress", {}).get("LOGPOINT") temp_archive_size = self.get_tmp_archive_size(res) if not logic_point: undo_archive_count = max_archive_count - undo_archive_size = undo_archive_count * single_archive_size if temp_archive_size == 0 else ( - (undo_archive_count - 1) * single_archive_size + temp_archive_size) - undo_archive_percent = 0 if max_archive_size == 0 else \ - (undo_archive_size / (max_archive_size * arch_clean_upper_limit / 100) * 100) + undo_archive_size = ( + undo_archive_count * single_archive_size + if temp_archive_size == 0 + else ( + (undo_archive_count - 1) * single_archive_size + temp_archive_size + ) + ) + undo_archive_percent = ( + 0 + if max_archive_size == 0 + else ( + undo_archive_size + / (max_archive_size * arch_clean_upper_limit / 100) + * 100 + ) + ) else: point_info = logic_point.split("-") asn = int(point_info[1], 16) offset = int(point_info[2], 16) undo_archive_count = max_archive_count - asn + 1 - undo_archive_size = temp_archive_size - offset if undo_archive_count == 0 \ - else (undo_archive_count - 1) * single_archive_size + temp_archive_size - offset - undo_archive_percent = 0 if max_archive_size == 0 else \ - (undo_archive_size / (max_archive_size * arch_clean_upper_limit / 100) * 100) + undo_archive_size = ( + temp_archive_size - offset + if undo_archive_count == 0 + else (undo_archive_count - 1) * single_archive_size + + temp_archive_size + - offset + ) + undo_archive_percent = ( + 0 + if max_archive_size == 0 + else ( + undo_archive_size + / (max_archive_size * arch_clean_upper_limit / 100) + * 100 + ) + ) undo_archive_percent = "{:.2f}%".format(undo_archive_percent) - logicrep_process_info = {"unrep_archive_count": str(undo_archive_count), - "unrep_archive_percent": str(undo_archive_percent)} + logicrep_process_info = { + "unrep_archive_count": str(undo_archive_count), + "unrep_archive_percent": str(undo_archive_percent), + } return undo_archive_size, max_archive_size, logicrep_process_info def sql_logicrep_info_query(self): @@ -474,8 +606,12 @@ class GetNodesInfo: """ res = {} report_key = [ - "MAX(SEQUENCE)", "LOGICREP_PROGRESS", "LREP_MODE", "MAX_ARCH_FILES_SIZE", - "ARCH_FILE_SIZE", "ARCH_CLEAN_UPPER_LIMIT" + "MAX(SEQUENCE)", + "LOGICREP_PROGRESS", + "LREP_MODE", + "MAX_ARCH_FILES_SIZE", + "ARCH_FILE_SIZE", + "ARCH_CLEAN_UPPER_LIMIT", ] return_code, sql_res = self.sql.query(self.logicrep_sql_file) if not return_code and sql_res: @@ -489,10 +625,14 @@ class GetNodesInfo: """ res = {} report_key = [ - "SYS_BACKUP_SETS", "CHECKPOINT_PAGES", - "CHECKPOINT_PERIOD", "GLOBAL_LOCK", - "LOCAL_LOCK", "LOCAL_TXN", "GLOBAL_TXN", - "DV_LRPL_DETAIL" + "SYS_BACKUP_SETS", + "CHECKPOINT_PAGES", + "CHECKPOINT_PERIOD", + "GLOBAL_LOCK", + "LOCAL_LOCK", + "LOCAL_TXN", + "GLOBAL_TXN", + "DV_LRPL_DETAIL", ] return_code, sql_res = self.sql.query(self.sql_file) if not return_code and sql_res: @@ -520,11 +660,9 @@ class GetNodesInfo: """ res = {} sql_res_list = sql_res.split("SQL>") - for index, sql_res in enumerate(sql_res_list[1:len(report_key) + 1]): + for index, sql_res in enumerate(sql_res_list[1 : len(report_key) + 1]): res.update( - { - report_key[index].lower(): dict(self.ctsql_result_parse(sql_res)) - } + {report_key[index].lower(): dict(self.ctsql_result_parse(sql_res))} ) return res @@ -549,10 +687,12 @@ class GetNodesInfo: """ cms_port_info, cms_port_err = self.shell_task(cms_port_cmd) if not cms_port_err and cms_port_info: - tmp_port_info = [re.split(r'\s+', val.strip(' ')) - for _, val in enumerate(cms_port_info.split('\n')) - if val][1:] - return {'cms_port': str(tmp_port_info[self.node_id][-1])} + tmp_port_info = [ + re.split(r"\s+", val.strip(" ")) + for _, val in enumerate(cms_port_info.split("\n")) + if val + ][1:] + return {"cms_port": str(tmp_port_info[self.node_id][-1])} return {} @@ -564,13 +704,28 @@ class GetNodesInfo: """ node_info, err_code = self.shell_task(cms_node_connected_cmd) if not err_code and node_info: - processed_info = [re.split(r'\s+', item.strip(' ')) for item in node_info.split('\n') if item] + processed_info = [ + re.split(r"\s+", item.strip(" ")) + for item in node_info.split("\n") + if item + ] remain_nums = len(processed_info[1:]) node_id_idx, ip_idx, voting_idx = 0, 2, 4 - node_data = [{'NODE_ID': item[node_id_idx], 'IP': item[ip_idx], 'VOTING': item[voting_idx]} - for item in processed_info[1:]] + node_data = [ + { + "NODE_ID": item[node_id_idx], + "IP": item[ip_idx], + "VOTING": item[voting_idx], + } + for item in processed_info[1:] + ] - res = {'cms_connected_domain': {'remaining_nodes_nums': remain_nums, 'remaining_nodes': node_data}} + res = { + "cms_connected_domain": { + "remaining_nodes_nums": remain_nums, + "remaining_nodes": node_data, + } + } return res return {} @@ -583,21 +738,28 @@ class GetNodesInfo: """ res = {} - id_to_key = {'0': 'node_id', '2': 'stat', '5': 'work_stat'} + id_to_key = {"0": "node_id", "2": "stat", "5": "work_stat"} cms_output, cms_err = self.shell_task(cms_stats_cmd) if not cms_err and cms_output: - tmp_info = [re.split(r'\s+', val.strip(' ')) - for _, val in enumerate(cms_output.split('\n')) - if val] - cms_stat = [{val: item[int(key)] for key, val in id_to_key.items()} for item in tmp_info[1:]] - cluster_stat = 0 if {'ONLINE'} == set([item.get('stat') for item in cms_stat]) else 1 + tmp_info = [ + re.split(r"\s+", val.strip(" ")) + for _, val in enumerate(cms_output.split("\n")) + if val + ] + cms_stat = [ + {val: item[int(key)] for key, val in id_to_key.items()} + for item in tmp_info[1:] + ] + cluster_stat = ( + 0 if {"ONLINE"} == set([item.get("stat") for item in cms_stat]) else 1 + ) stat_data = cms_stat[self.node_id] - work_stat = stat_data.get('work_stat') - stat_data['work_stat'] = int(work_stat) + work_stat = stat_data.get("work_stat") + stat_data["work_stat"] = int(work_stat) res.update(stat_data) - res.update({'cluster_stat': cluster_stat}) + res.update({"cluster_stat": cluster_stat}) return res @@ -611,7 +773,7 @@ class GetNodesInfo: """ cms_output, cms_err = self.shell_task(cms_disk_iostat_cmd) if not cms_err and cms_output: - return {'disk_iostat': cms_output.split('\n')[0]} + return {"disk_iostat": cms_output.split("\n")[0]} return {} @@ -624,15 +786,19 @@ class GetNodesInfo: output, err = self.shell_task(exec_cmd) if not err and output: - output = output.split('\n') - cpu_info, physical_mem = [item.strip() for item in re.split(r'[,:]', output[2].strip())], \ - [item.strip() for item in re.split(r'[,:]', output[3].strip())] - mem_unit = physical_mem[0].split(' ')[0] - cpu_res, mem_res = {('cpu_' + item.split(' ')[1]): item.split(' ')[0] + '%' - for item in cpu_info[1:5]}, \ - {('mem_' + item.split(' ')[1]): item.split(' ')[0] + mem_unit - for item in physical_mem[1:4]} - cpu_res.pop('cpu_ni') + output = output.split("\n") + cpu_info, physical_mem = [ + item.strip() for item in re.split(r"[,:]", output[2].strip()) + ], [item.strip() for item in re.split(r"[,:]", output[3].strip())] + mem_unit = physical_mem[0].split(" ")[0] + cpu_res, mem_res = { + ("cpu_" + item.split(" ")[1]): item.split(" ")[0] + "%" + for item in cpu_info[1:5] + }, { + ("mem_" + item.split(" ")[1]): item.split(" ")[0] + mem_unit + for item in physical_mem[1:4] + } + cpu_res.pop("cpu_ni") mem_res.update(cpu_res) return mem_res @@ -663,10 +829,14 @@ class GetNodesInfo: try: self.deploy_param = json.loads(file_reader(DEPLOY_PARAM_PATH)) except Exception as err: - LOG.error('[result] execution failed when read deploy_param.json, [err_msg] {}'.format(str(err))) + LOG.error( + "[result] execution failed when read deploy_param.json, [err_msg] {}".format( + str(err) + ) + ) return res - self.node_id = int(self.deploy_param.get('node_id')) + self.node_id = int(self.deploy_param.get("node_id")) self.mes_ssl_switch = self.deploy_param.get("mes_ssl_switch") self.storage_archive_fs = self.deploy_param.get("storage_archive_fs") @@ -675,15 +845,23 @@ class GetNodesInfo: self._init_ctsql_vars() # 恢复环境变量，避免cms命令执行失败 - split_env = os.environ['LD_LIBRARY_PATH'].split(":") - filtered_env = [single_env for single_env in split_env if "/opt/cantian/dbstor/lib" not in single_env] - os.environ['LD_LIBRARY_PATH'] = ":".join(filtered_env) + split_env = os.environ["LD_LIBRARY_PATH"].split(":") + filtered_env = [ + single_env + for single_env in split_env + if "/opt/cantian/dbstor/lib" not in single_env + ] + os.environ["LD_LIBRARY_PATH"] = ":".join(filtered_env) try: self.get_export_data(res) except Exception as err: - LOG.error('[result] execution failed when get specific export data. ' - '[err_msg] {}, [err_traceback] {}'.format(str(err), traceback.format_exc(limit=-1))) + LOG.error( + "[result] execution failed when get specific export data. " + "[err_msg] {}, [err_traceback] {}".format( + str(err), traceback.format_exc(limit=-1) + ) + ) return res return res @@ -691,13 +869,17 @@ class GetNodesInfo: def _init_ctsql_vars(self): ctsql_ini_path = glob.glob(CTSQL_INI_PATH)[0] ctsql_ini_data = file_reader(ctsql_ini_path) - encrypt_pwd = ctsql_ini_data[ctsql_ini_data.find('=') + 1:].strip() + encrypt_pwd = ctsql_ini_data[ctsql_ini_data.find("=") + 1 :].strip() try: self.decrypt_pwd = self.kmc_decrypt.decrypt(encrypt_pwd) except Exception as err: # 日志限频 if not self.ctsql_decrypt_error_flag: - LOG.error('[result] decrypt ctsql passwd failed, [err_msg] {}'.format(str(err))) + LOG.error( + "[result] decrypt ctsql passwd failed, [err_msg] {}".format( + str(err) + ) + ) self.ctsql_decrypt_error_flag = True self.ctsql_decrypt_error_flag = False self.sql.update_sys_data(self.node_id, self.decrypt_pwd) @@ -707,20 +889,26 @@ class GetDbstorInfo: def __init__(self): self.deploy_config = self.get_deploy_info() self.std_output = { - self.deploy_config.get("storage_dbstor_fs"): - { - 'limit': 0, 'used': 0, 'free': 0, - 'snapshotLimit': 0, 'snapshotUsed': 0, - 'fsId': '', 'linkState': '' - }, - self.deploy_config.get("storage_dbstor_page_fs"): - { - 'limit': 0, 'used': 0, 'free': 0, - 'snapshotLimit': 0, 'snapshotUsed': 0, - 'fsId': '', 'linkState': '' - } + self.deploy_config.get("storage_dbstor_fs"): { + "limit": 0, + "used": 0, + "free": 0, + "snapshotLimit": 0, + "snapshotUsed": 0, + "fsId": "", + "linkState": "", + }, + self.deploy_config.get("storage_dbstor_page_fs"): { + "limit": 0, + "used": 0, + "free": 0, + "snapshotLimit": 0, + "snapshotUsed": 0, + "fsId": "", + "linkState": "", + }, } - self.info_file_path = '/opt/cantian/common/data/dbstor_info.json' + self.info_file_path = "/opt/cantian/common/data/dbstor_info.json" self.index = 0 self.max_index = 10 self.last_time_stamp = None @@ -739,16 +927,20 @@ class GetDbstorInfo: break except Exception as err: try_times -= 1 - LOG.error("[dbstor info reader] fail to read dbstor info from '{}', " - "err_msg: {}, remaining attempts: {}".format(self.info_file_path, str(err), try_times)) + LOG.error( + "[dbstor info reader] fail to read dbstor info from '{}', " + "err_msg: {}, remaining attempts: {}".format( + self.info_file_path, str(err), try_times + ) + ) time.sleep(1) continue if not dbstor_info: - raise Exception('dbstor_info is empty.') + raise Exception("dbstor_info is empty.") dbstor_log_fs, dbstor_page_fs = dbstor_info - time_stamp, _ = dbstor_log_fs.pop('timestamp'), dbstor_page_fs.pop('timestamp') + time_stamp, _ = dbstor_log_fs.pop("timestamp"), dbstor_page_fs.pop("timestamp") if time_stamp != self.last_time_stamp: self.index, self.last_time_stamp = 0, time_stamp else: @@ -766,10 +958,12 @@ class GetDbstorInfo: if dbstor_info: dbstor_log_fs, dbstor_page_fs = dbstor_info - log_fs_name, page_fs_name = dbstor_log_fs.pop('fsName'), dbstor_page_fs.pop('fsName') + log_fs_name, page_fs_name = dbstor_log_fs.pop("fsName"), dbstor_page_fs.pop( + "fsName" + ) cur_res = {log_fs_name: dbstor_log_fs, page_fs_name: dbstor_page_fs} if self.index >= self.max_index: - cur_res.update({'work_stat': 6}) + cur_res.update({"work_stat": 6}) res.update(cur_res) return res diff --git a/ct_om/service/cantian_exporter/exporter/log.py b/ct_om/service/cantian_exporter/exporter/log.py index 88eaaba8..5b5a262b 100644 --- a/ct_om/service/cantian_exporter/exporter/log.py +++ b/ct_om/service/cantian_exporter/exporter/log.py @@ -44,15 +44,19 @@ def setup(project_name): log_path = _get_log_file_path(project_name) if log_path: file_log = handlers.RotatingFileHandler( - log_path, maxBytes=log_config.get("log_file_max_size"), - backupCount=log_config.get("log_file_backup_count")) + log_path, + maxBytes=log_config.get("log_file_max_size"), + backupCount=log_config.get("log_file_backup_count"), + ) log_root.addHandler(file_log) for handler in log_root.handlers: handler.setFormatter( logging.Formatter( fmt=log_config.get("logging_context_format_string"), - datefmt=log_config.get("log_date_format"))) + datefmt=log_config.get("log_date_format"), + ) + ) if log_config.get("debug"): log_root.setLevel(logging.DEBUG) diff --git a/ct_om/service/cantian_exporter/exporter/save_file.py b/ct_om/service/cantian_exporter/exporter/save_file.py index ccc5633f..0193aad9 100644 --- a/ct_om/service/cantian_exporter/exporter/save_file.py +++ b/ct_om/service/cantian_exporter/exporter/save_file.py @@ -11,17 +11,21 @@ upper_path = os.path.abspath(os.path.join(dir_name, "..")) class SaveFile: def __init__(self): - self.file_save_path = os.path.join(upper_path, 'exporter_data') + self.file_save_path = os.path.join(upper_path, "exporter_data") self.uid = os.getuid() self._init_config() - self.save = OrderedDict({name: os.path.join(self.file_save_path, name) - for name in os.listdir(self.file_save_path)}) + self.save = OrderedDict( + { + name: os.path.join(self.file_save_path, name) + for name in os.listdir(self.file_save_path) + } + ) @staticmethod def gen_file_name(): utc_now = datetime.utcnow() cur_time = utc_now.replace(tzinfo=timezone.utc).astimezone(tz=None) - return "%s.json" % str(cur_time.strftime('%Y%m%d%H%M%S')) + return "%s.json" % str(cur_time.strftime("%Y%m%d%H%M%S")) def _init_config(self): if not os.path.exists(self.file_save_path): @@ -44,7 +48,9 @@ class SaveFile: modes = stat.S_IWRITE | stat.S_IRUSR flags = os.O_WRONLY | os.O_TRUNC | os.O_CREAT - with os.fdopen(os.open(cur_file_path, flags, modes), 'w', encoding='utf-8') as file: + with os.fdopen( + os.open(cur_file_path, flags, modes), "w", encoding="utf-8" + ) as file: file.write(json.dumps(data_to_write)) os.chmod(cur_file_path, 0o640) os.chown(cur_file_path, self.uid, 1100) diff --git a/ct_om/service/cantian_exporter/exporter/tool.py b/ct_om/service/cantian_exporter/exporter/tool.py index e159a84b..81b3f894 100644 --- a/ct_om/service/cantian_exporter/exporter/tool.py +++ b/ct_om/service/cantian_exporter/exporter/tool.py @@ -13,11 +13,11 @@ def close_child_process(proc): os.killpg(proc.pid, signal.SIGKILL) except ProcessLookupError as err: _ = err - return 'success' + return "success" except Exception as err: return str(err) - return 'success' + return "success" def _exec_popen(cmd): @@ -27,8 +27,14 @@ def _exec_popen(cmd): return: status code, standard output, error output """ bash_cmd = ["bash"] - pobj = subprocess.Popen(bash_cmd, shell=False, stdin=subprocess.PIPE, - stdout=subprocess.PIPE, stderr=subprocess.PIPE, preexec_fn=os.setsid) + pobj = subprocess.Popen( + bash_cmd, + shell=False, + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + preexec_fn=os.setsid, + ) pobj.stdin.write(cmd.encode()) pobj.stdin.write(os.linesep.encode()) try: @@ -40,7 +46,11 @@ def _exec_popen(cmd): kill_fork_process_res = close_child_process(pobj) if kill_fork_process_res != "success": - return FAIL, "", "kill fork process failed, err_details: {}".format(kill_fork_process_res) + return ( + FAIL, + "", + "kill fork process failed, err_details: {}".format(kill_fork_process_res), + ) stdout, stderr = stdout.decode(), stderr.decode() if stdout[-1:] == os.linesep: @@ -57,8 +67,8 @@ class SimpleSql: self.node_id = None self.sql_sh_path = None self.time_out = 5 - self.ctsql_ip_addr = '127.0.0.1' - self.ctsql_port = '1611' + self.ctsql_ip_addr = "127.0.0.1" + self.ctsql_port = "1611" self.__decrypt_pwd = None def update_sys_data(self, cur_node_id, decrypt_pwd): @@ -66,14 +76,19 @@ class SimpleSql: self.__decrypt_pwd = decrypt_pwd def query(self, sql_file): - exec_cmd = "echo '{}' | ctsql sys@{}:{} -q -f \"{}\"".format(self.__decrypt_pwd, self.ctsql_ip_addr, - self.ctsql_port, sql_file) - return_code, stdout, stderr = _exec_popen('source ~/.bashrc&&{}'.format(exec_cmd)) + exec_cmd = "echo '{}' | ctsql sys@{}:{} -q -f \"{}\"".format( + self.__decrypt_pwd, self.ctsql_ip_addr, self.ctsql_port, sql_file + ) + return_code, stdout, stderr = _exec_popen( + "source ~/.bashrc&&{}".format(exec_cmd) + ) if return_code: stderr = str(stderr) stderr.replace(self.__decrypt_pwd, "*****") - LOG.error("[sql shell task] node {} execute cmd '{}' " - "failed, err: {}".format(self.node_id, self.sql_statement, str(stderr))) + LOG.error( + "[sql shell task] node {} execute cmd '{}' " + "failed, err: {}".format(self.node_id, self.sql_statement, str(stderr)) + ) return return_code, stdout diff --git a/ct_om/service/cantian_exporter/query_storage_info/get_dr_info.py b/ct_om/service/cantian_exporter/query_storage_info/get_dr_info.py index ebe22672..c7872247 100644 --- a/ct_om/service/cantian_exporter/query_storage_info/get_dr_info.py +++ b/ct_om/service/cantian_exporter/query_storage_info/get_dr_info.py @@ -2,17 +2,22 @@ import json import os import sys + try: from query_storage_info.rest_client import RestClient, read_helper except ImportError: # 非容灾场景不依赖requests库 pass from query_storage_info.response_parse import ResponseParse -from query_storage_info.rest_constant import Constant, \ - MetroDomainRunningStatus, VstorePairRunningStatus, HealthStatus +from query_storage_info.rest_constant import ( + Constant, + MetroDomainRunningStatus, + VstorePairRunningStatus, + HealthStatus, +) from exporter.log import EXPORTER_LOG as LOG -sys.path.append('/opt/cantian/action/dbstor') +sys.path.append("/opt/cantian/action/dbstor") from kmc_adapter import CApiWrapper @@ -49,10 +54,10 @@ class DRStatusCheck(object): err_msg = "Login failed" raise Exception(err_msg) rsp_code, rsp_result, rsp_data = result.get_rsp_data() - error_code = rsp_result.get('code') + error_code = rsp_result.get("code") if rsp_code != 0 or error_code != 0: - error_des = rsp_result.get('description') - error_sgt = rsp_result.get('suggestion') + error_des = rsp_result.get("description") + error_sgt = rsp_result.get("suggestion") err_msg = err_msg % (error_code, error_des, error_sgt) raise Exception(err_msg) return rsp_data @@ -62,10 +67,10 @@ class DRStatusCheck(object): err_msg = err_msg + ", Detail:[%s]%s.Suggestion:%s" result = ResponseParse(res) rsp_code, rsp_result, rsp_data = result.get_omtask_rsp_data() - if rsp_code != 0 or (rsp_result.get('code') and rsp_result.get('code') != 0): - error_des = rsp_result.get('description') - error_sgt = rsp_result.get('suggestion') - err_msg = err_msg % (rsp_result.get('code'), error_des, error_sgt) + if rsp_code != 0 or (rsp_result.get("code") and rsp_result.get("code") != 0): + error_des = rsp_result.get("description") + error_sgt = rsp_result.get("suggestion") + err_msg = err_msg % (rsp_result.get("code"), error_des, error_sgt) raise Exception(err_msg) return rsp_data @@ -77,7 +82,9 @@ class DRStatusCheck(object): self.dm_user = self.dr_deploy_params.get("dm_user") dm_pwd = self.dr_deploy_params.get("dm_pwd") if os.path.exists(DR_DEPLOY_PARAM): - self.kmc_decrypt = CApiWrapper(primary_keystore=PRIMARY_KEYSTORE, standby_keystore=STANDBY_KEYSTORE) + self.kmc_decrypt = CApiWrapper( + primary_keystore=PRIMARY_KEYSTORE, standby_keystore=STANDBY_KEYSTORE + ) self.kmc_decrypt.initialize() try: self.dm_pwd = self.kmc_decrypt.decrypt(dm_pwd) @@ -86,9 +93,13 @@ class DRStatusCheck(object): return self.kmc_decrypt.finalize() # 恢复环境变量，避免cms命令执行失败 - split_env = os.environ['LD_LIBRARY_PATH'].split(":") - filtered_env = [single_env for single_env in split_env if "/opt/cantian/dbstor/lib" not in single_env] - os.environ['LD_LIBRARY_PATH'] = ":".join(filtered_env) + split_env = os.environ["LD_LIBRARY_PATH"].split(":") + filtered_env = [ + single_env + for single_env in split_env + if "/opt/cantian/dbstor/lib" not in single_env + ] + os.environ["LD_LIBRARY_PATH"] = ":".join(filtered_env) self.rest_client = RestClient((self.dm_ip, self.dm_user, self.dm_pwd)) self.rest_client.login() @@ -100,7 +111,7 @@ class DRStatusCheck(object): "device_id": self.remote_device_id, "url": Constant.QUERY_SYSTEM_INFO.replace("{deviceId}", "xxx"), "method": "GET", - "body": {} + "body": {}, } res = self.rest_client.normal_request(url, data=data, method="post") err_msg = "Failed to query remote storage system info" @@ -126,7 +137,10 @@ class DRStatusCheck(object): :param remote_device_id: 远端设备id :return:list """ - url = Constant.QUERY_REMOTE_DEVICE_INFO.format(deviceId=self.device_id) + f"/{remote_device_id}" + url = ( + Constant.QUERY_REMOTE_DEVICE_INFO.format(deviceId=self.device_id) + + f"/{remote_device_id}" + ) res = self.rest_client.normal_request(url, "get") err_msg = "Failed to query remote device info" remote_device_info = self.result_parse(err_msg, res) @@ -160,13 +174,17 @@ class DRStatusCheck(object): return get_status(running_status, VstorePairRunningStatus) def query_rep_link_status(self): - url = Constant.IP_LINK.format(deviceId=self.device_id) + \ - "?DEVICEID=%s&LINKUSAGE=true&range=[0-10]" % self.remote_device_id + url = ( + Constant.IP_LINK.format(deviceId=self.device_id) + + "?DEVICEID=%s&LINKUSAGE=true&range=[0-10]" % self.remote_device_id + ) res = self.rest_client.normal_request(url, "get") err_msg = "Failed to rep link info" ip_links = self.result_parse(err_msg, res) - url = Constant.FC_LINK.format(deviceId=self.device_id) + \ - "?DEVICEID=%s&LINKUSAGE=true&range=[0-10]" % self.remote_device_id + url = ( + Constant.FC_LINK.format(deviceId=self.device_id) + + "?DEVICEID=%s&LINKUSAGE=true&range=[0-10]" % self.remote_device_id + ) res = self.rest_client.normal_request(url, "get") err_msg = "Failed to rep link info" fc_links = self.result_parse(err_msg, res) @@ -216,7 +234,7 @@ class DRStatusCheck(object): "remote_device_status": "Abnormal", "rep_link_status": "Abnormal", "metro_domain_status": "Abnormal", - "metro_vstore_status": "Abnormal" + "metro_vstore_status": "Abnormal", } self.remote_device_id = self.dr_deploy_params.get("remote_device_id") hyper_domain_id = self.dr_deploy_params.get("hyper_domain_id") @@ -248,7 +266,9 @@ class DRStatusCheck(object): except Exception as err: metro_domain_status = "Abnormal" try: - metro_vstore_status = self.query_hyper_metro_vstore_pair_info(vstore_pair_id) + metro_vstore_status = self.query_hyper_metro_vstore_pair_info( + vstore_pair_id + ) except Exception as err: metro_vstore_status = "Abnormal" try: @@ -265,6 +285,6 @@ class DRStatusCheck(object): "remote_device_status": remote_device_status, "rep_link_status": rep_link_status, "metro_domain_status": metro_domain_status, - "metro_vstore_status": metro_vstore_status + "metro_vstore_status": metro_vstore_status, } return data diff --git a/ct_om/service/cantian_exporter/query_storage_info/response_parse.py b/ct_om/service/cantian_exporter/query_storage_info/response_parse.py index d803468f..404cbb14 100644 --- a/ct_om/service/cantian_exporter/query_storage_info/response_parse.py +++ b/ct_om/service/cantian_exporter/query_storage_info/response_parse.py @@ -1,5 +1,6 @@ # coding=utf-8 + class ResponseParse(object): def __init__(self, res): """ @@ -14,11 +15,11 @@ class ResponseParse(object): if status_code == 200: res = self.res.json() if "error" in res: - ret_result = res.get('error') + ret_result = res.get("error") else: - ret_result = res.get('result') - error_code = ret_result['code'] - error_des = ret_result['description'] + ret_result = res.get("result") + error_code = ret_result["code"] + error_des = ret_result["description"] if error_des is None or error_code == 0: error_des = "success" return status_code, int(error_code), error_des @@ -31,10 +32,10 @@ class ResponseParse(object): if status_code == 200: rsp_code = 0 if "error" in self.res.json(): - ret_result = self.res.json().get('error') + ret_result = self.res.json().get("error") if "result" in self.res.json(): - ret_result = self.res.json().get('result') - ret_data = self.res.json().get('data') + ret_result = self.res.json().get("result") + ret_data = self.res.json().get("data") return rsp_code, ret_result, ret_data def get_omtask_rsp_data(self): @@ -45,8 +46,8 @@ class ResponseParse(object): if status_code == 200: rsp_code = 0 if "error" in self.res.json(): - ret_result = self.res.json().get('error') + ret_result = self.res.json().get("error") if "result" in self.res.json(): - ret_result = self.res.json().get('result') - ret_data = self.res.json().get('data') + ret_result = self.res.json().get("result") + ret_data = self.res.json().get("data") return rsp_code, ret_result, ret_data diff --git a/ct_om/service/cantian_exporter/query_storage_info/rest_client.py b/ct_om/service/cantian_exporter/query_storage_info/rest_client.py index c7c9aa6a..4f806973 100644 --- a/ct_om/service/cantian_exporter/query_storage_info/rest_client.py +++ b/ct_om/service/cantian_exporter/query_storage_info/rest_client.py @@ -26,66 +26,80 @@ class RestClient: def gen_timestamp(): utc_now = datetime.utcnow() cur_time = utc_now.replace(tzinfo=timezone.utc).astimezone(tz=None) - return str(cur_time.strftime('%Y%m%d%H%M%S')) + return str(cur_time.strftime("%Y%m%d%H%M%S")) @staticmethod def exception_handler(err_msg=None, cur_mode=None): - err_info = '[current_mode] {}, [err_info] {}'.format(cur_mode, err_msg) + err_info = "[current_mode] {}, [err_info] {}".format(cur_mode, err_msg) raise ExecutionError(err_info) @staticmethod def response_parse(res_data): status_code = res_data.status_code - err_code, err_details = -1, 'failed' + err_code, err_details = -1, "failed" if status_code == 200: exec_res = res_data.json() - err_code, err_details = \ - exec_res.get('error').get('code'), exec_res.get('error').get('description') + err_code, err_details = exec_res.get("error").get("code"), exec_res.get( + "error" + ).get("description") return status_code, int(err_code), err_details def update_cookies(self, res): - res_body, set_cookie = res.json().get('data'), res.headers.get('Set-Cookie') + res_body, set_cookie = res.json().get("data"), res.headers.get("Set-Cookie") - self.token, self.device_id = res_body.get('iBaseToken'), res_body.get('deviceid') + self.token, self.device_id = res_body.get("iBaseToken"), res_body.get( + "deviceid" + ) - match_res = re.findall(r'session=ismsession=\w+;', set_cookie) + match_res = re.findall(r"session=ismsession=\w+;", set_cookie) if match_res: self.ism_session = match_res[0][:-1] - def make_header(self, content_type='application/json'): - header = {'Content-type': content_type} + def make_header(self, content_type="application/json"): + header = {"Content-type": content_type} if self.token: - header['iBaseToken'] = self.token + header["iBaseToken"] = self.token if self.ism_session: - header['Cookie'] = self.ism_session + header["Cookie"] = self.ism_session return header def login(self, keep_session=False): - url = '{}{}:{}{}'.format(RestElemConstant.HTTPS, self.ip_addr, RestElemConstant.PORT, RestElemConstant.LOGIN) + url = "{}{}:{}{}".format( + RestElemConstant.HTTPS, + self.ip_addr, + RestElemConstant.PORT, + RestElemConstant.LOGIN, + ) user_info = { - 'username': self.user_name, - 'password': self.passwd, - 'scope': 0, - 'loginMode': 3, - 'timeConversion': 0, - 'isEncrypt': 'false' + "username": self.user_name, + "password": self.passwd, + "scope": 0, + "loginMode": 3, + "timeConversion": 0, + "isEncrypt": "false", } login_header = { - 'Content-type': 'application/json', - 'Cookie': '__LANGUAGE_KEY__=zh-CN; __IBASE_LANGUAGE_KEY__=zh-CN' + "Content-type": "application/json", + "Cookie": "__LANGUAGE_KEY__=zh-CN; __IBASE_LANGUAGE_KEY__=zh-CN", } requests.packages.urllib3.disable_warnings() with requests.session() as session: - res = session.post(url, data=json.dumps(user_info), headers=login_header, verify=False) + res = session.post( + url, data=json.dumps(user_info), headers=login_header, verify=False + ) status_code, err_code, err_details = self.response_parse(res) if err_code: - err_msg = ('Login DM failed {}, status_code: {}, err_code: {}, ' - 'err_details: {}'.format(self.ip_addr, status_code, err_code, err_details)) + err_msg = ( + "Login DM failed {}, status_code: {}, err_code: {}, " + "err_details: {}".format( + self.ip_addr, status_code, err_code, err_details + ) + ) raise Exception(err_msg) self.update_cookies(res) @@ -98,7 +112,7 @@ class RestClient: def logout(self): url = RestElemConstant.LOGOUT.format(deviceId=self.device_id) - res = self.normal_request(url, 'delete') + res = self.normal_request(url, "delete") result = ResponseParse(res) status_code, error_code, error_des = result.get_res_code() if status_code != 200 or error_code != 0: @@ -117,26 +131,34 @@ class RestClient: url = Constant.HTTPS + self.ip_addr + ":" + Constant.PORT + url if keep_session: req = self.session - self.token = self.res_login.get('data').get('ibasetoken') + self.token = self.res_login.get("data").get("ibasetoken") else: req = requests.session() headers = self.make_header() with req as session: - if method == 'put': - res = session.put(url, data=data, headers=headers, verify=False, timeout=timeout) - elif method == 'post': - res = session.post(url, data=data, headers=headers, verify=False, timeout=timeout) - elif method == 'get': - res = session.get(url, data=data, headers=headers, verify=False, timeout=timeout) - elif method == 'delete': - res = session.delete(url, data=data, headers=headers, verify=False, timeout=timeout) + if method == "put": + res = session.put( + url, data=data, headers=headers, verify=False, timeout=timeout + ) + elif method == "post": + res = session.post( + url, data=data, headers=headers, verify=False, timeout=timeout + ) + elif method == "get": + res = session.get( + url, data=data, headers=headers, verify=False, timeout=timeout + ) + elif method == "delete": + res = session.delete( + url, data=data, headers=headers, verify=False, timeout=timeout + ) res.close() return res def read_helper(file_path): - with open(file_path, 'r', encoding='utf-8') as f_handler: + with open(file_path, "r", encoding="utf-8") as f_handler: deploy_data = f_handler.read() return deploy_data diff --git a/ct_om/service/cantian_exporter/query_storage_info/rest_constant.py b/ct_om/service/cantian_exporter/query_storage_info/rest_constant.py index 42c617d7..78a23f77 100644 --- a/ct_om/service/cantian_exporter/query_storage_info/rest_constant.py +++ b/ct_om/service/cantian_exporter/query_storage_info/rest_constant.py @@ -1,55 +1,81 @@ class Constant: - PORT = '8088' - HTTPS = 'https://' - LOGIN = '/deviceManager/rest/xxxxx/login' - LOGOUT = '/deviceManager/rest/{deviceId}/sessions' - QUERY_POOL = '/deviceManager/rest/{deviceId}/storagepool' - CREATE_FS = '/deviceManager/rest/{deviceId}/filesystem' - QUERY_FILE_SYSTEM_NUM = '/deviceManager/rest/{deviceId}/filesystem/count' - DELETE_FS = '/deviceManager/rest/{deviceId}/filesystem/{id}' - NFS_SERVICE = '/deviceManager/rest/{deviceId}/nfsservice' - NFS_SHARE_ADD = '/deviceManager/rest/{deviceId}/NFSSHARE' - NFS_SHARE_ADD_CLIENT = '/deviceManager/rest/{deviceId}/NFS_SHARE_AUTH_CLIENT' - NFS_SHARE_DELETE = '/deviceManager/rest/{deviceId}/NFSSHARE/{id}' - NFS_SHARE_DEL_CLIENT = '/deviceManager/rest/{deviceId}/NFS_SHARE_AUTH_CLIENT/{id}' - NFS_SHARE_QUERY = '/deviceManager/rest/{deviceId}/NFSSHARE' - QUERY_VSTORE = '/deviceManager/rest/{deviceId}/vstore/count' - CREATE_VSTORE = '/deviceManager/rest/{deviceId}/vstore' - DELETE_VSTORE = '/deviceManager/rest/{deviceId}/vstore/{id}' + PORT = "8088" + HTTPS = "https://" + LOGIN = "/deviceManager/rest/xxxxx/login" + LOGOUT = "/deviceManager/rest/{deviceId}/sessions" + QUERY_POOL = "/deviceManager/rest/{deviceId}/storagepool" + CREATE_FS = "/deviceManager/rest/{deviceId}/filesystem" + QUERY_FILE_SYSTEM_NUM = "/deviceManager/rest/{deviceId}/filesystem/count" + DELETE_FS = "/deviceManager/rest/{deviceId}/filesystem/{id}" + NFS_SERVICE = "/deviceManager/rest/{deviceId}/nfsservice" + NFS_SHARE_ADD = "/deviceManager/rest/{deviceId}/NFSSHARE" + NFS_SHARE_ADD_CLIENT = "/deviceManager/rest/{deviceId}/NFS_SHARE_AUTH_CLIENT" + NFS_SHARE_DELETE = "/deviceManager/rest/{deviceId}/NFSSHARE/{id}" + NFS_SHARE_DEL_CLIENT = "/deviceManager/rest/{deviceId}/NFS_SHARE_AUTH_CLIENT/{id}" + NFS_SHARE_QUERY = "/deviceManager/rest/{deviceId}/NFSSHARE" + QUERY_VSTORE = "/deviceManager/rest/{deviceId}/vstore/count" + CREATE_VSTORE = "/deviceManager/rest/{deviceId}/vstore" + DELETE_VSTORE = "/deviceManager/rest/{deviceId}/vstore/{id}" CREATE_LIF = "/deviceManager/rest/{deviceId}/lif" DELETE_LIF = "/deviceManager/rest/{deviceId}/lif?NAME={name}" CREATE_CLONE_FS = "/deviceManager/rest/{deviceId}/filesystem" SPLIT_CLONE_FS = "/deviceManager/rest/{deviceId}/clone_fs_split" CREATE_FSSNAPSHOT = "/deviceManager/rest/{deviceId}/fssnapshot" ROLLBACK_SNAPSHOT = "/deviceManager/rest/{deviceId}/fssnapshot/rollback_fssnapshot" - QUERY_ROLLBACK_SNAPSHOT_PROCESS = "/deviceManager/rest/{deviceId}/FSSNAPSHOT/" \ - "query_fs_snapshot_rollback?PARENTNAME={fs_name}" + QUERY_ROLLBACK_SNAPSHOT_PROCESS = ( + "/deviceManager/rest/{deviceId}/FSSNAPSHOT/" + "query_fs_snapshot_rollback?PARENTNAME={fs_name}" + ) QUERY_LOGIC_PORT_INFO = "/deviceManager/rest/{deviceId}/lif" # 容灾查询操作 QUERY_SYSTEM_INFO = "/deviceManager/rest/{deviceId}/system/" QUERY_REMOTE_DEVICE_INFO = "/deviceManager/rest/{deviceId}/remote_device" QUERY_LICENSE_FEATURE = "/deviceManager/rest/{deviceId}/license/feature" - QUERY_HYPER_METRO_FILE_SYSTEM_PAIR = "/deviceManager/rest/{deviceId}/HyperMetroPair/associate" - QUERY_REPLICATION_FILE_SYSTEM_PAIR = "/deviceManager/rest/{deviceId}/replicationpair/associate" - QUERY_FILESYSTEM_FOR_REPLICATION = "/deviceManager/rest/{deviceId}/filesystem_for_replication" + QUERY_HYPER_METRO_FILE_SYSTEM_PAIR = ( + "/deviceManager/rest/{deviceId}/HyperMetroPair/associate" + ) + QUERY_REPLICATION_FILE_SYSTEM_PAIR = ( + "/deviceManager/rest/{deviceId}/replicationpair/associate" + ) + QUERY_FILESYSTEM_FOR_REPLICATION = ( + "/deviceManager/rest/{deviceId}/filesystem_for_replication" + ) # 容灾搭建操作 HYPER_METRO_DOMAIN = "/deviceManager/rest/{deviceId}/FsHyperMetroDomain" HYPER_METRO_VSTORE_PAIR = "/deviceManager/rest/{deviceId}/vstore_pair" HYPER_METRO_FILESYSTEM_PAIR = "/deviceManager/rest/{deviceId}/HyperMetroPair" - SPLIT_REMOTE_REPLICATION_FILESYSTEM_PAIR = "/deviceManager/rest/{deviceId}/REPLICATIONPAIR/split" - SYNC_REMOTE_REPLICATION_FILESYSTEM_PAIR = "/deviceManager/rest/{deviceId}/REPLICATIONPAIR/sync" - REMOTE_REPLICATION_FILESYSTEM_PAIR_OPT = "/deviceManager/rest/{deviceId}/REPLICATIONPAIR/{id}" + SPLIT_REMOTE_REPLICATION_FILESYSTEM_PAIR = ( + "/deviceManager/rest/{deviceId}/REPLICATIONPAIR/split" + ) + SYNC_REMOTE_REPLICATION_FILESYSTEM_PAIR = ( + "/deviceManager/rest/{deviceId}/REPLICATIONPAIR/sync" + ) + REMOTE_REPLICATION_FILESYSTEM_PAIR_OPT = ( + "/deviceManager/rest/{deviceId}/REPLICATIONPAIR/{id}" + ) DELETE_HYPER_METRO_PAIR = "/deviceManager/rest/{deviceId}/HyperMetroPair/{id}" DELETE_HYPER_METRO_VSTORE_PAIR = "/deviceManager/rest/{deviceId}/vstore_pair/{id}" - SPLIT_FILESYSTEM_HYPER_METRO_DOMAIN = "/deviceManager/rest/{deviceId}/SplitFsHyperMetroDomain" - DELETE_FILESYSTEM_HYPER_METRO_DOMAIN = "/deviceManager/rest/{deviceId}/FsHyperMetroDomain/{id}" - CANCEL_SECONDARY_WRITE_LOCK = "/deviceManager/rest/{deviceId}/REPLICATIONPAIR/CANCEL_SECONDARY_WRITE_LOCK" - SET_SECONDARY_WRITE_LOCK = "/deviceManager/rest/{deviceId}/REPLICATIONPAIR/SET_SECONDARY_WRITE_LOCK" - SWAP_ROLE_FS_HYPER_METRO_DOMAIN = "/deviceManager/rest/{deviceId}/SwapRoleFsHyperMetroDomain" + SPLIT_FILESYSTEM_HYPER_METRO_DOMAIN = ( + "/deviceManager/rest/{deviceId}/SplitFsHyperMetroDomain" + ) + DELETE_FILESYSTEM_HYPER_METRO_DOMAIN = ( + "/deviceManager/rest/{deviceId}/FsHyperMetroDomain/{id}" + ) + CANCEL_SECONDARY_WRITE_LOCK = ( + "/deviceManager/rest/{deviceId}/REPLICATIONPAIR/CANCEL_SECONDARY_WRITE_LOCK" + ) + SET_SECONDARY_WRITE_LOCK = ( + "/deviceManager/rest/{deviceId}/REPLICATIONPAIR/SET_SECONDARY_WRITE_LOCK" + ) + SWAP_ROLE_FS_HYPER_METRO_DOMAIN = ( + "/deviceManager/rest/{deviceId}/SwapRoleFsHyperMetroDomain" + ) SWAP_ROLE_REPLICATION_PAIR = "/deviceManager/rest/{deviceId}/REPLICATIONPAIR/switch" - CHANGE_FS_HYPER_METRO_DOMAIN_SECOND_ACCESS = "/deviceManager/rest/{deviceId}/ChangeFsHyperMetroDomainSecondAccess" + CHANGE_FS_HYPER_METRO_DOMAIN_SECOND_ACCESS = ( + "/deviceManager/rest/{deviceId}/ChangeFsHyperMetroDomainSecondAccess" + ) JOIN_FS_HYPER_METRO_DOMAIN = "/deviceManager/rest/{deviceId}/JoinFsHyperMetroDomain" QUERY_HYPER_METRO_PAIR = "/deviceManager/rest/{deviceId}/HyperMetroPair/{id}" IP_LINK = "/deviceManager/rest/{deviceId}/iplink" @@ -66,9 +92,9 @@ CANTIAN_DOMAIN_PREFIX = "CantianDomain_%s" class HealthStatus: - Normal = "1" # 正常 - Faulty = "2" # 故障 - Invalid = "14" # 失效 + Normal = "1" # 正常 + Faulty = "2" # 故障 + Invalid = "14" # 失效 class SystemRunningStatus: @@ -80,21 +106,21 @@ class SystemRunningStatus: class RemoteDeviceStatus: - LinkUp = "10" # 已连接 - LinkDown = "11" # 未连接 - Disabled = "31" # 已禁用 - Connecting = "101" # 正在连接 - AirGapLinkDown = "118" # Air Gap断开 + LinkUp = "10" # 已连接 + LinkDown = "11" # 未连接 + Disabled = "31" # 已禁用 + Connecting = "101" # 正在连接 + AirGapLinkDown = "118" # Air Gap断开 class ReplicationRunningStatus: - Normal = "1" # 正常 - Synchronizing = "23" # 正在同步 - TobeRecovered = "33" # 待恢复 - Interrupted = "34" # 异常断开 - Split = "26" # 已分裂 - Invalid = "35" # 失效 - Standby = "110" # 备用 + Normal = "1" # 正常 + Synchronizing = "23" # 正在同步 + TobeRecovered = "33" # 待恢复 + Interrupted = "34" # 异常断开 + Split = "26" # 已分裂 + Invalid = "35" # 失效 + Standby = "110" # 备用 class FilesystemRunningStatus: @@ -105,54 +131,54 @@ class FilesystemRunningStatus: class MetroDomainRunningStatus: - Normal = "0" # 正常 - Recovering = "1" # 恢复中 - Faulty = "2" # 故障 - Split = "3" # 分裂 - ForceStarted = "4" # 强制拉起 - Invalid = "5" # 失效 + Normal = "0" # 正常 + Recovering = "1" # 恢复中 + Faulty = "2" # 故障 + Split = "3" # 分裂 + ForceStarted = "4" # 强制拉起 + Invalid = "5" # 失效 class VstorePairRunningStatus: - Normal = "1" # 正常 - Unsynchronized = "25" # 未同步 - Split = "26" # 分裂 - Invalid = "35" # 失效 - ForceStarted = "93" # 强制启动 + Normal = "1" # 正常 + Unsynchronized = "25" # 未同步 + Split = "26" # 分裂 + Invalid = "35" # 失效 + ForceStarted = "93" # 强制启动 class FilesystemPairRunningStatus: - Normal = "1" # 正常 - Synchronizing = "23" # 同步中 - Invalid = "35" # 失效 - Paused = "41" # 暂停 - ForceStarted = "93" # 强制启动 + Normal = "1" # 正常 + Synchronizing = "23" # 同步中 + Invalid = "35" # 失效 + Paused = "41" # 暂停 + ForceStarted = "93" # 强制启动 ToBeSynchronized = "100" # 待同步 - Creating = "119" # 创建中 + Creating = "119" # 创建中 class SecresAccess: - AccessDenied = "1" # 禁止访问 - ReadOnly = "2" # 只读 - ReadAndWrite = "3" # 读写 + AccessDenied = "1" # 禁止访问 + ReadOnly = "2" # 只读 + ReadAndWrite = "3" # 读写 class PoolStatus: - PreCopy = "14" # 预拷贝 - Rebuilt = "16" # 重构 - Online = "27" # 在线 - Offline = "28" # 离线 - Balancing = "32" # 正在均衡 - Initializing = "53" # 初始化中 - Deleting = "106" # 删除中 + PreCopy = "14" # 预拷贝 + Rebuilt = "16" # 重构 + Online = "27" # 在线 + Offline = "28" # 离线 + Balancing = "32" # 正在均衡 + Initializing = "53" # 初始化中 + Deleting = "106" # 删除中 class PoolHealth: - Normal = "1" # 正常 - Faulty = "2" # 故障 - Degraded = "5" # 降级 + Normal = "1" # 正常 + Faulty = "2" # 故障 + Degraded = "5" # 降级 class DomainAccess: ReadAndWrite = "3" # 读写 - ReadOnly = "1" # 只读 \ No newline at end of file + ReadOnly = "1" # 只读 diff --git a/ct_om/service/ctcli/display_as_table.py b/ct_om/service/ctcli/display_as_table.py index 2ab9b321..a7c74e7c 100644 --- a/ct_om/service/ctcli/display_as_table.py +++ b/ct_om/service/ctcli/display_as_table.py @@ -4,8 +4,12 @@ class DisplayAsTable: @staticmethod def print_exception_info(err_type, err_detail=""): - print(("[result] command execution failed, [err_type] {}, [err_detail] {}, " - "use [ctctl help] to get the commands we support.").format(err_type, err_detail)) + print( + ( + "[result] command execution failed, [err_type] {}, [err_detail] {}, " + "use [ctctl help] to get the commands we support." + ).format(err_type, err_detail) + ) @staticmethod def print_info(dict_info, key_header, key_max_len): @@ -13,11 +17,15 @@ class DisplayAsTable: print("\r") for idx, val in enumerate(key_header): dict_item_size = key_max_len[val] + 4 - data_str = str(dict_item[val]).center(dict_item_size, '-' if dict_item[val] == '-' else ' ') + data_str = str(dict_item[val]).center( + dict_item_size, "-" if dict_item[val] == "-" else " " + ) icon = "|" if dict_item[val] == "-": icon = "+" - data_str = (icon if idx == 0 else '') + data_str[1: len(data_str)] + icon + data_str = ( + (icon if idx == 0 else "") + data_str[1 : len(data_str)] + icon + ) print(data_str, end="") print("\r") @@ -57,7 +65,7 @@ class DisplayAsTable: for idx, val in enumerate(key_header): item_size = key_max_len_dict.get(val) + 4 data_str = val.center(item_size) - data_str = ('|' if idx == 0 else '') + data_str[1:len(data_str)] + '|' + data_str = ("|" if idx == 0 else "") + data_str[1 : len(data_str)] + "|" print(data_str, end="") self.print_info(list_info, list(key_header), key_max_len_dict) diff --git a/ct_om/service/ctcli/handle_info.py b/ct_om/service/ctcli/handle_info.py index 3fc3be29..1340506b 100644 --- a/ct_om/service/ctcli/handle_info.py +++ b/ct_om/service/ctcli/handle_info.py @@ -27,16 +27,16 @@ class HandleInfo: self.print_table = DisplayAsTable() self.commands_info = json_data_reader(str(Path(cur_abs_path, "commands.json"))) self.input_process_method = { - 'common': self.common_params_handler, - 'logs_collection': self.logs_collection_handler + "common": self.common_params_handler, + "logs_collection": self.logs_collection_handler, } @staticmethod def _reg_match_dir(log_dir): - reg_string = r'(/[-\w~]+)' + reg_string = r"(/[-\w~]+)" reg_res = re.findall(reg_string, log_dir) - connector = ''.join(reg_res) - return connector == log_dir or '{}/'.format(connector) == log_dir + connector = "".join(reg_res) + return connector == log_dir or "{}/".format(connector) == log_dir @staticmethod def basic_params_check(right_params, input_params): @@ -56,7 +56,9 @@ class HandleInfo: def format_param_check(self, input_params): format_key = input_params.get("format") if not format_key or format_key not in ("json", "table"): - err_msg = "format param type must be one of ('json', table), not {}".format(format_key) + err_msg = "format param type must be one of ('json', table), not {}".format( + format_key + ) return err_msg, ABNORMAL_STATE if format_key == "table": self.has_format = True @@ -67,18 +69,25 @@ class HandleInfo: self.data_to_uds.update({"command": str(self.cmd)}) str_params = [param for param in params if "=" in param] - params_to_uds = {param.split("=")[0]: param.split("=")[1] for param in str_params} + params_to_uds = { + param.split("=")[0]: param.split("=")[1] for param in str_params + } - if self.commands_info.get(self.cmd).get("params check") == 'True': - check_res, err_code = self.basic_params_check(self.commands_info.get(self.cmd).get("check value"), - params_to_uds) + if self.commands_info.get(self.cmd).get("params check") == "True": + check_res, err_code = self.basic_params_check( + self.commands_info.get(self.cmd).get("check value"), params_to_uds + ) if err_code: - return self._params_exception_handler(err_type="params name check error", err_detail=check_res) + return self._params_exception_handler( + err_type="params name check error", err_detail=check_res + ) if "format" in params_to_uds: check_res, err_code = self.format_param_check(params_to_uds) if err_code: - return self._params_exception_handler(err_type="format param error", err_detail=check_res) + return self._params_exception_handler( + err_type="format param error", err_detail=check_res + ) params_to_uds.pop("format") @@ -86,41 +95,66 @@ class HandleInfo: return self.data_to_uds, NORMAL_STATE def logs_collection_handler(self, input_params): - params_item = [(item_idx, item_val) for item_idx, item_val in enumerate(input_params) if '=' in item_val] + params_item = [ + (item_idx, item_val) + for item_idx, item_val in enumerate(input_params) + if "=" in item_val + ] if len(params_item) > 2 or len(params_item) <= 1: - return self._params_exception_handler(err_type="wrong logs collection params number", - err_detail="The input parameter names " - "can only be 'log_dir' and 'type'") + return self._params_exception_handler( + err_type="wrong logs collection params number", + err_detail="The input parameter names " + "can only be 'log_dir' and 'type'", + ) head_idx, tail_idx = params_item[0][0], params_item[-1][0] if (tail_idx - head_idx) > 1: - err_msg = "input log_dir '{}' is invalid".format(" ".join(input_params[head_idx:tail_idx])) - return self._params_exception_handler(err_type="invalid input log dir", err_detail=err_msg) + err_msg = "input log_dir '{}' is invalid".format( + " ".join(input_params[head_idx:tail_idx]) + ) + return self._params_exception_handler( + err_type="invalid input log dir", err_detail=err_msg + ) log_dir_param = input_params[head_idx] - type_param = ''.join(input_params[tail_idx:]) - params_to_uds = {param.split("=")[0]: param.split("=")[1] for param in [log_dir_param, type_param]} + type_param = "".join(input_params[tail_idx:]) + params_to_uds = { + param.split("=")[0]: param.split("=")[1] + for param in [log_dir_param, type_param] + } - check_res, err_code = self.basic_params_check(self.commands_info.get(self.cmd).get("check value"), - params_to_uds.keys()) + check_res, err_code = self.basic_params_check( + self.commands_info.get(self.cmd).get("check value"), params_to_uds.keys() + ) if err_code: - return self._params_exception_handler(err_type="params name check error", err_detail=check_res) + return self._params_exception_handler( + err_type="params name check error", err_detail=check_res + ) - log_dir, log_collection_type = params_to_uds.get('log_dir'), params_to_uds.get('type') + log_dir, log_collection_type = params_to_uds.get("log_dir"), params_to_uds.get( + "type" + ) if not self._reg_match_dir(log_dir): - return self._params_exception_handler(err_type="input log dir invalid", - err_detail="input log dir '{}' invalid".format(log_dir)) - if log_collection_type not in ('all', 'recent'): - return self._params_exception_handler(err_type="logs collection type error", - err_detail="logs collection type must be one of [all, recent]" - " not '{}'".format(log_collection_type)) + return self._params_exception_handler( + err_type="input log dir invalid", + err_detail="input log dir '{}' invalid".format(log_dir), + ) + if log_collection_type not in ("all", "recent"): + return self._params_exception_handler( + err_type="logs collection type error", + err_detail="logs collection type must be one of [all, recent]" + " not '{}'".format(log_collection_type), + ) self.data_to_uds = {"command": self.cmd, "param": params_to_uds} return self.data_to_uds, NORMAL_STATE def is_unilateral_execution(self): """commands processed only on the ctcli side""" - info_dict = {cmd: cmd_detail.get("description") for cmd, cmd_detail in self.commands_info.items()} + info_dict = { + cmd: cmd_detail.get("description") + for cmd, cmd_detail in self.commands_info.items() + } self.print_table.display_single_table(info_dict, mode="help") def input_params_handler(self, params): @@ -132,11 +166,13 @@ class HandleInfo: if not params_item: self.cmd = " ".join(params) else: - self.cmd = " ".join(params[:params_item[0][0]]) + self.cmd = " ".join(params[: params_item[0][0]]) if self.cmd not in self.commands_info: - return self._params_exception_handler(err_type="input commands error", - err_detail="'ctctl {}' not supported".format(self.cmd)) + return self._params_exception_handler( + err_type="input commands error", + err_detail="'ctctl {}' not supported".format(self.cmd), + ) handler_method = self.commands_info.get(self.cmd).get("handler") return self.input_process_method.get(handler_method)(params) @@ -155,9 +191,11 @@ class HandleInfo: else: self.receipt_data_processing(receipt_info.get("data")) else: - err_msg = "[ctctl {}] may fail, the receipt info is {}, which is an empty string, " \ - "does not conform to the format returned by ctmgr, " \ - "check logs for more details.".format(self.cmd, ctmgr_info) + err_msg = ( + "[ctctl {}] may fail, the receipt info is {}, which is an empty string, " + "does not conform to the format returned by ctmgr, " + "check logs for more details.".format(self.cmd, ctmgr_info) + ) self.print_table.display_table(err_msg) def receipt_data_processing(self, data_info): diff --git a/ct_om/service/ctcli/main.py b/ct_om/service/ctcli/main.py index 2e7471c4..6ffdbfa9 100644 --- a/ct_om/service/ctcli/main.py +++ b/ct_om/service/ctcli/main.py @@ -11,6 +11,6 @@ def main(params): handle_info.receipt_info_handler(receive_data) -if __name__ == '__main__': +if __name__ == "__main__": input_params = sys.argv[1:] main(input_params) diff --git a/ct_om/service/ctcli/params_factory/param_config.py b/ct_om/service/ctcli/params_factory/param_config.py index b45c4cc7..10896d33 100644 --- a/ct_om/service/ctcli/params_factory/param_config.py +++ b/ct_om/service/ctcli/params_factory/param_config.py @@ -9,18 +9,22 @@ def log_query(log_path, *args): if args[0]: return info tmp = json.dumps(info) - tmp = "{" + tmp[1: len(tmp) - 1] + "}" + tmp = "{" + tmp[1 : len(tmp) - 1] + "}" return tmp def collection_logs(receipt_info, *args): if not receipt_info: - return "[ctctl collection logs] may fail, the receipt info is {}, " \ - "which is an empty string".format(receipt_info) + return ( + "[ctctl collection logs] may fail, the receipt info is {}, " + "which is an empty string".format(receipt_info) + ) if "timed out" in receipt_info: - return "start log collection successful, " \ - "use [ctctl logs progress query] to get current collection progress" + return ( + "start log collection successful, " + "use [ctctl logs progress query] to get current collection progress" + ) return receipt_info @@ -29,9 +33,11 @@ def logs_progress_query(log_progress_path, *args): if log_progress_path: load_info = json_file_reader(log_progress_path) if not load_info: - return "no log collection information, " \ - "may be the log is still being generated or " \ - "no log collection has been performed" + return ( + "no log collection information, " + "may be the log is still being generated or " + "no log collection has been performed" + ) return [json.loads(item) for item in load_info.split("\n") if item] @@ -42,5 +48,5 @@ PARAM_PREPARE = { "help": "direct execution", "log query": log_query, "collection logs": collection_logs, - "logs progress query": logs_progress_query + "logs progress query": logs_progress_query, } diff --git a/ct_om/service/ctcli/params_factory/tools.py b/ct_om/service/ctcli/params_factory/tools.py index 17e0b7ea..d2511a7f 100644 --- a/ct_om/service/ctcli/params_factory/tools.py +++ b/ct_om/service/ctcli/params_factory/tools.py @@ -14,13 +14,17 @@ def table_log_reader(log_path, size=0): tmp_list_info = [item.split(",") for item in ori_log_info.split("\n") if item] key_set = ("client_pid", "client_uid", "command", "running_status", "begin_time") - info_dict = [{item.split("=")[0].strip(): item.split("=")[1][1:-1] - for item in log - if item.split("=")[0].strip() in key_set} - for log in tmp_list_info] + info_dict = [ + { + item.split("=")[0].strip(): item.split("=")[1][1:-1] + for item in log + if item.split("=")[0].strip() in key_set + } + for log in tmp_list_info + ] if size: res_size = min(size, len(info_dict)) - return info_dict[len(info_dict) - res_size:] + return info_dict[len(info_dict) - res_size :] return info_dict diff --git a/ct_om/service/ctcli/uds_client.py b/ct_om/service/ctcli/uds_client.py index 892eebbf..02f785a7 100644 --- a/ct_om/service/ctcli/uds_client.py +++ b/ct_om/service/ctcli/uds_client.py @@ -1,7 +1,7 @@ import json import socket -SERVER_ADDRESS = '/opt/cantian/ct_om/service/ct_om.sock' +SERVER_ADDRESS = "/opt/cantian/ct_om/service/ct_om.sock" SOCKET_TYPE = socket.SOCK_STREAM SOCKET_FAMILY = socket.AF_UNIX RECEIVE_DATA_SIZE = 1024 diff --git a/ct_om/service/ctmgr/checker/collection_logs_checker.py b/ct_om/service/ctmgr/checker/collection_logs_checker.py index 8fdcf8a1..57441bed 100644 --- a/ct_om/service/ctmgr/checker/collection_logs_checker.py +++ b/ct_om/service/ctmgr/checker/collection_logs_checker.py @@ -5,9 +5,13 @@ class LogEnumCheck: @staticmethod def check(input_params_dict): - _type = input_params_dict.get('type') - if _type not in ('recent', 'all'): - TASK_LOG.error('collection logs fail, type must be recent or all, type is {}'.format(_type)) + _type = input_params_dict.get("type") + if _type not in ("recent", "all"): + TASK_LOG.error( + "collection logs fail, type must be recent or all, type is {}".format( + _type + ) + ) return False return True diff --git a/ct_om/service/ctmgr/checkers.py b/ct_om/service/ctmgr/checkers.py index cd997831..031add63 100644 --- a/ct_om/service/ctmgr/checkers.py +++ b/ct_om/service/ctmgr/checkers.py @@ -7,7 +7,7 @@ def check_none(value): def check_required(input_dict, check_item, check_rule): - check_value = input_dict.get(check_item, '') + check_value = input_dict.get(check_item, "") if check_rule: return len(str(check_value)) > 0 @@ -16,13 +16,13 @@ def check_required(input_dict, check_item, check_rule): def check_type(input_dict, check_item, check_rule): type_list = { - 'int': int, - 'string': str, - 'list': list, - 'tuple': tuple, - 'dict': dict, - 'float': float, - 'bool': bool, + "int": int, + "string": str, + "list": list, + "tuple": tuple, + "dict": dict, + "float": float, + "bool": bool, } check_value = input_dict.get(check_item, None) if check_none(check_value): @@ -58,7 +58,7 @@ def check_enum(input_dict, check_item, check_rule): if check_none(check_value): return True - enum_list = check_rule.split('|') + enum_list = check_rule.split("|") if isinstance(check_value, bool): check_value = str(check_value).upper() @@ -72,7 +72,7 @@ def check_int_range(input_dict, check_item, check_rule): if check_none(check_value): return True - min_value, max_value = [int(x) for x in check_rule.split('~')] + min_value, max_value = [int(x) for x in check_rule.split("~")] return min_value <= int(check_value) <= max_value @@ -82,7 +82,7 @@ def check_str_length_range(input_dict, check_item, check_rule): if check_none(check_value): return True - min_value, max_value = [int(x) for x in check_rule.split('~')] + min_value, max_value = [int(x) for x in check_rule.split("~")] return min_value <= len(str(check_value)) <= max_value @@ -127,10 +127,12 @@ def check_str_list_range(input_dict, check_item, check_rule): if check_none(check_value): return True - if not check_str_list(input_dict=input_dict, check_item=check_item, check_rule=check_rule): + if not check_str_list( + input_dict=input_dict, check_item=check_item, check_rule=check_rule + ): return False - min_value, max_value = [int(x) for x in check_rule.split('~')] + min_value, max_value = [int(x) for x in check_rule.split("~")] return min_value <= len(check_value) <= max_value @@ -164,10 +166,12 @@ def check_int_list_range(input_dict, check_item, check_rule): if check_none(check_value): return True - if not check_int_list(input_dict=input_dict, check_item=check_item, check_rule=check_rule): + if not check_int_list( + input_dict=input_dict, check_item=check_item, check_rule=check_rule + ): return False - min_value, max_value = [int(x) for x in check_rule.split('~')] + min_value, max_value = [int(x) for x in check_rule.split("~")] return min_value <= len(check_value) <= max_value @@ -191,7 +195,7 @@ def check_array_length(input_dict, check_item, check_rule): if not isinstance(check_value, list): return False - min_value, max_value = [int(x) for x in check_rule.split('~')] + min_value, max_value = [int(x) for x in check_rule.split("~")] return min_value <= len(check_value) <= max_value @@ -210,5 +214,5 @@ CHECKER = { "intList": check_int_list, "intListRange": check_int_list_range, "nonRepeatList": check_non_repeat_list, - "arrayLength": check_array_length -} \ No newline at end of file + "arrayLength": check_array_length, +} diff --git a/ct_om/service/ctmgr/common/common_tool.py b/ct_om/service/ctmgr/common/common_tool.py index 8c7b4be5..1544884b 100644 --- a/ct_om/service/ctmgr/common/common_tool.py +++ b/ct_om/service/ctmgr/common/common_tool.py @@ -5,4 +5,4 @@ class TimeTool: @staticmethod def get_current_time(): - return time.strftime('%Y-%m-%d %H:%M:%S', time.localtime(time.time())) + return time.strftime("%Y-%m-%d %H:%M:%S", time.localtime(time.time())) diff --git a/ct_om/service/ctmgr/common/output_tool.py b/ct_om/service/ctmgr/common/output_tool.py index 25b663c3..acc162b3 100644 --- a/ct_om/service/ctmgr/common/output_tool.py +++ b/ct_om/service/ctmgr/common/output_tool.py @@ -22,13 +22,8 @@ class CommonResult: def __str__(self): result = { - "data": { - "ctmgr_common_output": self.output_data - }, - "error": { - "code": int(self.error_code), - "description": self.description - } + "data": {"ctmgr_common_output": self.output_data}, + "error": {"code": int(self.error_code), "description": self.description}, } return json.dumps(result) @@ -43,7 +38,7 @@ class CommonResult: self.description = str(description) -if __name__ == '__main__': +if __name__ == "__main__": res = CommonResult() res.set_output_data("task_obj.task_execute(input_params_dict)") res.set_output_data("doing XXX success") diff --git a/ct_om/service/ctmgr/log_tool/om_log.py b/ct_om/service/ctmgr/log_tool/om_log.py index 079a14df..52752857 100644 --- a/ct_om/service/ctmgr/log_tool/om_log.py +++ b/ct_om/service/ctmgr/log_tool/om_log.py @@ -10,10 +10,26 @@ LOG_DIR_MODE_OCT = 0o700 LOG_FILE_MODE = 640 LOG_FILE_MODE_OCT = 0o640 SENSITIVE_STR = [ - 'Password', 'passWord', 'PASSWORD', 'password', 'Pswd', - 'PSWD', 'pwd', 'signature', 'HmacSHA256', 'newPasswd', - 'private', 'certfile', 'secret', 'token', 'Token', 'pswd', - 'passwd', 'mysql -u', 'session', 'cookie' + "Password", + "passWord", + "PASSWORD", + "password", + "Pswd", + "PSWD", + "pwd", + "signature", + "HmacSHA256", + "newPasswd", + "private", + "certfile", + "secret", + "token", + "Token", + "pswd", + "passwd", + "mysql -u", + "session", + "cookie", ] @@ -29,7 +45,7 @@ def _get_log_file_path(project): return os.path.join(logger_dir, "{}.log".format(project)) - return '' + return "" class DefaultLogFilter(logging.Filter): @@ -53,8 +69,10 @@ def setup(project_name): log_path = _get_log_file_path(project_name) if log_path: file_log = handlers.RotatingFileHandler( - log_path, maxBytes=log_config.get("log_file_max_size"), - backupCount=log_config.get("log_file_backup_count")) + log_path, + maxBytes=log_config.get("log_file_max_size"), + backupCount=log_config.get("log_file_backup_count"), + ) log_root.addHandler(file_log) log_root.addFilter(DefaultLogFilter()) @@ -65,7 +83,9 @@ def setup(project_name): handler.setFormatter( logging.Formatter( fmt=log_config.get("logging_context_format_string"), - datefmt=log_config.get("log_date_format"))) + datefmt=log_config.get("log_date_format"), + ) + ) if log_config.get("debug"): log_root.setLevel(logging.DEBUG) @@ -84,7 +104,10 @@ class LOGGER: """ 审计日志需要参数： client_pid, client_uid, cmd, result, begin_time, finish_time """ - def __init__(self, client_pid, client_uid, begin_time, finish_time='', request_time=''): + + def __init__( + self, client_pid, client_uid, begin_time, finish_time="", request_time="" + ): self.begin_time = begin_time self.request_time = request_time self.finish_time = finish_time @@ -98,9 +121,18 @@ class LOGGER: self.finish_time = finish_time def format_log_message(self, cmd, result): - return 'client_pid=[{}],client_uid=[{}],command=[{}],running_status=[{}],begin_time=[{}],request_time=[{}],' \ - 'finish_time=[{}]'.format( - self.client_pid, self.client_uid, cmd, result, self.begin_time, self.request_time, self.finish_time) + return ( + "client_pid=[{}],client_uid=[{}],command=[{}],running_status=[{}],begin_time=[{}],request_time=[{}]," + "finish_time=[{}]".format( + self.client_pid, + self.client_uid, + cmd, + result, + self.begin_time, + self.request_time, + self.finish_time, + ) + ) def info(self, cmd, result): LOG.info(self.format_log_message(cmd, result)) diff --git a/ct_om/service/ctmgr/log_tool/om_log_config.py b/ct_om/service/ctmgr/log_tool/om_log_config.py index d34ba51c..fc493757 100644 --- a/ct_om/service/ctmgr/log_tool/om_log_config.py +++ b/ct_om/service/ctmgr/log_tool/om_log_config.py @@ -8,15 +8,15 @@ CONSOLE_CONF = { "log": { "use_syslog": False, "debug": False, - "log_dir": str(Path('{}/../ctmgr_log'.format(str(dir_name)))), + "log_dir": str(Path("{}/../ctmgr_log".format(str(dir_name)))), "log_file_max_size": 1048576, "log_file_backup_count": 5, "log_date_format": "%Y-%m-%d %H:%M:%S", "logging_default_format_string": "time=[%(asctime)s],level=[%(levelname)s],pid=[%(process)d]," - "thread=[%(threadName)s],tid=[%(thread)d]," - "file=[%(filename)s:%(lineno)d %(funcName)s],%(message)s", + "thread=[%(threadName)s],tid=[%(thread)d]," + "file=[%(filename)s:%(lineno)d %(funcName)s],%(message)s", "logging_context_format_string": "time=[%(asctime)s],level=[%(levelname)s],pid=[%(process)d]," - "thread=[%(threadName)s],tid=[%(thread)d]," - "file=[%(filename)s:%(lineno)d %(funcName)s],%(message)s" + "thread=[%(threadName)s],tid=[%(thread)d]," + "file=[%(filename)s:%(lineno)d %(funcName)s],%(message)s", } } diff --git a/ct_om/service/ctmgr/logs_collection/execute.py b/ct_om/service/ctmgr/logs_collection/execute.py index 6c01b65e..779a8ad2 100644 --- a/ct_om/service/ctmgr/logs_collection/execute.py +++ b/ct_om/service/ctmgr/logs_collection/execute.py @@ -5,16 +5,20 @@ import argparse from functools import wraps from pathlib import Path -sys.path.append('/opt/cantian/ct_om/service') -sys.path.append('/opt/cantian/ct_om/service/ctmgr') +sys.path.append("/opt/cantian/ct_om/service") +sys.path.append("/opt/cantian/ct_om/service/ctmgr") from ctmgr.logs_collection.tools import LockFile from ctmgr.log_tool.om_log import LOGS_COLLECTION as LOG from ctmgr.logs_collection.logs_collection import LogsCollection -parser = argparse.ArgumentParser(description='Required parameters for running this script') -parser.add_argument('--path', '-p', help='path to save all tar.gz bags', required=True) -parser.add_argument('--type', '-t', help='recent: recent logs, all: all logs', required=True) +parser = argparse.ArgumentParser( + description="Required parameters for running this script" +) +parser.add_argument("--path", "-p", help="path to save all tar.gz bags", required=True) +parser.add_argument( + "--type", "-t", help="recent: recent logs, all: all logs", required=True +) p_args = parser.parse_args() cur_abs_path, _ = os.path.split(os.path.abspath(__file__)) @@ -24,6 +28,7 @@ def exter_attack(func): @wraps(func) def wrapper(*args, **kwargs): return func(*args, **kwargs) + return wrapper @@ -36,13 +41,17 @@ def main(target_path, mode): try: lock_file.lock(file_handler) except IOError: - LOG.info('there is already a log collection process, another process[pid:{}]' - 'try to lock the file, but failed.'.format(os.getpid())) - print('there is already a log collection process, please try again later...') + LOG.info( + "there is already a log collection process, another process[pid:{}]" + "try to lock the file, but failed.".format(os.getpid()) + ) + print("there is already a log collection process, please try again later...") else: - LOG.info('success to lock the file, current process id:{}'.format(os.getpid())) + LOG.info("success to lock the file, current process id:{}".format(os.getpid())) execute(mode, target_path) - print('log collection ends, use [ctctl logs progress query] to get log collection details') + print( + "log collection ends, use [ctctl logs progress query] to get log collection details" + ) finally: lock_file.unlock(file_handler) file_handler.close() diff --git a/ct_om/service/ctmgr/logs_collection/logs_collection.py b/ct_om/service/ctmgr/logs_collection/logs_collection.py index 295cb16d..c15ffd2e 100644 --- a/ct_om/service/ctmgr/logs_collection/logs_collection.py +++ b/ct_om/service/ctmgr/logs_collection/logs_collection.py @@ -16,7 +16,7 @@ cur_abs_path, _ = os.path.split(os.path.abspath(__file__)) def json_data_reader(data_path): - with open(data_path, 'r', encoding='utf-8') as file: + with open(data_path, "r", encoding="utf-8") as file: info = file.read() return json.loads(info) @@ -32,16 +32,24 @@ class LogsCollection: self.max_gather_vol = 0 self.sh_task_time_out = 600 self.record_progress = RecordLogPackingProgress(cur_abs_path) - self.config_info = json_data_reader(str(Path(cur_abs_path, 'config.json'))) + self.config_info = json_data_reader(str(Path(cur_abs_path, "config.json"))) @staticmethod def get_cur_timestamp(flag=None): utc_now = datetime.utcnow() - if flag == 'name': - return utc_now.replace(tzinfo=timezone.utc).astimezone(tz=None).strftime('%Y%m%d%H%M%S') + if flag == "name": + return ( + utc_now.replace(tzinfo=timezone.utc) + .astimezone(tz=None) + .strftime("%Y%m%d%H%M%S") + ) - return utc_now.replace(tzinfo=timezone.utc).astimezone(tz=None).strftime('%Y-%m-%d %H:%M:%S') + return ( + utc_now.replace(tzinfo=timezone.utc) + .astimezone(tz=None) + .strftime("%Y-%m-%d %H:%M:%S") + ) @staticmethod def path_authority_judgment(file_path): @@ -56,7 +64,7 @@ class LogsCollection: @staticmethod def reg_handler(log_name_prefix, match_string): """匹配出待采集的文件""" - if match_string.startswith(log_name_prefix) and match_string.endswith('tar.gz'): + if match_string.startswith(log_name_prefix) and match_string.endswith("tar.gz"): return True return False @@ -68,13 +76,13 @@ class LogsCollection: LOG.info("Modify file mode before collection.") cmd = "sudo /opt/cantian/action/change_log_priority.sh" res = self.shell_task(cmd, "modify file mode") - if res != 'success': + if res != "success": err_msg = "Modify files mode failed." LOG.error(err_msg) LOG.info("Modify file success.") def packing_files(self, log_file_list, tar_name, mode): - """ 将当前模块日志归档为一个压缩文件 + """将当前模块日志归档为一个压缩文件 :param log_file_list: 当前模块日志列表 :param tar_name: 压缩文件名 @@ -84,23 +92,29 @@ class LogsCollection: for log_file in log_file_list: log_names.append(log_file) log_directory, log_name = os.path.split(log_file) - log_name_prefix = log_name.split('.')[0] - archive_logs = [(item, get_file_creation_time(str(Path(log_directory, item)))) - for item in os.listdir(log_directory) - if self.reg_handler(log_name_prefix, item)] + log_name_prefix = log_name.split(".")[0] + archive_logs = [ + (item, get_file_creation_time(str(Path(log_directory, item)))) + for item in os.listdir(log_directory) + if self.reg_handler(log_name_prefix, item) + ] # 近期日志采集场景，仅采集实时日志和归档日志 - if mode == 'recent': + if mode == "recent": archive_logs.sort(key=lambda x: (x[1], x[0]), reverse=True) cur_size = int(os.path.getsize(str(Path(log_directory, log_name)))) for archive_log_name, _ in archive_logs: if cur_size >= self.max_gather_vol: break - cur_size += int(os.path.getsize(str(Path(log_directory, archive_log_name)))) + cur_size += int( + os.path.getsize(str(Path(log_directory, archive_log_name))) + ) log_names.append(os.path.join(log_directory, archive_log_name)) else: - log_names.extend([os.path.join(log_directory, name) for name, _ in archive_logs]) + log_names.extend( + [os.path.join(log_directory, name) for name, _ in archive_logs] + ) - with tarfile.open(f'{tar_name}', 'w:gz') as tar: + with tarfile.open(f"{tar_name}", "w:gz") as tar: for pack_name in log_names: tar.add(pack_name) @@ -111,39 +125,61 @@ class LogsCollection: :param task_type: 用于区分当前方法的用途 :return: 字符串，'success' or 'fail' """ - res_state = 'success' + res_state = "success" try: - proc = subprocess.Popen(shlex.split(spec_cmd), stdout=subprocess.PIPE, shell=False) + proc = subprocess.Popen( + shlex.split(spec_cmd), stdout=subprocess.PIPE, shell=False + ) except Exception as err: - LOG.error('[{}] execute cmd: {} failed, err_msg: {}'.format(task_type, spec_cmd, err)) - return 'fail' + LOG.error( + "[{}] execute cmd: {} failed, err_msg: {}".format( + task_type, spec_cmd, err + ) + ) + return "fail" try: res, state = proc.communicate(timeout=self.sh_task_time_out) except Exception as err: - LOG.error('[{}] execute cmd: {} failed, err_msg: {}'.format(task_type, spec_cmd, err)) - return 'fail' + LOG.error( + "[{}] execute cmd: {} failed, err_msg: {}".format( + task_type, spec_cmd, err + ) + ) + return "fail" if state: - LOG.error('[{}] execute cmd: {} failed, res_err: {}'.format(task_type, spec_cmd, res.decode('utf-8'))) - res_state = 'fail' + LOG.error( + "[{}] execute cmd: {} failed, res_err: {}".format( + task_type, spec_cmd, res.decode("utf-8") + ) + ) + res_state = "fail" else: - LOG.info('[{}] execute cmd: {} succeed, res_msg: {}'.format(task_type, spec_cmd, res.decode('utf-8'))) + LOG.info( + "[{}] execute cmd: {} succeed, res_msg: {}".format( + task_type, spec_cmd, res.decode("utf-8") + ) + ) return res_state def generate_logs(self, item, main_path, mode): res = [] - prefix_to_type = {'py': 'python3', 'sh': 'sh'} - script_type, script_path = item.get('script_type'), item.get('script_path') - size = item.get('size') - file_dir = item.get('dir') - tar_name = item.get('tar_name') + prefix_to_type = {"py": "python3", "sh": "sh"} + script_type, script_path = item.get("script_type"), item.get("script_path") + size = item.get("size") + file_dir = item.get("dir") + tar_name = item.get("tar_name") if mode == "recent": file_dir = file_dir + "| head -n %s" % size - cmd = (prefix_to_type.get(script_type) + ' ' + script_path) % (main_path, file_dir, tar_name) - state = self.shell_task(cmd, 'generate logs') + cmd = (prefix_to_type.get(script_type) + " " + script_path) % ( + main_path, + file_dir, + tar_name, + ) + state = self.shell_task(cmd, "generate logs") res.append(state) return res @@ -152,8 +188,12 @@ class LogsCollection: if os.path.exists(target_path): self.removing_dirs(target_path) - self.record_progress.record_cur_progress((cur_time, cur_time), 'for details see logs', - 'for details see logs', ('fail', 'time_out', 'None')) + self.record_progress.record_cur_progress( + (cur_time, cur_time), + "for details see logs", + "for details see logs", + ("fail", "time_out", "None"), + ) def removing_dirs(self, dir_to_remove): """递归删除包含输入目录在内的所有文件 @@ -171,44 +211,65 @@ class LogsCollection: os.remove(dir_to_remove) def sub_module_packing(self, item, main_path, idx, mode): - start_time, collect_state = self.get_cur_timestamp(), 'done' + start_time, collect_state = self.get_cur_timestamp(), "done" - self.max_gather_vol = int(item.get('size')) * self.packing_ratio * pow(1024, 2) - tar_name, generate_type = item.get('tar_name'), item.get('generate_type') + self.max_gather_vol = int(item.get("size")) * self.packing_ratio * pow(1024, 2) + tar_name, generate_type = item.get("tar_name"), item.get("generate_type") is_repeat_generate = item.get("is_repeat_generate", False) log_name = None if generate_type == "script generated": - log_file_dir = item.get('dir') + log_file_dir = item.get("dir") log_directory, log_name = os.path.split(log_file_dir) gen_res = self.generate_logs(item, main_path, mode) statistics_res = Counter(gen_res) - LOG.info(f"[generate logs ends], end_time:{self.get_cur_timestamp()}, all:{len(gen_res)}, " - f"success: {statistics_res.get('success', 0)}, fail: {statistics_res.get('fail', 0)}") + LOG.info( + f"[generate logs ends], end_time:{self.get_cur_timestamp()}, all:{len(gen_res)}, " + f"success: {statistics_res.get('success', 0)}, fail: {statistics_res.get('fail', 0)}" + ) else: - log_file_list = [item.get('dir')] if not is_repeat_generate \ - else glob.glob(item.get('dir'), recursive=True) + log_file_list = ( + [item.get("dir")] + if not is_repeat_generate + else glob.glob(item.get("dir"), recursive=True) + ) for log_file_dir in log_file_list: # 分离日志目录和日志名 log_directory, log_name = os.path.split(log_file_dir) - LOG.info('[submodule log collection starts] child_module: {}, ' - 'generate_type: {}'.format(log_name, generate_type)) + LOG.info( + "[submodule log collection starts] child_module: {}, " + "generate_type: {}".format(log_name, generate_type) + ) if not os.path.exists(log_file_dir): LOG.error( - 'log_source_path: {} does not exist log collection failed and exited'.format(log_file_dir)) - self.record_cur_progress(("done", 'not exist'), (idx, generate_type, log_name), start_time) + "log_source_path: {} does not exist log collection failed and exited".format( + log_file_dir + ) + ) + self.record_cur_progress( + ("done", "not exist"), + (idx, generate_type, log_name), + start_time, + ) return False - if not self.path_authority_judgment(log_file_dir) or not self.path_authority_judgment(log_directory): + if not self.path_authority_judgment( + log_file_dir + ) or not self.path_authority_judgment(log_directory): LOG.error( - "log_file_dir: '{}' or log_content: '{}' permission denied".format(log_file_dir, log_directory)) + "log_file_dir: '{}' or log_content: '{}' permission denied".format( + log_file_dir, log_directory + ) + ) self.pre_execute() sub_module_tar_path = os.path.join(main_path, tar_name) self.packing_files(log_file_list, sub_module_tar_path, mode) - self.record_cur_progress((collect_state, 'None'), (idx, generate_type, log_name), start_time) + self.record_cur_progress( + (collect_state, "None"), (idx, generate_type, log_name), start_time + ) return True def record_cur_progress(self, collect_state, submodule_info, start_time): @@ -216,14 +277,20 @@ class LogsCollection: idx, generate_type, name_pre = submodule_info end_time = self.get_cur_timestamp() try: - cur_percent = str('%.2f' % ((idx + 1) * 100 / len(self.config_info))) + '%' + cur_percent = str("%.2f" % ((idx + 1) * 100 / len(self.config_info))) + "%" except ZeroDivisionError: - cur_percent = 'error' - self.record_progress.record_cur_progress((start_time, end_time), - name_pre, generate_type, (state, err_type, cur_percent)) - - LOG.info('[submodule log collection ends] child_module: {}, pack_status: {}, ' - 'cur_progress: {}'.format(name_pre, state, cur_percent)) + cur_percent = "error" + self.record_progress.record_cur_progress( + (start_time, end_time), + name_pre, + generate_type, + (state, err_type, cur_percent), + ) + + LOG.info( + "[submodule log collection ends] child_module: {}, pack_status: {}, " + "cur_progress: {}".format(name_pre, state, cur_percent) + ) @timeout(1800) def execute(self, target_path, mode): @@ -231,9 +298,11 @@ class LogsCollection: target: 由用户指定的压缩包存放路径 mode: all:打包近期日志， recent：全量打包 """ - LOG.info('[logs collection] starts') + LOG.info("[logs collection] starts") self.pre_execute() - LOG.info(f'[generate logs starts], start_time:{self.get_cur_timestamp()}, mode: {mode}') + LOG.info( + f"[generate logs starts], start_time:{self.get_cur_timestamp()}, mode: {mode}" + ) if not os.path.exists(target_path): try: @@ -245,8 +314,8 @@ class LogsCollection: finally: os.chmod(target_path, 0o750) - time_stamp = self.get_cur_timestamp(flag='name') - final_tar_file_name = 'cantian_log_{}.tar.gz'.format(time_stamp) + time_stamp = self.get_cur_timestamp(flag="name") + final_tar_file_name = "cantian_log_{}.tar.gz".format(time_stamp) main_path = os.path.join(target_path, time_stamp) try: os.mkdir(main_path) @@ -255,17 +324,25 @@ class LogsCollection: LOG.error(err_msg) raise IOError(err_msg) from err - res = Counter([str(self.sub_module_packing(item, main_path, idx, mode)) - for idx, item in enumerate(self.config_info)]) + res = Counter( + [ + str(self.sub_module_packing(item, main_path, idx, mode)) + for idx, item in enumerate(self.config_info) + ] + ) # 将各模块日志压缩为一个归档日志 final_tar_file_path = str(Path(target_path, final_tar_file_name)) os.chdir(main_path) - with tarfile.open(f'{final_tar_file_path}', 'w:gz') as tar: + with tarfile.open(f"{final_tar_file_path}", "w:gz") as tar: for file_name in os.listdir(main_path): tar.add(file_name) os.chdir(cur_abs_path) # 修改归档日志权限 os.chmod(final_tar_file_path, 0o440) self.removing_dirs(main_path) - LOG.info('[logs collection ends], success: {}, fail: {}'.format(res.get('True', 0), res.get('False', 0))) + LOG.info( + "[logs collection ends], success: {}, fail: {}".format( + res.get("True", 0), res.get("False", 0) + ) + ) diff --git a/ct_om/service/ctmgr/logs_collection/tools.py b/ct_om/service/ctmgr/logs_collection/tools.py index bd9da6af..53c9342b 100644 --- a/ct_om/service/ctmgr/logs_collection/tools.py +++ b/ct_om/service/ctmgr/logs_collection/tools.py @@ -12,9 +12,14 @@ def timeout(sec): @functools.wraps(func) def wrapped_func(*args): def handle_timeout(signum, frame): - err_msg = 'err_type: execution timeout, err_msg: ' \ - 'cmd "ctctl collection logs" timed out after {} minutes'.format(sec // 60) + err_msg = ( + "err_type: execution timeout, err_msg: " + 'cmd "ctctl collection logs" timed out after {} minutes'.format( + sec // 60 + ) + ) raise TimeoutError(err_msg) + signal.signal(signal.SIGALRM, handle_timeout) signal.alarm(sec) try: @@ -22,12 +27,15 @@ def timeout(sec): finally: signal.alarm(0) return result + return wrapped_func + return decorator class LockFile: """持锁状态下对文件标识符进行修改""" + @staticmethod def lock(handle): fcntl.flock(handle, fcntl.LOCK_EX | fcntl.LOCK_NB) @@ -45,7 +53,9 @@ class RecordLogPackingProgress: def clear_original_data(self): modes = stat.S_IWRITE | stat.S_IRUSR flags = os.O_WRONLY | os.O_TRUNC | os.O_CREAT - with os.fdopen(os.open(self.record_file, flags, modes), "w", encoding="utf-8") as file: + with os.fdopen( + os.open(self.record_file, flags, modes), "w", encoding="utf-8" + ) as file: file.truncate() def record_cur_progress(self, time_interval, module, generate_type, state): @@ -55,12 +65,14 @@ class RecordLogPackingProgress: info_dict = dict() modes = stat.S_IWUSR | stat.S_IRUSR flags = os.O_WRONLY | os.O_CREAT - with os.fdopen(os.open(self.record_file, flags, modes), "a", encoding="utf-8") as file: + with os.fdopen( + os.open(self.record_file, flags, modes), "a", encoding="utf-8" + ) as file: info_dict.update({"start_time": start_time}) info_dict.update({"end_time": end_time}) info_dict.update({"log_name": module}) info_dict.update({"generate_type": generate_type}) info_dict.update({"status": status}) - info_dict.update({"err_type": err_type}) + info_dict.update({"err_type": err_type}) info_dict.update({"percentage": percentage}) file.write(json.dumps(info_dict) + "\n") diff --git a/ct_om/service/ctmgr/task.py b/ct_om/service/ctmgr/task.py index 668488f7..3c487dd8 100644 --- a/ct_om/service/ctmgr/task.py +++ b/ct_om/service/ctmgr/task.py @@ -15,14 +15,18 @@ task_dir = {} # 用于保存任务类，格式：{命令: 命令对应的类} def build_uds_obj(recv_commend, commend_data): - param_check = commend_data.get('paramCheck', {}) - rest_obj = UdsTask(task_name=recv_commend, handler='cantian', param_check_dict=param_check) + param_check = commend_data.get("paramCheck", {}) + rest_obj = UdsTask( + task_name=recv_commend, handler="cantian", param_check_dict=param_check + ) - param_check = commend_data.get('checkFunc') + param_check = commend_data.get("checkFunc") try: check_func = get_check_func(param_check) except Exception as error: - DEPLOY_LOG.error('paramCheck format error in tasks.json element: {}'.format(recv_commend)) + DEPLOY_LOG.error( + "paramCheck format error in tasks.json element: {}".format(recv_commend) + ) raise error if check_func: @@ -32,15 +36,22 @@ def build_uds_obj(recv_commend, commend_data): def build_cmd_obj(recv_commend, commend_data): - cmd_line = commend_data.get('cmd') - param_check = commend_data.get('paramCheck', {}) - rest_obj = CmdTask(task_name=recv_commend, handler='cmd', cmd_line=cmd_line, param_check_dict=param_check) - - param_check = commend_data.get('checkFunc') + cmd_line = commend_data.get("cmd") + param_check = commend_data.get("paramCheck", {}) + rest_obj = CmdTask( + task_name=recv_commend, + handler="cmd", + cmd_line=cmd_line, + param_check_dict=param_check, + ) + + param_check = commend_data.get("checkFunc") try: check_func = get_check_func(param_check) except Exception as error: - DEPLOY_LOG.error('paramCheck format error in tasks.json element: {}'.format(recv_commend)) + DEPLOY_LOG.error( + "paramCheck format error in tasks.json element: {}".format(recv_commend) + ) raise error if check_func: @@ -50,17 +61,24 @@ def build_cmd_obj(recv_commend, commend_data): def build_shell_obj(recv_commend, commend_data): - file_path = commend_data.get('filePath') - sh_input = commend_data.get('sh_input') - param_check = commend_data.get('paramCheck', {}) - rest_obj = ShellTask(task_name=recv_commend, handler='shell', file_path=file_path, sh_input=sh_input, - param_check_dict=param_check) - - param_check = commend_data.get('checkFunc') + file_path = commend_data.get("filePath") + sh_input = commend_data.get("sh_input") + param_check = commend_data.get("paramCheck", {}) + rest_obj = ShellTask( + task_name=recv_commend, + handler="shell", + file_path=file_path, + sh_input=sh_input, + param_check_dict=param_check, + ) + + param_check = commend_data.get("checkFunc") try: check_func = get_check_func(param_check) except Exception as error: - DEPLOY_LOG.error('paramCheck format error in tasks.json element: {}'.format(recv_commend)) + DEPLOY_LOG.error( + "paramCheck format error in tasks.json element: {}".format(recv_commend) + ) raise error if check_func: @@ -70,17 +88,24 @@ def build_shell_obj(recv_commend, commend_data): def build_py_obj(recv_commend, commend_data): - file_path = commend_data.get('filePath') - py_input = commend_data.get('py_input') - param_check = commend_data.get('paramCheck', {}) - rest_obj = PyTask(task_name=recv_commend, handler='py', file_path=file_path, py_input=py_input, - param_check_dict=param_check) - - param_check = commend_data.get('checkFunc') + file_path = commend_data.get("filePath") + py_input = commend_data.get("py_input") + param_check = commend_data.get("paramCheck", {}) + rest_obj = PyTask( + task_name=recv_commend, + handler="py", + file_path=file_path, + py_input=py_input, + param_check_dict=param_check, + ) + + param_check = commend_data.get("checkFunc") try: check_func = get_check_func(param_check) except Exception as error: - DEPLOY_LOG.error('paramCheck format error in tasks.json element: {}'.format(recv_commend)) + DEPLOY_LOG.error( + "paramCheck format error in tasks.json element: {}".format(recv_commend) + ) raise error if check_func: @@ -90,46 +115,61 @@ def build_py_obj(recv_commend, commend_data): def build_audit_obj(recv_commend, commend_data): - file_path = commend_data.get('filePath') - param_check = commend_data.get('paramCheck', {}) - rest_obj = AuditTask(task_name=recv_commend, handler='audit_py', file_path=file_path, py_input='', - params_check_dict=param_check) + file_path = commend_data.get("filePath") + param_check = commend_data.get("paramCheck", {}) + rest_obj = AuditTask( + task_name=recv_commend, + handler="audit_py", + file_path=file_path, + py_input="", + params_check_dict=param_check, + ) return rest_obj def build_log_obj(recv_commend, commend_data): - param_check = commend_data.get('paramCheck', {}) - rest_obj = QueryLogTask(task_name=recv_commend, handler='log', params_check_dict=param_check) + param_check = commend_data.get("paramCheck", {}) + rest_obj = QueryLogTask( + task_name=recv_commend, handler="log", params_check_dict=param_check + ) return rest_obj def build_inspection_obj(recv_commend, commend_data): - param_check = commend_data.get('paramCheck', {}) - rest_odj = InspectionTask(task_name=recv_commend, handler="inspection_py", params_check_dict=param_check) + param_check = commend_data.get("paramCheck", {}) + rest_odj = InspectionTask( + task_name=recv_commend, handler="inspection_py", params_check_dict=param_check + ) return rest_odj def build_log_progress_query_obj(recv_commend, commend_data): - param_check = commend_data.get('paramCheck', {}) - query_obj = QueryLogProgressTask(task_name=recv_commend, handler='log_progress', params_check_dict=param_check) + param_check = commend_data.get("paramCheck", {}) + query_obj = QueryLogProgressTask( + task_name=recv_commend, handler="log_progress", params_check_dict=param_check + ) return query_obj def build_logs_collection_obj(recv_commend, commend_data): - log_path, input_param = commend_data.get('filePath'), commend_data.get('py_input') - param_check = commend_data.get('paramCheck', {}) - logs_collect_obj = LogsCollectionTask(task_name=recv_commend, - handler='log_py', - file_path=log_path, - py_input=input_param, - params_check_dict=param_check) - - param_check = commend_data.get('checkFunc') + log_path, input_param = commend_data.get("filePath"), commend_data.get("py_input") + param_check = commend_data.get("paramCheck", {}) + logs_collect_obj = LogsCollectionTask( + task_name=recv_commend, + handler="log_py", + file_path=log_path, + py_input=input_param, + params_check_dict=param_check, + ) + + param_check = commend_data.get("checkFunc") try: check_func = get_check_func(param_check) except Exception as error: - DEPLOY_LOG.error('paramCheck format error in tasks.json element: {}'.format(recv_commend)) + DEPLOY_LOG.error( + "paramCheck format error in tasks.json element: {}".format(recv_commend) + ) raise error if check_func: @@ -139,48 +179,56 @@ def build_logs_collection_obj(recv_commend, commend_data): def get_check_func(param_check): - check_path = param_check.split('.') + check_path = param_check.split(".") if len(check_path) == 2: check_file_name, check_class_name = check_path - check_func = getattr(import_module('checker.{}'.format(check_file_name)), check_class_name).check + check_func = getattr( + import_module("checker.{}".format(check_file_name)), check_class_name + ).check return check_func elif len(check_path) == 1 and not check_path[0]: - return '' + return "" else: - DEPLOY_LOG.error('filePath: {} in tasks.json with error format'.format(param_check)) - raise Exception('filePath: {} in tasks.json with error format'.format(param_check)) + DEPLOY_LOG.error( + "filePath: {} in tasks.json with error format".format(param_check) + ) + raise Exception( + "filePath: {} in tasks.json with error format".format(param_check) + ) def load_tasks(): dir_name, _ = os.path.split(os.path.abspath(__file__)) - task_config = str(Path('{}/tasks.json'.format(dir_name))) - with open(task_config, 'r', encoding='utf8') as file_path: + task_config = str(Path("{}/tasks.json".format(dir_name))) + with open(task_config, "r", encoding="utf8") as file_path: json_data = json.load(file_path) for recv_commend, commend_data in json_data.items(): - handler = commend_data.get('handler') + handler = commend_data.get("handler") try: - task_dir[recv_commend] = BUILD_OBJ_FUNCS.get(handler)(recv_commend, commend_data) + task_dir[recv_commend] = BUILD_OBJ_FUNCS.get(handler)( + recv_commend, commend_data + ) except Exception as error: - DEPLOY_LOG.error('load task.json fail, error: {}'.format(error)) + DEPLOY_LOG.error("load task.json fail, error: {}".format(error)) raise error DEPLOY_LOG.info("load task.json success") BUILD_OBJ_FUNCS = { - 'cantian': build_uds_obj, - 'cmd': build_cmd_obj, - 'shell': build_shell_obj, - 'py': build_py_obj, - 'audit_py': build_audit_obj, - 'log': build_log_obj, - 'inspection_py': build_inspection_obj, - 'log_progress': build_log_progress_query_obj, - 'log_py': build_logs_collection_obj + "cantian": build_uds_obj, + "cmd": build_cmd_obj, + "shell": build_shell_obj, + "py": build_py_obj, + "audit_py": build_audit_obj, + "log": build_log_obj, + "inspection_py": build_inspection_obj, + "log_progress": build_log_progress_query_obj, + "log_py": build_logs_collection_obj, } -if __name__ == '__main__': - input_params = {'key1': 'path/path/xxx.log', 'key2': 0} +if __name__ == "__main__": + input_params = {"key1": "path/path/xxx.log", "key2": 0} load_tasks() diff --git a/ct_om/service/ctmgr/task_obj.py b/ct_om/service/ctmgr/task_obj.py index 6fc0b344..5cd5362d 100644 --- a/ct_om/service/ctmgr/task_obj.py +++ b/ct_om/service/ctmgr/task_obj.py @@ -10,8 +10,8 @@ from checkers import CHECKER from common.output_tool import CommonResult from common.common_tool import TimeTool -FAIL = 'fail' -SUCCESS = 'success' +FAIL = "fail" +SUCCESS = "success" class TASK(metaclass=ABCMeta): @@ -37,7 +37,9 @@ class TASK(metaclass=ABCMeta): TASK_LOG.error("check function %s not exist" % param_key) continue - if not CHECKER.get(param_key)(input_params_dict, check_key, param_value): + if not CHECKER.get(param_key)( + input_params_dict, check_key, param_value + ): TASK_LOG.error("%s check %s not pass" % (check_key, param_key)) return False @@ -70,16 +72,20 @@ class UdsTask(TASK): try: res = uds_client.client_socket(output_data) except Exception as error: - result = CommonResult(output_data='cmd {} send uds request failed'.format(self.task_name), error_code=1, - description='cmd {} send uds request failed \n [ERROR] {}'.format( - self.task_name, error)) + result = CommonResult( + output_data="cmd {} send uds request failed".format(self.task_name), + error_code=1, + description="cmd {} send uds request failed \n [ERROR] {}".format( + self.task_name, error + ), + ) return result result.set_output_data(res) task_logger.set_finish_time(TimeTool.get_current_time()) - error_code = eval(result.__str__()).get('error', {}).get('code', '') - if str(error_code) != '0': + error_code = eval(result.__str__()).get("error", {}).get("code", "") + if str(error_code) != "0": task_logger.error(self.task_name, FAIL) else: task_logger.info(self.task_name, SUCCESS) @@ -101,22 +107,31 @@ class CmdTask(TASK): result = CommonResult() for param_key in input_params.keys(): - self.cmd_line = self.cmd_line.replace('${%s}' % param_key, str(input_params[param_key])) + self.cmd_line = self.cmd_line.replace( + "${%s}" % param_key, str(input_params[param_key]) + ) task_logger.set_request_time(TimeTool.get_current_time()) try: - command_result = subprocess.Popen(self.cmd_line.split(' '), stdout=subprocess.PIPE, shell=False) + command_result = subprocess.Popen( + self.cmd_line.split(" "), stdout=subprocess.PIPE, shell=False + ) except Exception as error: task_logger.set_finish_time(TimeTool.get_current_time()) task_logger.info(self.task_name, FAIL) - result = CommonResult(output_data='{} execute cmd {} failed'.format(self.task_name, self.cmd_line), - error_code=1, - description='{} execute cmd {} failed \n [ERROR] {}'.format( - self.task_name, self.cmd_line, error)) + result = CommonResult( + output_data="{} execute cmd {} failed".format( + self.task_name, self.cmd_line + ), + error_code=1, + description="{} execute cmd {} failed \n [ERROR] {}".format( + self.task_name, self.cmd_line, error + ), + ) return result - res = command_result.communicate(timeout=self.time_out)[0].decode('utf-8') + res = command_result.communicate(timeout=self.time_out)[0].decode("utf-8") result.set_output_data(res) task_logger.set_finish_time(TimeTool.get_current_time()) task_logger.info(self.task_name, SUCCESS) @@ -139,8 +154,11 @@ class ShellTask(TASK): self.time_out = 5 if not self.basic_check(): - DEPLOY_LOG.error('[error] {} not exist, init ShellTask failed, please check param in cmd {}'.format( - self.file_path, self.task_name)) + DEPLOY_LOG.error( + "[error] {} not exist, init ShellTask failed, please check param in cmd {}".format( + self.file_path, self.task_name + ) + ) raise Exception def basic_check(self): @@ -148,28 +166,35 @@ class ShellTask(TASK): def task_execute(self, input_params, task_logger): params_format_save = self.sh_input - cmd_line = 'sh {}'.format(self.file_path) + cmd_line = "sh {}".format(self.file_path) for param_key in input_params.keys(): - self.sh_input = self.sh_input.replace('${%s}' % param_key, str(input_params[param_key])) - cmd_line = cmd_line + ' ' + str(self.sh_input) + self.sh_input = self.sh_input.replace( + "${%s}" % param_key, str(input_params[param_key]) + ) + cmd_line = cmd_line + " " + str(self.sh_input) result = CommonResult() task_logger.set_request_time(TimeTool.get_current_time()) try: - sh_result = subprocess.Popen(cmd_line.split(' '), stdout=subprocess.PIPE, shell=False) + sh_result = subprocess.Popen( + cmd_line.split(" "), stdout=subprocess.PIPE, shell=False + ) except Exception as error: task_logger.set_finish_time(TimeTool.get_current_time()) task_logger.info(self.task_name, FAIL) - result.set_output_data('call shell file: {} failed'.format(self.file_path)) + result.set_output_data("call shell file: {} failed".format(self.file_path)) result.set_error_code(1) - result.set_description('{} call shell file: {} with params: {} failed \n [ERROR] {}'.format( - self.task_name, self.file_path, self.sh_input, error)) + result.set_description( + "{} call shell file: {} with params: {} failed \n [ERROR] {}".format( + self.task_name, self.file_path, self.sh_input, error + ) + ) self.sh_input = params_format_save return result - res = sh_result.communicate(timeout=self.time_out)[0].decode('utf-8') + res = sh_result.communicate(timeout=self.time_out)[0].decode("utf-8") result.set_output_data(res) task_logger.set_finish_time(TimeTool.get_current_time()) task_logger.info(self.task_name, SUCCESS) @@ -192,8 +217,11 @@ class PyTask(TASK): self.py_input = py_input self.time_out = 5 if not self.basic_check(): - DEPLOY_LOG.error('[error] {} not exist, init ShellTask failed, please check param in cmd {}'.format( - self.file_path, self.task_name)) + DEPLOY_LOG.error( + "[error] {} not exist, init ShellTask failed, please check param in cmd {}".format( + self.file_path, self.task_name + ) + ) raise Exception def basic_check(self): @@ -201,40 +229,62 @@ class PyTask(TASK): def task_execute(self, input_params, task_logger): params_format_save = self.py_input - cmd_line = 'python3 {}'.format(self.file_path) + cmd_line = "python3 {}".format(self.file_path) for param_key in input_params.keys(): - self.py_input = self.py_input.replace('${%s}' % param_key, str(input_params[param_key])) - cmd_line = cmd_line + ' ' + str(self.py_input) + self.py_input = self.py_input.replace( + "${%s}" % param_key, str(input_params[param_key]) + ) + cmd_line = cmd_line + " " + str(self.py_input) result = CommonResult() - TASK_LOG.info('task: {} calling py file: {}, using cmd: {}'.format(self.task_name, self.file_path, cmd_line)) + TASK_LOG.info( + "task: {} calling py file: {}, using cmd: {}".format( + self.task_name, self.file_path, cmd_line + ) + ) task_logger.set_request_time(TimeTool.get_current_time()) try: - py_result = subprocess.Popen(cmd_line.split(' '), stdout=subprocess.PIPE, shell=False) + py_result = subprocess.Popen( + cmd_line.split(" "), stdout=subprocess.PIPE, shell=False + ) except Exception as error: - TASK_LOG.error('task: {} calling py file: {}, using cmd: {} fail, error: {}'.format( - self.task_name, self.file_path, cmd_line, error)) + TASK_LOG.error( + "task: {} calling py file: {}, using cmd: {} fail, error: {}".format( + self.task_name, self.file_path, cmd_line, error + ) + ) task_logger.set_finish_time(TimeTool.get_current_time()) task_logger.info(self.task_name, FAIL) - result = CommonResult(output_data='{} call py file: {} failed'.format(self.task_name, self.file_path), - error_code=1, - description='{} call py file: {} with params: {} failed \n [ERROR] {}'.format( - self.task_name, self.file_path, self.py_input, error)) + result = CommonResult( + output_data="{} call py file: {} failed".format( + self.task_name, self.file_path + ), + error_code=1, + description="{} call py file: {} with params: {} failed \n [ERROR] {}".format( + self.task_name, self.file_path, self.py_input, error + ), + ) self.py_input = params_format_save return result try: - res = py_result.communicate(timeout=self.time_out)[0].decode('utf-8') + res = py_result.communicate(timeout=self.time_out)[0].decode("utf-8") except Exception as error: - TASK_LOG.error('task: {} calling py file: {} fail to obtain result, error: {}'.format( - self.task_name, self.file_path, error)) + TASK_LOG.error( + "task: {} calling py file: {} fail to obtain result, error: {}".format( + self.task_name, self.file_path, error + ) + ) task_logger.set_finish_time(TimeTool.get_current_time()) task_logger.info(self.task_name, FAIL) - result = CommonResult(output_data='execute file: {} failed'.format(self.file_path), - error_code=1, - description='{} call py file: {} with params: {} failed \n [ERROR] {}'.format( - self.task_name, self.file_path, self.py_input, error)) + result = CommonResult( + output_data="execute file: {} failed".format(self.file_path), + error_code=1, + description="{} call py file: {} with params: {} failed \n [ERROR] {}".format( + self.task_name, self.file_path, self.py_input, error + ), + ) self.py_input = params_format_save return result diff --git a/ct_om/service/ctmgr/tasks/audit_task.py b/ct_om/service/ctmgr/tasks/audit_task.py index a2ffdce0..241dcfe4 100644 --- a/ct_om/service/ctmgr/tasks/audit_task.py +++ b/ct_om/service/ctmgr/tasks/audit_task.py @@ -9,31 +9,35 @@ from log_tool.om_log import TASK_LOG MAX_AUDIT_NUM = 10 DIR_NAME, _ = os.path.split(os.path.abspath(__file__)) -INSPECTION_PATH = str(Path('{}/../inspections'.format(DIR_NAME))) +INSPECTION_PATH = str(Path("{}/../inspections".format(DIR_NAME))) class AuditTask(PyTask): def __init__(self, task_name, handler, file_path, py_input, params_check_dict): super().__init__(task_name, handler, file_path, py_input, params_check_dict) - self.output_data = '' + self.output_data = "" self.audit_path = INSPECTION_PATH def task_execute(self, input_params, task_logger): - res = super(AuditTask, self).task_execute(input_params=input_params, task_logger=task_logger) + res = super(AuditTask, self).task_execute( + input_params=input_params, task_logger=task_logger + ) - audit_info = eval(eval(res.__str__()).get('data', {}).get('ctmgr_common_output', '')) + audit_info = eval( + eval(res.__str__()).get("data", {}).get("ctmgr_common_output", "") + ) - audit_result = audit_info.get('RESULT') - if str(audit_result) == '0': - self.output_data = str(audit_info).replace('\n', '') + audit_result = audit_info.get("RESULT") + if str(audit_result) == "0": + self.output_data = str(audit_info).replace("\n", "") self.write_audit() - TASK_LOG.info('show cantian status success') - res.set_output_data(str(audit_info.get('CMS_STAT'))) + TASK_LOG.info("show cantian status success") + res.set_output_data(str(audit_info.get("CMS_STAT"))) return res else: - TASK_LOG.error('show cantian status fail') - res.set_output_data('') + TASK_LOG.error("show cantian status fail") + res.set_output_data("") res.set_error_code(1) return res @@ -50,9 +54,9 @@ class AuditTask(PyTask): flags = os.O_WRONLY | os.O_TRUNC | os.O_CREAT utc_now = datetime.utcnow() cur_time = utc_now.replace(tzinfo=timezone.utc).astimezone(tz=None) - audit_file = 'inspection_{}'.format(str(cur_time.strftime("%Y%m%d%H%M%S"))) - audit_file_path = str(Path(self.audit_path + '/' + audit_file)) - with os.fdopen(os.open(audit_file_path, flags, modes), 'w', encoding='utf-8') as file: + audit_file = "inspection_{}".format(str(cur_time.strftime("%Y%m%d%H%M%S"))) + audit_file_path = str(Path(self.audit_path + "/" + audit_file)) + with os.fdopen( + os.open(audit_file_path, flags, modes), "w", encoding="utf-8" + ) as file: file.write(self.output_data) - - diff --git a/ct_om/service/ctmgr/tasks/inspection/inspection_task.py b/ct_om/service/ctmgr/tasks/inspection/inspection_task.py index f931bc12..7d088a4d 100644 --- a/ct_om/service/ctmgr/tasks/inspection/inspection_task.py +++ b/ct_om/service/ctmgr/tasks/inspection/inspection_task.py @@ -13,11 +13,13 @@ from common.common_tool import TimeTool MAX_AUDIT_NUM = 10 DIR_NAME, _ = os.path.split(os.path.abspath(__file__)) -INSPECTION_PATH = str(Path('{}/../../inspections'.format(DIR_NAME))) -FAIL = 'fail' -SUCCESS = 'success' -SUCCESS_ENUM = [0, '0'] -INSPECTION_JSON_FILE = "/opt/cantian/ct_om/service/ctmgr/tasks/inspection/inspection_config.json" +INSPECTION_PATH = str(Path("{}/../../inspections".format(DIR_NAME))) +FAIL = "fail" +SUCCESS = "success" +SUCCESS_ENUM = [0, "0"] +INSPECTION_JSON_FILE = ( + "/opt/cantian/ct_om/service/ctmgr/tasks/inspection/inspection_config.json" +) class InspectionTask(TASK): @@ -33,30 +35,32 @@ class InspectionTask(TASK): @staticmethod def read_inspection_config(): - with open(INSPECTION_JSON_FILE, encoding='utf-8') as file: + with open(INSPECTION_JSON_FILE, encoding="utf-8") as file: inspection_map = json.load(file) return inspection_map @staticmethod - def format_single_inspection_result(inspection_item, inspection_detail, execute_result, inspection_result): + def format_single_inspection_result( + inspection_item, inspection_detail, execute_result, inspection_result + ): return_value = { - 'inspection_item': inspection_item, - 'component': inspection_detail.get("component"), - 'inspection_result': execute_result, - 'inspection_detail': inspection_result.get('data'), - 'description_zn': inspection_detail.get("description_zn"), - 'description_en': inspection_detail.get("description_en") + "inspection_item": inspection_item, + "component": inspection_detail.get("component"), + "inspection_result": execute_result, + "inspection_detail": inspection_result.get("data"), + "description_zn": inspection_detail.get("description_zn"), + "description_en": inspection_detail.get("description_en"), } if inspection_result and isinstance(inspection_result, dict): - err_info = inspection_result.get('error', {}) - error_code = err_info.get('code') + err_info = inspection_result.get("error", {}) + error_code = err_info.get("code") if error_code is None: return return_value if error_code not in SUCCESS_ENUM: - return_value['inspection_result'] = FAIL + return_value["inspection_result"] = FAIL return return_value @@ -76,11 +80,16 @@ class InspectionTask(TASK): if input_params == "all": inspection_items = list(self.inspection_map.keys()) - if not isinstance(input_params, list) and input_params != 'all': - TASK_LOG.error("inspection input error, input value is: " + str(input_params)) - result = CommonResult(output_data="inspection input error, input value is: " + str(input_params), - error_code=1, - description="input must be string \"all\" or [xxx, xxx]") + if not isinstance(input_params, list) and input_params != "all": + TASK_LOG.error( + "inspection input error, input value is: " + str(input_params) + ) + result = CommonResult( + output_data="inspection input error, input value is: " + + str(input_params), + error_code=1, + description='input must be string "all" or [xxx, xxx]', + ) task_logger.set_finish_time(TimeTool.get_current_time()) task_logger.info(self.task_name, FAIL) return result @@ -88,8 +97,9 @@ class InspectionTask(TASK): for inspection_item in inspection_items: inspection_detail = self.inspection_map.get(inspection_item) - single_check_result, single_result = self.param_check_single(inspection_item, - inspection_detail, task_logger) + single_check_result, single_result = self.param_check_single( + inspection_item, inspection_detail, task_logger + ) if not single_check_result: if single_result: return single_result @@ -97,18 +107,25 @@ class InspectionTask(TASK): continue try: - single_inspection_result = json.loads(self.task_execute_single(inspection_detail)) + single_inspection_result = json.loads( + self.task_execute_single(inspection_detail) + ) except Exception as err: - TASK_LOG.error("excute %s inspection failed with error: %s" % (inspection_item, str(err))) - formated_inspection_result = self.format_single_inspection_result(inspection_item, - inspection_detail, FAIL, {}) + TASK_LOG.error( + "excute %s inspection failed with error: %s" + % (inspection_item, str(err)) + ) + formated_inspection_result = self.format_single_inspection_result( + inspection_item, inspection_detail, FAIL, {} + ) self.inspection_result.append(formated_inspection_result) self.fail_list.append(inspection_item) continue - formated_inspection_result = self.format_single_inspection_result(inspection_item, inspection_detail, - SUCCESS, single_inspection_result) - if formated_inspection_result.get('inspection_result') == FAIL: + formated_inspection_result = self.format_single_inspection_result( + inspection_item, inspection_detail, SUCCESS, single_inspection_result + ) + if formated_inspection_result.get("inspection_result") == FAIL: self.inspection_result.append(formated_inspection_result) self.fail_list.append(inspection_item) continue @@ -117,12 +134,18 @@ class InspectionTask(TASK): self.success_list.append(inspection_item) if not self.success_list: - result.set_output_data("inspection item: %s failed" % ' '.join(self.fail_list)) + result.set_output_data( + "inspection item: %s failed" % " ".join(self.fail_list) + ) elif not self.fail_list: - result.set_output_data("inspection item: %s success" % ' '.join(self.success_list)) + result.set_output_data( + "inspection item: %s success" % " ".join(self.success_list) + ) else: - result.set_output_data("inspection item: %s success, inspection item: %s failed" - % (' '.join(self.success_list), ' '.join(self.fail_list))) + result.set_output_data( + "inspection item: %s success, inspection item: %s failed" + % (" ".join(self.success_list), " ".join(self.fail_list)) + ) self.write_audit() @@ -135,17 +158,24 @@ class InspectionTask(TASK): if not inspection_detail: TASK_LOG.error("inspection item %s not exist" % inspection_item) - result = CommonResult(output_data=inspection_item + " not exist", error_code=1, - description="please check input, inspection item %s not exist" - % inspection_item) + result = CommonResult( + output_data=inspection_item + " not exist", + error_code=1, + description="please check input, inspection item %s not exist" + % inspection_item, + ) task_logger.set_finish_time(TimeTool.get_current_time()) task_logger.info(self.task_name, FAIL) return False, result - if not os.path.exists(inspection_detail.get('inspection_file_path')): - TASK_LOG.error("inspection file: %s not exist" % str(inspection_detail.get('inspection_file_path'))) - formated_inspection_result = self.format_single_inspection_result(inspection_item, - inspection_detail, FAIL, None) + if not os.path.exists(inspection_detail.get("inspection_file_path")): + TASK_LOG.error( + "inspection file: %s not exist" + % str(inspection_detail.get("inspection_file_path")) + ) + formated_inspection_result = self.format_single_inspection_result( + inspection_item, inspection_detail, FAIL, None + ) self.inspection_result.append(formated_inspection_result) self.fail_list.append(inspection_item) return False, result @@ -153,13 +183,18 @@ class InspectionTask(TASK): return True, result def task_execute_single(self, inspection_detail): - inspection_item_file = inspection_detail.get('inspection_file_path') - inspection_item_input = inspection_detail.get('input_param') - - single_inspection_popen = subprocess.Popen(['/usr/bin/python3', inspection_item_file, inspection_item_input], - stdout=subprocess.PIPE, shell=False) - single_inspection_result = single_inspection_popen.communicate(timeout=self.time_out)[0].decode('utf-8') - single_inspection_result = single_inspection_result.replace("\'", "\"") + inspection_item_file = inspection_detail.get("inspection_file_path") + inspection_item_input = inspection_detail.get("input_param") + + single_inspection_popen = subprocess.Popen( + ["/usr/bin/python3", inspection_item_file, inspection_item_input], + stdout=subprocess.PIPE, + shell=False, + ) + single_inspection_result = single_inspection_popen.communicate( + timeout=self.time_out + )[0].decode("utf-8") + single_inspection_result = single_inspection_result.replace("'", '"') return single_inspection_result @@ -174,7 +209,7 @@ class InspectionTask(TASK): :param inspection_output: 调用巡检项脚本的输出 整理一键巡检格式, 结果写到 self.output_data """ - self.output_data['data'] = self.inspection_result + self.output_data["data"] = self.inspection_result def write_audit(self): self.format_inspection_result() @@ -191,7 +226,16 @@ class InspectionTask(TASK): flags = os.O_WRONLY | os.O_TRUNC | os.O_CREAT utc_now = datetime.utcnow() cur_time = utc_now.replace(tzinfo=timezone.utc).astimezone(tz=None) - audit_file = 'inspection_{}'.format(str(cur_time.strftime("%Y%m%d%H%M%S"))) - audit_file_path = str(Path(self.audit_path + '/' + audit_file)) - with os.fdopen(os.open(audit_file_path, flags, modes), 'w', encoding='utf-8') as file: - file.write(json.dumps(self.output_data, indent=4, separators=(',', ': '), ensure_ascii=False)) + audit_file = "inspection_{}".format(str(cur_time.strftime("%Y%m%d%H%M%S"))) + audit_file_path = str(Path(self.audit_path + "/" + audit_file)) + with os.fdopen( + os.open(audit_file_path, flags, modes), "w", encoding="utf-8" + ) as file: + file.write( + json.dumps( + self.output_data, + indent=4, + separators=(",", ": "), + ensure_ascii=False, + ) + ) diff --git a/ct_om/service/ctmgr/tasks/log_progress_query.py b/ct_om/service/ctmgr/tasks/log_progress_query.py index 8034fd82..5a8eec6e 100644 --- a/ct_om/service/ctmgr/tasks/log_progress_query.py +++ b/ct_om/service/ctmgr/tasks/log_progress_query.py @@ -8,8 +8,10 @@ from common.common_tool import TimeTool cur_abs_path, _ = os.path.split(os.path.abspath(__file__)) upper_path = os.path.abspath(os.path.join(cur_abs_path, "..")) -LOG_FILE_PATH = str(Path('{}/logs_collection'.format(upper_path), 'log_packing_progress.json')) -SUCCESS = 'success' +LOG_FILE_PATH = str( + Path("{}/logs_collection".format(upper_path), "log_packing_progress.json") +) +SUCCESS = "success" class QueryLogProgressTask(TASK): @@ -20,11 +22,13 @@ class QueryLogProgressTask(TASK): def task_execute(self, input_params, task_logger: LOGGER): task_logger.set_finish_time(TimeTool.get_current_time()) task_logger.info(self.task_name, SUCCESS) - TASK_LOG.info('doing logs collection progress query, path is: {}'.format(LOG_FILE_PATH)) + TASK_LOG.info( + "doing logs collection progress query, path is: {}".format(LOG_FILE_PATH) + ) return CommonResult(output_data=LOG_FILE_PATH) -if __name__ == '__main__': +if __name__ == "__main__": lt = QueryLogProgressTask(None, None) lg = LOGGER(None, None, None) lt.task_execute({}, lg) diff --git a/ct_om/service/ctmgr/tasks/log_query.py b/ct_om/service/ctmgr/tasks/log_query.py index e8428d59..fcc2a1f4 100644 --- a/ct_om/service/ctmgr/tasks/log_query.py +++ b/ct_om/service/ctmgr/tasks/log_query.py @@ -8,7 +8,7 @@ from common.common_tool import TimeTool from log_tool.om_log import TASK_LOG LOG_FILE_PATH = str(Path(LOG.handlers[0].baseFilename)) -SUCCESS = 'success' +SUCCESS = "success" class QueryLogTask(TASK): @@ -19,11 +19,11 @@ class QueryLogTask(TASK): def task_execute(self, input_params, task_logger: LOGGER): task_logger.set_finish_time(TimeTool.get_current_time()) task_logger.info(self.task_name, SUCCESS) - TASK_LOG.info('doing log queryn, path is: {}'.format(LOG_FILE_PATH)) + TASK_LOG.info("doing log queryn, path is: {}".format(LOG_FILE_PATH)) return CommonResult(output_data=LOG_FILE_PATH) -if __name__ == '__main__': +if __name__ == "__main__": lt = QueryLogTask(None, None) lg = LOGGER(None, None, None) lt.task_execute({}, lg) diff --git a/ct_om/service/ctmgr/tasks/logs_collections_task.py b/ct_om/service/ctmgr/tasks/logs_collections_task.py index 21e34475..96b28182 100644 --- a/ct_om/service/ctmgr/tasks/logs_collections_task.py +++ b/ct_om/service/ctmgr/tasks/logs_collections_task.py @@ -8,7 +8,7 @@ from log_tool.om_log import LOGGER, DEPLOY_LOG, TASK_LOG from common.output_tool import CommonResult from common.common_tool import TimeTool -SUCCESS, FAIL = 'success', 'fail' +SUCCESS, FAIL = "success", "fail" cur_abs_path, _ = os.path.split(os.path.abspath(__file__)) @@ -20,8 +20,11 @@ class LogsCollectionTask(TASK): self.py_input = py_input self.time_out = 2 if not self.basic_check(): - DEPLOY_LOG.error('[error] {} not exist, init ShellTask failed, please check param in cmd {}'.format( - self.file_path, self.task_name)) + DEPLOY_LOG.error( + "[error] {} not exist, init ShellTask failed, please check param in cmd {}".format( + self.file_path, self.task_name + ) + ) raise Exception def basic_check(self): @@ -29,25 +32,39 @@ class LogsCollectionTask(TASK): def task_execute(self, input_params, task_logger): params_format_save = self.py_input - exec_cmd = 'python3 {}'.format(self.file_path) + exec_cmd = "python3 {}".format(self.file_path) for param_key in input_params.keys(): - self.py_input = self.py_input.replace('${%s}' % param_key, str(input_params.get(param_key))) - exec_cmd = exec_cmd + ' ' + str(self.py_input) + self.py_input = self.py_input.replace( + "${%s}" % param_key, str(input_params.get(param_key)) + ) + exec_cmd = exec_cmd + " " + str(self.py_input) result = CommonResult() - TASK_LOG.info('task: {} calling py file: {}, using cmd: {}'.format(self.task_name, self.file_path, exec_cmd)) + TASK_LOG.info( + "task: {} calling py file: {}, using cmd: {}".format( + self.task_name, self.file_path, exec_cmd + ) + ) task_logger.set_request_time(TimeTool.get_current_time()) try: - py_result = subprocess.Popen(shlex.split(exec_cmd), stdout=subprocess.PIPE, shell=False) + py_result = subprocess.Popen( + shlex.split(exec_cmd), stdout=subprocess.PIPE, shell=False + ) except Exception as err: - TASK_LOG.error('task: {} calling py file: {}, using cmd: {} fail, ' - 'error: {}'.format(self.task_name, self.file_path, exec_cmd, err)) + TASK_LOG.error( + "task: {} calling py file: {}, using cmd: {} fail, " + "error: {}".format(self.task_name, self.file_path, exec_cmd, err) + ) task_logger.set_finish_time(TimeTool.get_current_time()) task_logger.info(self.task_name, FAIL) - result = CommonResult(output_data='', error_code=1, - description='{} call py file: {} with params: {} failed \n [ERROR] {}'.format( - self.task_name, self.file_path, self.py_input, err)) + result = CommonResult( + output_data="", + error_code=1, + description="{} call py file: {} with params: {} failed \n [ERROR] {}".format( + self.task_name, self.file_path, self.py_input, err + ), + ) self.py_input = params_format_save return result @@ -55,7 +72,7 @@ class LogsCollectionTask(TASK): try: std_res, std_err = py_result.communicate(timeout=self.time_out) except Exception as err: - TASK_LOG.info('task: {} background execution start'.format(self.task_name)) + TASK_LOG.info("task: {} background execution start".format(self.task_name)) result.set_output_data(str(err)) task_logger.set_finish_time(TimeTool.get_current_time()) task_logger.info(self.task_name, SUCCESS) @@ -66,17 +83,23 @@ class LogsCollectionTask(TASK): # 日志采集脚本执行过程中异常退出 if not std_res or std_err: - err_msg = 'cmd: {} itself failed'.format(exec_cmd) - TASK_LOG.error('task: {} calling py file: {}, using cmd: {} fail, ' - 'error: {}'.format(self.task_name, self.file_path, exec_cmd, err_msg)) + err_msg = "cmd: {} itself failed".format(exec_cmd) + TASK_LOG.error( + "task: {} calling py file: {}, using cmd: {} fail, " + "error: {}".format(self.task_name, self.file_path, exec_cmd, err_msg) + ) task_logger.set_finish_time(TimeTool.get_current_time()) task_logger.info(self.task_name, FAIL) - result = CommonResult(output_data='', error_code=1, - description='{} call py file: {} failed, [ERROR] {}'.format( - self.task_name, self.file_path, err_msg)) + result = CommonResult( + output_data="", + error_code=1, + description="{} call py file: {} failed, [ERROR] {}".format( + self.task_name, self.file_path, err_msg + ), + ) return result - result.set_output_data(std_res.decode('utf-8')) + result.set_output_data(std_res.decode("utf-8")) task_logger.set_finish_time(TimeTool.get_current_time()) task_logger.info(self.task_name, SUCCESS) return result diff --git a/ct_om/service/ctmgr/uds_client.py b/ct_om/service/ctmgr/uds_client.py index 1b870a4b..48f6dcb9 100644 --- a/ct_om/service/ctmgr/uds_client.py +++ b/ct_om/service/ctmgr/uds_client.py @@ -4,7 +4,7 @@ import socket from log_tool.om_log import TASK_LOG from common.output_tool import CommonResult -SERVER_ADDRESS = '' +SERVER_ADDRESS = "" SOCKET_TYPE = socket.SOCK_STREAM SOCKET_FAMILY = socket.AF_UNIX @@ -19,23 +19,32 @@ def client_socket(send_data): try: sock.connect(SERVER_ADDRESS) except socket.error as error: - TASK_LOG.error('client connect failed, error: {}'.format(error)) - return CommonResult(output_data='client connect failed', error_code=1, - description='connect failed, error: {}'.format(error)) + TASK_LOG.error("client connect failed, error: {}".format(error)) + return CommonResult( + output_data="client connect failed", + error_code=1, + description="connect failed, error: {}".format(error), + ) try: sock.sendall(json.dumps(send_data).encode()) except socket.error as error: - result = CommonResult(output_data='uds request failed', error_code=1, - description='send data error: {}'.format(error)) + result = CommonResult( + output_data="uds request failed", + error_code=1, + description="send data error: {}".format(error), + ) return result try: recv_data = sock.recv(RECEIVE_DATA_SIZE).decode() except socket.error as error: - result = CommonResult(output_data='uds request failed', error_code=1, - description='receive data error: {}'.format(error)) - TASK_LOG.error('send/receive data error: {}'.format(error)) + result = CommonResult( + output_data="uds request failed", + error_code=1, + description="receive data error: {}".format(error), + ) + TASK_LOG.error("send/receive data error: {}".format(error)) return result result.set_output_data(recv_data) @@ -45,5 +54,5 @@ def client_socket(send_data): return result -if __name__ == '__main__': +if __name__ == "__main__": client_socket('{"command": "show cantina status", "param": {}}') diff --git a/ct_om/service/ctmgr/uds_server.py b/ct_om/service/ctmgr/uds_server.py index 8c020190..85de790d 100644 --- a/ct_om/service/ctmgr/uds_server.py +++ b/ct_om/service/ctmgr/uds_server.py @@ -15,7 +15,7 @@ threadPool = ThreadPoolExecutor(max_workers=5) task_dir = task.task_dir dir_name, _ = os.path.split(os.path.abspath(__file__)) -SERVER_ADDRESS = str(Path('{}/../ct_om.sock'.format(dir_name))) +SERVER_ADDRESS = str(Path("{}/../ct_om.sock".format(dir_name))) SOCKET_FAMILY = socket.AF_UNIX SOCKET_TYPE = socket.SOCK_STREAM @@ -25,17 +25,22 @@ USER_UID = (6004,) def get_socket_msg(conn, client_addr): - credit = conn.getsockopt(socket.SOL_SOCKET, socket.SO_PEERCRED, struct.calcsize('3i')) + credit = conn.getsockopt( + socket.SOL_SOCKET, socket.SO_PEERCRED, struct.calcsize("3i") + ) # input_json为字符串类型json："{'command': 'show cantina statu', 'param': {}}" input_json = conn.recv(RECEIVE_DATA_SIZE).decode() - pid, uid, gid = struct.unpack('3i', credit) + pid, uid, gid = struct.unpack("3i", credit) if int(uid) not in USER_UID: - result = CommonResult(output_data='uid from client is not correct', error_code=1, - description='current uid from client is {}'.format(uid)) + result = CommonResult( + output_data="uid from client is not correct", + error_code=1, + description="current uid from client is {}".format(uid), + ) conn.sendall(result.__str__().encode()) else: - begin_time = time.strftime('%Y-%m-%d %H:%M:%S', time.localtime(time.time())) + begin_time = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime(time.time())) task_log = LOGGER(client_pid=pid, client_uid=uid, begin_time=begin_time) result = execute_task(input_json, task_log) # 接收cli命令并进行预检验，执行 conn.sendall(result.__str__().encode()) @@ -48,16 +53,16 @@ def server_socket(): os.unlink(SERVER_ADDRESS) # 绑定套接字 if sock.bind(SERVER_ADDRESS): - DEPLOY_LOG.error('socket bind error, SERVER_ADDRESS: {}'.format(SERVER_ADDRESS)) - raise Exception('socket bind error, SERVER_ADDRESS: {}'.format(SERVER_ADDRESS)) + DEPLOY_LOG.error("socket bind error, SERVER_ADDRESS: {}".format(SERVER_ADDRESS)) + raise Exception("socket bind error, SERVER_ADDRESS: {}".format(SERVER_ADDRESS)) else: - DEPLOY_LOG.info('socket bind success, address is {}'.format(SERVER_ADDRESS)) + DEPLOY_LOG.info("socket bind success, address is {}".format(SERVER_ADDRESS)) if sock.listen(1): - DEPLOY_LOG.error('socket listen error') - raise Exception('socket listen error') + DEPLOY_LOG.error("socket listen error") + raise Exception("socket listen error") - DEPLOY_LOG.info('socket listen begin') + DEPLOY_LOG.info("socket listen begin") while KEEP_LISTEN: connection, client_address = sock.accept() @@ -68,13 +73,16 @@ def server_socket(): def execute_task(input_data, task_log): input_data_dict = ast.literal_eval(input_data) - command = input_data_dict.get('command') - input_params_dict = input_data_dict.get('param') + command = input_data_dict.get("command") + input_params_dict = input_data_dict.get("param") if command not in task_dir.keys(): - task_log.error(cmd=command, result='fail') - result = CommonResult(output_data='execute {} failed'.format(command), error_code=1, - description='cli command: {} not valid'.format(command)) + task_log.error(cmd=command, result="fail") + result = CommonResult( + output_data="execute {} failed".format(command), + error_code=1, + description="cli command: {} not valid".format(command), + ) return result task_obj = task_dir[command] @@ -83,21 +91,29 @@ def execute_task(input_data, task_log): result = task_obj.task_execute(input_params_dict, task_log) except Exception as err: exc_type, exc_value, exc_tb = sys.exc_info() - TASK_LOG.error("exece command: %s faild, inner error is: %s, detial is %s" % - (command, str(err), str((exc_type, exc_value, exc_tb)))) - task_log.error(cmd=command, result='fail') - result = CommonResult(output_data='execute {} failed'.format(command), error_code=1, - description='param check failed for command: {}'.format(command)) + TASK_LOG.error( + "exece command: %s faild, inner error is: %s, detial is %s" + % (command, str(err), str((exc_type, exc_value, exc_tb))) + ) + task_log.error(cmd=command, result="fail") + result = CommonResult( + output_data="execute {} failed".format(command), + error_code=1, + description="param check failed for command: {}".format(command), + ) return result else: - task_log.error(cmd=command, result='fail') - result = CommonResult(output_data='execute {} failed'.format(command), error_code=1, - description='param check failed for command: {}'.format(command)) + task_log.error(cmd=command, result="fail") + result = CommonResult( + output_data="execute {} failed".format(command), + error_code=1, + description="param check failed for command: {}".format(command), + ) return result -if __name__ == '__main__': +if __name__ == "__main__": task.load_tasks() server_socket() diff --git a/pkg/admin/scripts/fetch_cls_stat.py b/pkg/admin/scripts/fetch_cls_stat.py index 42277f7b..a6231a9f 100644 --- a/pkg/admin/scripts/fetch_cls_stat.py +++ b/pkg/admin/scripts/fetch_cls_stat.py @@ -19,8 +19,13 @@ def _exec_popen(cmd, values=None): if not values: values = [] bash_cmd = ["bash"] - pobj = subprocess.Popen(bash_cmd, shell=False, stdin=subprocess.PIPE, - stdout=subprocess.PIPE, stderr=subprocess.PIPE) + pobj = subprocess.Popen( + bash_cmd, + shell=False, + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + ) py_version = platform.python_version() if py_version[0] == "3": @@ -51,17 +56,17 @@ def _exec_popen(cmd, values=None): def get_user(): config_path = Path(CONFIG_PATH) config_list = json.loads(config_path.read_text()) - return config_list["deploy_user"].split(':')[0] + return config_list["deploy_user"].split(":")[0] def parse_node_stat(node_stat): - keys = ['NODE_ID', 'NAME', 'STAT', 'PRE_STAT'] + keys = ["NODE_ID", "NAME", "STAT", "PRE_STAT"] values = node_stat.split() stat_json = {} - for (idx, key) in enumerate(keys): + for idx, key in enumerate(keys): stat_json[key] = values[idx] online = False - if stat_json['STAT'] == 'ONLINE': + if stat_json["STAT"] == "ONLINE": online = True return (online, stat_json) @@ -70,7 +75,7 @@ def fetch_cms_stat(): user = get_user() cmd = 'su - %s -c "cms stat" | tail -n +2' % user _, output, _ = _exec_popen(cmd) - output = output.split('\n') + output = output.split("\n") cms_stat_json = {} if len(output) <= 1: return (False, cms_stat_json) @@ -81,19 +86,19 @@ def fetch_cms_stat(): detail_json.append(stat_json) if online: online_cnt += 1 - cms_stat_json['DETAIL'] = detail_json + cms_stat_json["DETAIL"] = detail_json if online_cnt == 0: - cms_stat_json['STATUS'] = 'OFFLINE' + cms_stat_json["STATUS"] = "OFFLINE" elif online_cnt == len(output): - cms_stat_json['STATUS'] = 'ONLINE' + cms_stat_json["STATUS"] = "ONLINE" else: - cms_stat_json['STATUS'] = 'PARTIALLY_ONLINE' + cms_stat_json["STATUS"] = "PARTIALLY_ONLINE" return (True, cms_stat_json) def gen_fault_result(): result_json = {} - result_json['RESULT'] = -1 + result_json["RESULT"] = -1 return json.dumps(result_json) @@ -102,10 +107,10 @@ def fetch_cls_stat(): if not success: return gen_fault_result() status_json = {} - status_json['CMS_STAT'] = cms_stat_json - status_json['RESULT'] = 0 + status_json["CMS_STAT"] = cms_stat_json + status_json["RESULT"] = 0 return json.dumps(status_json) -if __name__ == '__main__': +if __name__ == "__main__": print(fetch_cls_stat()) diff --git a/pkg/deploy/action/cantian/Common.py b/pkg/deploy/action/cantian/Common.py index 9f83a095..9b480678 100644 --- a/pkg/deploy/action/cantian/Common.py +++ b/pkg/deploy/action/cantian/Common.py @@ -10,6 +10,7 @@ import sys + sys.dont_write_bytecode = True try: @@ -30,8 +31,10 @@ class DefaultValue(object): """ Default value of some variables """ + def __init__(self): pass + # file mode MAX_FILE_MODE = 640 MIN_FILE_MODE = 400 @@ -39,13 +42,13 @@ class DefaultValue(object): MID_FILE_MODE = 500 KEY_DIRECTORY_MODE = 700 MAX_DIRECTORY_MODE = 750 - KEY_DIRECTORY_MODE_STR = '0700' + KEY_DIRECTORY_MODE_STR = "0700" MIN_FILE_PERMISSION = 0o400 MID_FILE_PERMISSION = 0o500 KEY_FILE_PERMISSION = 0o600 KEY_DIRECTORY_PERMISSION = 0o700 CANTIAND_CONF_NAME = "cantiand.ini" - + # get os version and python version CURRENT_OS = platform.system() PY_VERSION = platform.python_version() @@ -66,7 +69,7 @@ class DefaultValue(object): while True: # find the top path to be created (tmp_dir, top_dir_name) = os.path.split(tmp_dir) - if (os.path.exists(tmp_dir) or top_dir_name == ""): + if os.path.exists(tmp_dir) or top_dir_name == "": tmp_dir = os.path.join(tmp_dir, top_dir_name) break return tmp_dir @@ -115,8 +118,13 @@ class DefaultValue(object): bash_cmd = ["bash"] if not stdin_list: stdin_list = [] - pobj = subprocess.Popen(bash_cmd, shell=False, stdin=subprocess.PIPE, - stdout=subprocess.PIPE, stderr=subprocess.PIPE) + pobj = subprocess.Popen( + bash_cmd, + shell=False, + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + ) # in python 3, the stand output and stand error is # unicode, we must decode it before return diff --git a/pkg/deploy/action/cantian/bind_cpu_config.py b/pkg/deploy/action/cantian/bind_cpu_config.py index 87458d73..a9ea00c7 100644 --- a/pkg/deploy/action/cantian/bind_cpu_config.py +++ b/pkg/deploy/action/cantian/bind_cpu_config.py @@ -10,6 +10,7 @@ import grp from log import LOGGER from get_config_info import get_value + sys.path.append(os.path.join(os.path.dirname(os.path.abspath(__file__)), "..")) from update_config import update_dbstor_conf @@ -21,12 +22,15 @@ MES_MODULE = "MES_BIND_CPU" MES_CPU_INFO = "MES_CPU_INFO" CANTIAN_NUMA_INFO = "CANTIAN_NUMA_CPU_INFO" MYSQL_NUMA_INFO = "MYSQL_NUMA_CPU_INFO" -MODULE_LIST = [XNET_MODULE, MES_MODULE] # 目前只支持xnet, mes模块的绑核，后续扩展秩序添加在这里即可 +MODULE_LIST = [ + XNET_MODULE, + MES_MODULE, +] # 目前只支持xnet, mes模块的绑核，后续扩展秩序添加在这里即可 dbstor_file_module_dict = { "XNET_CPU": XNET_MODULE, "MES_CPU": MES_MODULE, "IOD_CPU": "", - "ULOG_CPU": "" + "ULOG_CPU": "", } # 用于更新dbstor.ini文件，后续扩展加上 BIND_NUMA_NODE_NUM = 2 # 物理机时绑定的 NUMA 前n个节点数，只支持绑前2个 @@ -43,9 +47,9 @@ def cpu_info_to_cpu_list(cpu_list_str): cpu_list = [] - for part in cpu_list_str.split(','): - if '-' in part: - start, end = map(int, part.split('-')) + for part in cpu_list_str.split(","): + if "-" in part: + start, end = map(int, part.split("-")) cpu_list.extend(range(start, end + 1)) else: cpu_list.append(int(part)) @@ -58,10 +62,12 @@ def cpu_list_to_cpu_info(cpu_list): Converts a list of CPU IDs (either as integers or a comma-separated string) into a string in the format '1-3,5-6'. """ if isinstance(cpu_list, str): - cpu_list = cpu_list.split(',') + cpu_list = cpu_list.split(",") if not all(isinstance(cpu, (int, str)) for cpu in cpu_list): - raise ValueError("cpu_list should contain integers or strings that can be converted to integers") + raise ValueError( + "cpu_list should contain integers or strings that can be converted to integers" + ) cpu_list = sorted(set(map(int, cpu_list))) @@ -87,11 +93,12 @@ def cpu_list_to_cpu_info(cpu_list): return ",".join(ranges) + def get_json_config(path): """ Retrieves the NUMA configuration from the specified JSON file. """ - with open(path, 'r', encoding='utf-8') as file: + with open(path, "r", encoding="utf-8") as file: config_json = json.load(file) return config_json @@ -122,7 +129,9 @@ def write_json_config_file(path, config): except KeyError: LOGGER.error(f"User '{user_name}' or group '{group_name}' not found.") except PermissionError: - LOGGER.error("Permission denied: cannot change file ownership. Run as root or with sufficient privileges.") + LOGGER.error( + "Permission denied: cannot change file ownership. Run as root or with sufficient privileges." + ) class NumaConfigBase: @@ -141,8 +150,13 @@ class NumaConfigBase: if not values: values = [] bash_cmd = ["bash"] - pobj = subprocess.Popen(bash_cmd, shell=False, stdin=subprocess.PIPE, - stdout=subprocess.PIPE, stderr=subprocess.PIPE) + pobj = subprocess.Popen( + bash_cmd, + shell=False, + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + ) if gPyVersion[0] == "3": pobj.stdin.write(cmd.encode()) @@ -190,7 +204,7 @@ class NumaConfigBase: """ Performs pre-checks before proceeding with NUMA configuration. """ - if platform.machine() != 'aarch64': + if platform.machine() != "aarch64": LOGGER.info("System is not aarch64") return @@ -199,7 +213,7 @@ class NumaConfigBase: raise Exception(err_msg) def get_module_bind_cpu_list(self, module_thread_num): - """ 获取模块绑核的 CPU 列表 """ + """获取模块绑核的 CPU 列表""" result_ranges = [] count = module_thread_num @@ -208,14 +222,19 @@ class NumaConfigBase: # 过滤掉已经绑定的 CPU if self.bind_cpu_list: available_cpu_for_binding_filtered = { - numa_id: [cpu for cpu in available_cpu_list if cpu not in self.bind_cpu_list] + numa_id: [ + cpu for cpu in available_cpu_list if cpu not in self.bind_cpu_list + ] for numa_id, available_cpu_list in self.available_cpu_for_binding_dict.items() } else: available_cpu_for_binding_filtered = self.available_cpu_for_binding_dict while count > 0: - for numa_id, available_cpu_list in available_cpu_for_binding_filtered.items(): + for ( + numa_id, + available_cpu_list, + ) in available_cpu_for_binding_filtered.items(): if numa_pointer[numa_id] < len(available_cpu_list): result_ranges.append(available_cpu_list[numa_pointer[numa_id]]) numa_pointer[numa_id] += 1 @@ -224,7 +243,9 @@ class NumaConfigBase: break for numa_id, available_cpu_list in available_cpu_for_binding_filtered.items(): - self.available_cpu_for_binding_dict[numa_id] = available_cpu_list[numa_pointer[numa_id]:] + self.available_cpu_for_binding_dict[numa_id] = available_cpu_list[ + numa_pointer[numa_id] : + ] return result_ranges @@ -260,7 +281,11 @@ class NumaConfigBase: try: with open(cantian_conf_file, "r+", encoding="utf-8") as file: config = file.readlines() - existing_keys = {line.split("=", maxsplit=1)[0].strip() for line in config if "=" in line} + existing_keys = { + line.split("=", maxsplit=1)[0].strip() + for line in config + if "=" in line + } updated_keys = set() removed_keys = set() @@ -274,7 +299,10 @@ class NumaConfigBase: key, value = line.split("=", maxsplit=1) key = key.strip() - if key in cantiand_cpu_info and cantiand_cpu_info[key] in ("-del", "-remove"): + if key in cantiand_cpu_info and cantiand_cpu_info[key] in ( + "-del", + "-remove", + ): removed_keys.add(key) continue @@ -294,9 +322,13 @@ class NumaConfigBase: file.truncate() if updated_keys: - LOGGER.info(f"Updated keys in {cantian_conf_file}: {', '.join(updated_keys)}") + LOGGER.info( + f"Updated keys in {cantian_conf_file}: {', '.join(updated_keys)}" + ) if removed_keys: - LOGGER.info(f"Removed keys in {cantian_conf_file}: {', '.join(removed_keys)}") + LOGGER.info( + f"Removed keys in {cantian_conf_file}: {', '.join(removed_keys)}" + ) except Exception as e: LOGGER.error(f"Failed to update {cantian_conf_file}: {e}") @@ -323,18 +355,22 @@ class PhysicalCpuConfig(NumaConfigBase): return False def init_cpu_info(self): - """ 获取物理机上的所有 CPU 相关信息 """ + """获取物理机上的所有 CPU 相关信息""" # 获取物理机上所有 CPU 的列表 - ret_code, result, stderr = self._exec_popen('/usr/bin/lscpu | grep -i "On-line CPU(s) list"') + ret_code, result, stderr = self._exec_popen( + '/usr/bin/lscpu | grep -i "On-line CPU(s) list"' + ) if ret_code: raise Exception(f"Failed to get CPU list, err: {stderr}") - _result = result.strip().split(':') + _result = result.strip().split(":") if len(_result) != 2: raise Exception(f"NUMA info parsing failed, result: {result}") self.all_cpu_list = cpu_info_to_cpu_list(_result[1].strip()) - ret_code, result, stderr = self._exec_popen('/usr/bin/lscpu | grep -i "NUMA node[0-9] CPU(s)"') + ret_code, result, stderr = self._exec_popen( + '/usr/bin/lscpu | grep -i "NUMA node[0-9] CPU(s)"' + ) if ret_code: raise Exception(f"Failed to get NUMA node info, err: {stderr}") @@ -343,7 +379,7 @@ class PhysicalCpuConfig(NumaConfigBase): # 解析 lscpu 输出中的 NUMA node 信息 lines = result.strip().splitlines() for line in lines: - match = re.search(r'NUMA node(\d+) CPU\(s\):\s+([\d,\-]+)', line) + match = re.search(r"NUMA node(\d+) CPU\(s\):\s+([\d,\-]+)", line) if match: numa_id = int(match.group(1)) cpu_range_str = match.group(2) @@ -355,15 +391,21 @@ class PhysicalCpuConfig(NumaConfigBase): # 更新 available_cpu_for_binding_dict，移除 CPU IDs from 0 to 11 self.available_cpu_for_binding_dict = {} for numa_id, cpu_list in list(self.numa_info_dict.items())[:BIND_NUMA_NODE_NUM]: - valid_cpu_list = [cpu for cpu in cpu_list if cpu >= 12] # 只保留 ID >= 12 的 CPU + valid_cpu_list = [ + cpu for cpu in cpu_list if cpu >= 12 + ] # 只保留 ID >= 12 的 CPU self.available_cpu_for_binding_dict[numa_id] = valid_cpu_list if not self.available_cpu_for_binding_dict or any( - not valid_cpu_list for valid_cpu_list in self.available_cpu_for_binding_dict.values()): - raise Exception("No valid CPU binding available for any NUMA node or some NUMA nodes have no valid CPUs.") + not valid_cpu_list + for valid_cpu_list in self.available_cpu_for_binding_dict.values() + ): + raise Exception( + "No valid CPU binding available for any NUMA node or some NUMA nodes have no valid CPUs." + ) def update_bind_cpu_info(self): - """ 获取绑定的 CPU 列表，支持手动配置 """ + """获取绑定的 CPU 列表，支持手动配置""" numa_config = get_json_config(CONFIG_PATH) bind_cpu_list = [] @@ -376,22 +418,32 @@ class PhysicalCpuConfig(NumaConfigBase): continue if module_id_key in numa_config and numa_config[module_id_key]: - manually_configured_cpus = cpu_info_to_cpu_list(numa_config[module_id_key]) + manually_configured_cpus = cpu_info_to_cpu_list( + numa_config[module_id_key] + ) if self.check_cpu_list_invalid(manually_configured_cpus): - err_msg = (f"Invalid CPU binding in {module_id_key}. " - f"Cannot use CPUs outside the available range or in 0-5.") + err_msg = ( + f"Invalid CPU binding in {module_id_key}. " + f"Cannot use CPUs outside the available range or in 0-5." + ) LOGGER.error(err_msg) raise Exception(err_msg) duplicate_cpus = set(manually_configured_cpus) & set(self.bind_cpu_list) if duplicate_cpus: - err_msg = (f"Currently bound CPUs: {self.bind_cpu_list}. " - f"Conflict in CPU binding for {module_id_key}: CPUs {duplicate_cpus} are already bound.") + err_msg = ( + f"Currently bound CPUs: {self.bind_cpu_list}. " + f"Conflict in CPU binding for {module_id_key}: CPUs {duplicate_cpus} are already bound." + ) LOGGER.error(err_msg) raise Exception(err_msg) - LOGGER.info(f"{module_id_key} is manually configured, skipping CPU binding generation.") + LOGGER.info( + f"{module_id_key} is manually configured, skipping CPU binding generation." + ) bind_cpu_list.extend(manually_configured_cpus) - self.bind_cpu_dict[module_id_key] = ",".join(map(str, manually_configured_cpus)) + self.bind_cpu_dict[module_id_key] = ",".join( + map(str, manually_configured_cpus) + ) self.bind_cpu_list = bind_cpu_list continue @@ -401,7 +453,9 @@ class PhysicalCpuConfig(NumaConfigBase): try: module_info = int(module_info) if not (1 <= module_info <= 10): - LOGGER.warning(f"Module {module_name} thread number out of range (1-10).") + LOGGER.warning( + f"Module {module_name} thread number out of range (1-10)." + ) numa_config[module_id_key] = "" continue except ValueError: @@ -419,7 +473,9 @@ class PhysicalCpuConfig(NumaConfigBase): 获取 MYSQL_CPU_INFO：将 NUMA 节点内移除绑定的 CPU 后，剩余 CPU 转换为 NUMA 节点范围字符串形式 """ # 深拷贝 NUMA 节点信息 - remaining_numa_info = {numa_id: list(cpu_list) for numa_id, cpu_list in self.numa_info_dict.items()} + remaining_numa_info = { + numa_id: list(cpu_list) for numa_id, cpu_list in self.numa_info_dict.items() + } # 从 NUMA 节点中移除已绑定的 CPU for cpu in self.bind_cpu_list: @@ -460,7 +516,9 @@ class PhysicalCpuConfig(NumaConfigBase): # 如果存在 MES_MODULE，也需要写入cantiand.ini mes_module_key = f"{MES_MODULE}_ID" if cpu_config_info.get(mes_module_key): - cantiand_cpu_info[MES_CPU_INFO] = cpu_list_to_cpu_info(cpu_config_info[mes_module_key]) + cantiand_cpu_info[MES_CPU_INFO] = cpu_list_to_cpu_info( + cpu_config_info[mes_module_key] + ) else: cantiand_cpu_info[MES_CPU_INFO] = "-del" @@ -476,17 +534,18 @@ class PhysicalCpuConfig(NumaConfigBase): self.update_cantian_mysql_info() - class ContainerCpuConfig(NumaConfigBase): def __init__(self): super().__init__() def update_cpu_info(self): - """ 获取容器中的所有 CPU 列表 """ - if not os.path.exists('/sys/fs/cgroup/cpuset/cpuset.cpus'): + """获取容器中的所有 CPU 列表""" + if not os.path.exists("/sys/fs/cgroup/cpuset/cpuset.cpus"): raise Exception("cpuset.cpus path does not exist in container.") - ret_code, result, stderr = self._exec_popen('cat /sys/fs/cgroup/cpuset/cpuset.cpus') + ret_code, result, stderr = self._exec_popen( + "cat /sys/fs/cgroup/cpuset/cpuset.cpus" + ) if ret_code: raise Exception(f"Failed to get CPU list in container, err: {stderr}") @@ -521,7 +580,9 @@ class ContainerCpuConfig(NumaConfigBase): module_info = int(module_info) if not (1 <= module_info <= 10): numa_config[module_id_key] = "" - LOGGER.warning(f"Module {module_name} thread number out of range (1-10).") + LOGGER.warning( + f"Module {module_name} thread number out of range (1-10)." + ) continue except ValueError: numa_config[module_id_key] = "" @@ -575,7 +636,9 @@ class ContainerCpuConfig(NumaConfigBase): # 如果存在 MES_MODULE，也需要写入cantiand.ini mes_module_key = f"{MES_MODULE}_ID" if mes_module_key in cpu_config_info and cpu_config_info[mes_module_key]: - cantiand_cpu_info[MES_CPU_INFO] = cpu_list_to_cpu_info(cpu_config_info[mes_module_key]) + cantiand_cpu_info[MES_CPU_INFO] = cpu_list_to_cpu_info( + cpu_config_info[mes_module_key] + ) else: cantiand_cpu_info[MES_CPU_INFO] = "-del" @@ -605,7 +668,7 @@ class ConfigManager: write_json_config_file(CONFIG_PATH, numa_config) def update_cpu_config(self): - """ 更新 CPU 配置 """ + """更新 CPU 配置""" if get_value("cantian_in_container") == "0": manager = PhysicalCpuConfig() else: @@ -625,4 +688,4 @@ if __name__ == "__main__": else: config_manager.update_cpu_config() except Exception as e: - LOGGER.error(f"An unexpected error occurred: {str(e)}") \ No newline at end of file + LOGGER.error(f"An unexpected error occurred: {str(e)}") diff --git a/pkg/deploy/action/cantian/cantian_funclib.py b/pkg/deploy/action/cantian/cantian_funclib.py index 8783d388..f733e662 100644 --- a/pkg/deploy/action/cantian/cantian_funclib.py +++ b/pkg/deploy/action/cantian/cantian_funclib.py @@ -5,6 +5,7 @@ import sys import grp + sys.dont_write_bytecode = True try: import os @@ -30,15 +31,17 @@ except ImportError as import_err: py_verion = platform.python_version() -SYS_PATH = os.environ["PATH"].split(':') +SYS_PATH = os.environ["PATH"].split(":") class CommonValue(object): """ common value for some variables """ + def __init__(self): pass + # file mode MAX_FILE_MODE = 640 MIN_FILE_MODE = 400 @@ -52,7 +55,7 @@ class CommonValue(object): MID_DIRECTORY_MODE_GROUP = 740 MAX_DIRECTORY_MODE = 750 - KEY_DIRECTORY_MODE_STR = '0700' + KEY_DIRECTORY_MODE_STR = "0700" MIN_FILE_PERMISSION = 0o400 MID_FILE_PERMISSION = 0o500 @@ -64,6 +67,7 @@ class DefaultConfigValue(object): """ default value for cantiand, cms, gss config """ + def __init__(self): pass @@ -86,7 +90,11 @@ class DefaultConfigValue(object): "BUFFER_LRU_SEARCH_THRE": 40, "BUFFER_PAGE_CLEAN_RATIO": 0.1, "_DEADLOCK_DETECT_INTERVAL": 1000, - "INTERCONNECT_CHANNEL_NUM": 3 if (mes_type == "UC" or mes_type == "UC_RDMA") and deploy_mode != "file" else 32, + "INTERCONNECT_CHANNEL_NUM": ( + 3 + if (mes_type == "UC" or mes_type == "UC_RDMA") and deploy_mode != "file" + else 32 + ), "_UNDO_AUTO_SHRINK": "FALSE", "_CHECKPOINT_TIMED_TASK_DELAY": 100, "DBWR_PROCESSES": 8, @@ -147,7 +155,11 @@ class DefaultConfigValue(object): "INSTANCE_ID": 0, "INTERCONNECT_PORT": "1601", "LSNR_PORT": 1611, - "INTERCONNECT_TYPE": mes_type if (mes_type == "UC" or mes_type == "UC_RDMA") and deploy_mode != "file" else "TCP", + "INTERCONNECT_TYPE": ( + mes_type + if (mes_type == "UC" or mes_type == "UC_RDMA") and deploy_mode != "file" + else "TCP" + ), "INTERCONNECT_BY_PROFILE": "FALSE", "INSTANCE_NAME": "cantian", "ENABLE_SYSDBA_LOGIN": "FALSE", @@ -161,8 +173,8 @@ class DefaultConfigValue(object): "LSNR_ADDR": "127.0.0.1", "SHARED_PATH": "", "ARCHIVE_DEST_1": "", - "MAX_ARCH_FILES_SIZE" : "300G", - "PAGE_CLEAN_MODE" : "ALL", + "MAX_ARCH_FILES_SIZE": "300G", + "PAGE_CLEAN_MODE": "ALL", "ENABLE_IDX_KEY_LEN_CHECK": "FALSE", "EMPTY_STRING_AS_NULL": "FALSE", "_CHECKPOINT_MERGE_IO": "FALSE", @@ -174,9 +186,9 @@ class DefaultConfigValue(object): "SHM_MEMORY_REDUCTION_RATIO": "1", "MYSQL_DEPLOY_GROUP_ID": mysql_group_id, "CTC_MAX_INST_PER_NODE": 6, - "SQL_STATISTIC_TIME_LIMIT" :1000000 + "SQL_STATISTIC_TIME_LIMIT": 1000000, } - + CANTIAND_DBG_CONFIG = { "DBWR_PROCESSES": 8, "SESSIONS": 8192, @@ -209,7 +221,7 @@ class DefaultConfigValue(object): "CR_POOL_SIZE": "2G", "CR_POOL_COUNT": 4, "VARIANT_MEMORY_AREA_SIZE": "1G", - "REPLAY_PRELOAD_PROCESSES":0, + "REPLAY_PRELOAD_PROCESSES": 0, "LOG_REPLAY_PROCESSES": 64, "_LOG_MAX_FILE_SIZE": "1G", "RECYCLEBIN": "FALSE", @@ -221,7 +233,11 @@ class DefaultConfigValue(object): "INSTANCE_ID": 0, "INTERCONNECT_PORT": "1601", "LSNR_PORT": 1611, - "INTERCONNECT_TYPE": mes_type if (mes_type == "UC" or mes_type == "UC_RDMA") and deploy_mode != "file" else "TCP", + "INTERCONNECT_TYPE": ( + mes_type + if (mes_type == "UC" or mes_type == "UC_RDMA") and deploy_mode != "file" + else "TCP" + ), "INTERCONNECT_BY_PROFILE": "FALSE", "INSTANCE_NAME": "cantian", "ENABLE_SYSDBA_LOGIN": "FALSE", @@ -244,23 +260,23 @@ class DefaultConfigValue(object): "KMC_KEY_FILES": None, "SHM_MEMORY_REDUCTION_RATIO": "1", "MYSQL_DEPLOY_GROUP_ID": mysql_group_id, - "CTC_MAX_INST_PER_NODE": 6 + "CTC_MAX_INST_PER_NODE": 6, } MES_CONFIG = { "MES_SSL_SWITCH": mes_ssl_switch, "MES_SSL_KEY_PWD": None, "MES_SSL_CRT_KEY_PATH": "/opt/cantian/common/config/certificates", - "KMC_KEY_FILES": f"({PRIMARY_KEYSTORE}, {STANDBY_KEYSTORE})" + "KMC_KEY_FILES": f"({PRIMARY_KEYSTORE}, {STANDBY_KEYSTORE})", } if deploy_mode == "dss": - CANTIAND_CONFIG.update({ - "CTSTORE_INST_PATH": "UDS:/opt/cantian/dss/.dss_unix_d_socket" - }) - CANTIAND_DBG_CONFIG.update({ - "CTSTORE_INST_PATH": "UDS:/opt/cantian/dss/.dss_unix_d_socket" - }) - + CANTIAND_CONFIG.update( + {"CTSTORE_INST_PATH": "UDS:/opt/cantian/dss/.dss_unix_d_socket"} + ) + CANTIAND_DBG_CONFIG.update( + {"CTSTORE_INST_PATH": "UDS:/opt/cantian/dss/.dss_unix_d_socket"} + ) + CANTIAND_CONFIG.update(MES_CONFIG) CANTIAND_DBG_CONFIG.update(MES_CONFIG) @@ -320,8 +336,13 @@ def exec_popen(cmd): :return: status code, standard output, error output """ bash_cmd = ["bash"] - pobj = subprocess.Popen(bash_cmd, shell=False, stdin=subprocess.PIPE, - stdout=subprocess.PIPE, stderr=subprocess.PIPE) + pobj = subprocess.Popen( + bash_cmd, + shell=False, + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + ) if py_verion[0] == "3": stdout, stderr = pobj.communicate(cmd.encode(), timeout=1800) @@ -377,59 +398,97 @@ def check_path(path_type_in): def system_check_linux(path_len, path_type_in): i = 0 - a_ascii, a_uppercase_ascii, blank_ascii, num0_ascii, num9_ascii, sep1_ascii, sep2_ascii,\ - sep3_ascii, sep4_ascii, sep5_ascii, z_ascii, z_uppercase_ascii = check_ascii() + ( + a_ascii, + a_uppercase_ascii, + blank_ascii, + num0_ascii, + num9_ascii, + sep1_ascii, + sep2_ascii, + sep3_ascii, + sep4_ascii, + sep5_ascii, + z_ascii, + z_uppercase_ascii, + ) = check_ascii() ascii_list = [blank_ascii, sep1_ascii, sep2_ascii, sep4_ascii, sep5_ascii] for i in range(0, path_len): char_check = ord(path_type_in[i]) - if not (a_ascii <= char_check <= z_ascii - or a_uppercase_ascii <= char_check <= z_uppercase_ascii - or num0_ascii <= char_check <= num9_ascii - or char_check in ascii_list): + if not ( + a_ascii <= char_check <= z_ascii + or a_uppercase_ascii <= char_check <= z_uppercase_ascii + or num0_ascii <= char_check <= num9_ascii + or char_check in ascii_list + ): return False return True def system_check_windows(path_len, path_type_in): i = 0 - a_ascii, a_uppercase_ascii, blank_ascii, num0_ascii, num9_ascii, sep1_ascii, sep2_ascii,\ - sep3_ascii, sep4_ascii, sep5_ascii, z_ascii, z_uppercase_ascii = check_ascii() + ( + a_ascii, + a_uppercase_ascii, + blank_ascii, + num0_ascii, + num9_ascii, + sep1_ascii, + sep2_ascii, + sep3_ascii, + sep4_ascii, + sep5_ascii, + z_ascii, + z_uppercase_ascii, + ) = check_ascii() ascii_list = [blank_ascii, sep1_ascii, sep2_ascii, sep3_ascii, sep4_ascii] for i in range(0, path_len): char_check = ord(path_type_in[i]) - if not (a_ascii <= char_check <= z_ascii - or a_uppercase_ascii <= char_check <= z_uppercase_ascii - or num0_ascii <= char_check <= num9_ascii - or char_check in ascii_list): + if not ( + a_ascii <= char_check <= z_ascii + or a_uppercase_ascii <= char_check <= z_uppercase_ascii + or num0_ascii <= char_check <= num9_ascii + or char_check in ascii_list + ): return False return True def check_ascii(): - a_ascii = ord('a') - z_ascii = ord('z') - a_uppercase_ascii = ord('A') - z_uppercase_ascii = ord('Z') - num0_ascii = ord('0') - num9_ascii = ord('9') - blank_ascii = ord(' ') + a_ascii = ord("a") + z_ascii = ord("z") + a_uppercase_ascii = ord("A") + z_uppercase_ascii = ord("Z") + num0_ascii = ord("0") + num9_ascii = ord("9") + blank_ascii = ord(" ") sep1_ascii = ord(os.sep) - sep2_ascii = ord('_') - sep3_ascii = ord(':') - sep4_ascii = ord('-') - sep5_ascii = ord('.') + sep2_ascii = ord("_") + sep3_ascii = ord(":") + sep4_ascii = ord("-") + sep5_ascii = ord(".") ascii_list = [ - a_ascii, a_uppercase_ascii, blank_ascii, num0_ascii, num9_ascii, sep1_ascii, sep2_ascii, - sep3_ascii, sep4_ascii, sep5_ascii, z_ascii, z_uppercase_ascii + a_ascii, + a_uppercase_ascii, + blank_ascii, + num0_ascii, + num9_ascii, + sep1_ascii, + sep2_ascii, + sep3_ascii, + sep4_ascii, + sep5_ascii, + z_ascii, + z_uppercase_ascii, ] return ascii_list def check_ssh_connection(ips): - ''' + """ check ssh connection without password, if success to connect the node user trust to the node has be created - ''' + """ failed_ip = [] success_ip = [] ssh = get_abs_path("ssh") @@ -441,30 +500,39 @@ def check_ssh_connection(ips): cmd += "-o ServerAliveInterval=100 -o ServerAliveCountMax=36 " cmd += "-n 'echo Last login'" process = Execution(cmd) - idx =\ - process.expect(['Permission denied', - 'Last login', - 'Are you sure you want to continue connecting', - 'Password', 'ssh:', TimeoutException, - EOFException], 60) + idx = process.expect( + [ + "Permission denied", + "Last login", + "Are you sure you want to continue connecting", + "Password", + "ssh:", + TimeoutException, + EOFException, + ], + 60, + ) if idx == 0: failed_ip.append(ip) elif idx == 1: success_ip.append(ip) process.send_line("exit") elif idx == 2: - process.send_line('yes') - idx = process.expect(['Permission denied', 'Last login', - 'Password', 'ssh:'], 60) + process.send_line("yes") + idx = process.expect( + ["Permission denied", "Last login", "Password", "ssh:"], 60 + ) if idx == 0: failed_ip.append(ip) elif idx == 1: success_ip.append(ip) process.send_line("exit") elif idx == 2: - raise Exception("Check ssh connection" - " failed,check your ssh" - " configure file please.") + raise Exception( + "Check ssh connection" + " failed,check your ssh" + " configure file please." + ) elif idx == 3: raise Exception(str(process.context_buffer)) @@ -478,10 +546,12 @@ def check_ssh_connection(ips): # password authentication method, # so we must expect Password key word # to avoid to wait to timeout - raise Exception("Check ssh" - " connection failed," - " check your ssh" - " configure file please.") + raise Exception( + "Check ssh" + " connection failed," + " check your ssh" + " configure file please." + ) elif idx == 4: raise Exception(str(process.context_buffer)) @@ -495,6 +565,7 @@ class CommandTool(object): """ class for CommandTool """ + def __init__(self, log): self.log = log @@ -507,16 +578,16 @@ class CommandTool(object): raise SshToolException("Can't find bash command.") def __execute(self, arg): - ''' + """ execute shell command by ssh to login remote host arg - list for argument, ip address and shell command - ''' + """ ip = arg[0] cmd = arg[1] ssh_options = " -o ServerAliveInterval=100 " ssh_options += " -o ServerAliveCountMax=36 " cmd = "export TMOUT=0; %s" % cmd - ssh_cmd = "ssh %s %s \"%s\"" % (ssh_options, ip, cmd) + ssh_cmd = 'ssh %s %s "%s"' % (ssh_options, ip, cmd) return [ip, exec_popen(ssh_cmd)] def __scp(self, arg): @@ -534,14 +605,14 @@ class CommandTool(object): def __interactive_input(self, process, ip, pw1, pw2): - pw_str = 'Please enter password' + pw_str = "Please enter password" self.log("Expect(%s) on: [%s]" % (ip, pw_str)) - process.expect(['Please enter password']) + process.expect(["Please enter password"]) self.log("Send(%s) password." % ip) process.send_line(pw1) if pw2: self.log("Expect(%s) on: [%s]" % (ip, pw_str)) - process.expect(['Please enter password']) + process.expect(["Please enter password"]) self.log("Send(%s) password." % ip) process.send_line(pw2) @@ -567,9 +638,7 @@ class CommandTool(object): if ip: process = Execution("%s %s %s" % (self.ssh, ssh_options, ip)) pdict = user[1] - self.log("ssh session info:\n%s %s %s" % (self.ssh, - ssh_options, - ip)) + self.log("ssh session info:\n%s %s %s" % (self.ssh, ssh_options, ip)) else: process = Execution("%s" % (self.bash)) self.log("bash session") @@ -592,19 +661,19 @@ class CommandTool(object): self.__interactive_input(process, ip, user[1], user[2]) self.log("Expect(%s) on: [Done, Upgrade Failed]" % ip) - idx = process.expect(['Done', 'Upgrade Failed'], timeout=51200) + idx = process.expect(["Done", "Upgrade Failed"], timeout=51200) if idx == 0: self.log("Expect(%s) received Done." % ip) - process.send_line('exit') - return [ip, ('0', str(process.context_before))] + process.send_line("exit") + return [ip, ("0", str(process.context_before))] self.log("Expect(%s) received Upgrade Failed." % ip) - process.send_line('exit') - return [ip, ('1', str(process.context_buffer))] + process.send_line("exit") + return [ip, ("1", str(process.context_buffer))] except (TimeoutException, EOFException) as err: self.log("Expect(%s) timeout." % ip) if process: - process.send_line('exit') - return [ip, ('1', str(err) + '\n' + str(process.context_buffer))] + process.send_line("exit") + return [ip, ("1", str(err) + "\n" + str(process.context_buffer))] def execute_local(self, cmd): ret_code, output, errput = exec_popen(cmd) @@ -612,27 +681,25 @@ class CommandTool(object): return ret_code, output def expect_execute(self, ip_cmd_map): - ''' + """ execute shell command with expect - ''' + """ try: pool = Pool(len(ip_cmd_map)) result = pool.map(self.__expect_execute, ip_cmd_map) return self.__parse(result) except KeyboardInterrupt as e: - #captured and processed by the caller + # captured and processed by the caller raise def execute_in_node(self, ip_cmd_map): - ''' - ''' + """ """ pool = Pool(len(ip_cmd_map)) result = pool.map(self.__execute, ip_cmd_map) return self.__parse(result) def scp_in_node(self, ip_dest_map): - ''' - ''' + """ """ pool = Pool(len(ip_dest_map)) result = pool.map(self.__scp, ip_dest_map) return self.__parse(result) @@ -647,23 +714,22 @@ class CommandTool(object): success_node = [] failed_node = [] for tmp_rs in result: - if str(rs[1][0]) != '0': + if str(rs[1][0]) != "0": ret_code = 1 failed_node.append(tmp_rs) success_node.append(tmp_rs) return ret_code, success_node, failed_node def expect_ctsql(self, ip_cmd_map): - ''' + """ expect execute ctsql and sql command - ''' + """ pool = Pool(len(ip_cmd_map)) result = pool.map(self.__expect_ctsql, ip_cmd_map) return self.__parse(result) def __expect_ctsql(self, arg): - ''' - ''' + """ """ ip = arg[0] ctsql = arg[1] sql = arg[2] @@ -679,21 +745,22 @@ class CommandTool(object): process.send_line(ctsql) if passwd: - process.expect(['Please enter password']) + process.expect(["Please enter password"]) process.send_line(passwd) - process.expect(['SQL>']) + process.expect(["SQL>"]) process.send_line(sql) - idx = process.expect(['rows fetched', 'Succeed', 'CT-', 'SQL>'], - timeout=600) + idx = process.expect( + ["rows fetched", "Succeed", "CT-", "SQL>"], timeout=600 + ) if idx == 0 or idx == 1: - process.send_line('exit') - return [ip, ('0', str(process.context_before))] - process.send_line('exit') - return [ip, '1', str(process.context_buffer)] + process.send_line("exit") + return [ip, ("0", str(process.context_before))] + process.send_line("exit") + return [ip, "1", str(process.context_buffer)] except (TimeoutException, EOFException): if process: - process.send_line('exit') - return [ip, ('1', str(process.context_buffer))] + process.send_line("exit") + return [ip, ("1", str(process.context_buffer))] class ExpectException(Exception): @@ -718,13 +785,16 @@ class Execution(object): if py_verion[0] == "3": ALLOWED_STRING_TYPES = (str,) else: - ALLOWED_STRING_TYPES = (type(b''), type(''), type(u''),) + ALLOWED_STRING_TYPES = ( + type(b""), + type(""), + type(""), + ) LINE_SEPERATOR = os.linesep - CTRLF = '\r\n' + CTRLF = "\r\n" - def __init__(self, command, timeout=1800, max_read_size=4096, - delimiter=None): + def __init__(self, command, timeout=1800, max_read_size=4096, delimiter=None): self.matcher = None self.context_before = None @@ -748,11 +818,13 @@ class Execution(object): try: from termios import CEOF from termios import CINTR + (self._INTR, self._EOF) = (CINTR, CEOF) except ImportError: try: from termios import VEOF from termios import VINTR + tmp_fp = sys.__stdin__.fileno() self._INTR = ord(termios.tcgetattr(tmp_fp)[6][VINTR]) self._EOF = ord(termios.tcgetattr(tmp_fp)[6][VEOF]) @@ -763,13 +835,13 @@ class Execution(object): @staticmethod def _ascii(content): if not isinstance(content, bytes): - return content.encode('ascii') + return content.encode("ascii") return content @staticmethod def _utf8(content): if not isinstance(content, bytes): - return content.encode('utf-8') + return content.encode("utf-8") return content def __del__(self): @@ -781,29 +853,29 @@ class Execution(object): def __str__(self): tmp_s = list() - tmp_s.append('%r' % self) - tmp_s.append('after: %r' % self.context_after) - tmp_s.append('pid: %s' % str(self.child_pid)) - tmp_s.append('child_fd: %s' % str(self.child_fd)) - tmp_s.append('closed: %s' % str(self.is_closed)) - tmp_s.append('timeout: %s' % str(self.timeout)) - tmp_s.append('delimiter: %s' % str(self.delimiter)) - tmp_s.append('maxReadSize: %s' % str(self.max_read_size)) - return '\n'.join(tmp_s) + tmp_s.append("%r" % self) + tmp_s.append("after: %r" % self.context_after) + tmp_s.append("pid: %s" % str(self.child_pid)) + tmp_s.append("child_fd: %s" % str(self.child_fd)) + tmp_s.append("closed: %s" % str(self.is_closed)) + tmp_s.append("timeout: %s" % str(self.timeout)) + tmp_s.append("delimiter: %s" % str(self.delimiter)) + tmp_s.append("maxReadSize: %s" % str(self.max_read_size)) + return "\n".join(tmp_s) def _excute(self, command): self.args = shlex.split(command) if self.child_pid is not None: - raise ExpectException('The pid member must be None.') + raise ExpectException("The pid member must be None.") if self.command is None: - raise ExpectException('The command member must not be None.') + raise ExpectException("The command member must not be None.") try: self.child_pid, self.child_fd = pty.fork() except OSError as err: # pragma: no cover - raise ExpectException('pty.fork() failed: ' + str(err)) from err + raise ExpectException("pty.fork() failed: " + str(err)) from err if self.child_pid == pty.CHILD: # child @@ -843,19 +915,18 @@ class Execution(object): # give kernel time to update process status. time.sleep(self.close_delay) if self.is_alive() and not self.terminate(): - raise ExpectException('Could not terminate the child.') + raise ExpectException("Could not terminate the child.") self.child_fd = -1 self.is_closed = True def set_echo(self, state): - err_msg = ('method set_echo() may not be available on' - ' this operating system.') + err_msg = "method set_echo() may not be available on" " this operating system." try: child_attr = termios.tcgetattr(self.child_fd) except termios.error as e: if e.args[0] == errno.EINVAL: - raise IOError(e.args[0], '%s: %s.' % (e.args[1], err_msg)) from e + raise IOError(e.args[0], "%s: %s." % (e.args[1], err_msg)) from e raise if state: @@ -867,12 +938,12 @@ class Execution(object): termios.tcsetattr(self.child_fd, termios.TCSANOW, child_attr) except IOError as e: if e.args[0] == errno.EINVAL: - raise IOError(e.args[0], '%s: %s.' % (e.args[1], err_msg)) from e + raise IOError(e.args[0], "%s: %s." % (e.args[1], err_msg)) from e raise def read_non_block(self, size=1, timeout=-1): if self.is_closed: - raise ValueError('I/O operation on closed file.') + raise ValueError("I/O operation on closed file.") if timeout == -1: timeout = self.timeout @@ -882,16 +953,16 @@ class Execution(object): rfds, _, _ = self.select([self.child_fd], [], [], 0) if not rfds: self.eof_flag = True - raise EOFException('End Of File (EOF). Braindead platform.') + raise EOFException("End Of File (EOF). Braindead platform.") rfds, _, _ = self.select([self.child_fd], [], [], timeout) if not rfds: if not self.is_alive(): self.eof_flag = True - raise EOFException('Reach end of File (EOF).') + raise EOFException("Reach end of File (EOF).") else: - raise TimeoutException('Timeout exceeded.') + raise TimeoutException("Timeout exceeded.") if self.child_fd in rfds: try: @@ -899,17 +970,19 @@ class Execution(object): except OSError as e: if e.args[0] == errno.EIO: self.eof_flag = True - raise EOFException('Reach End Of File (EOF). ' - 'Exception style platform.') from e + raise EOFException( + "Reach End Of File (EOF). " "Exception style platform." + ) from e raise - if child_data == b'': + if child_data == b"": self.eof_flag = True - raise EOFException('Reach end Of File (EOF).' - ' Empty string style platform.') + raise EOFException( + "Reach end Of File (EOF)." " Empty string style platform." + ) return child_data - raise ExpectException('Reached an unexpected state.') + raise ExpectException("Reached an unexpected state.") # pragma: no cover def read(self, size=-1, timeout=-1): @@ -926,7 +999,7 @@ class Execution(object): def _send(self, content): return os.write(self.child_fd, content) - def send_line(self, content=''): + def send_line(self, content=""): send_count = self.send(content) send_count = send_count + self.send(self.LINE_SEPERATOR) return send_count @@ -978,7 +1051,7 @@ class Execution(object): if os.WIFEXITED(child_status) or os.WIFSIGNALED(child_status): self.is_terminated = True elif os.WIFSTOPPED(child_status): - raise ExpectException('process already been stopped.') + raise ExpectException("process already been stopped.") def wait_child_process(self, wait_pid_options): try: @@ -986,7 +1059,7 @@ class Execution(object): except OSError as error: # No child processes if error.errno == errno.ECHILD: - raise ExpectException('process already not exist.') from error + raise ExpectException("process already not exist.") from error else: raise error return child_pid, child_status @@ -1004,9 +1077,13 @@ class Execution(object): def raise_pattern_type_error(self, pattern): raise TypeError( - 'got %s as pattern, must be one' - ' of: %s, pexpect.EOFException, pexpect.TIMEOUTException' - % (type(pattern), ', '.join([str(ast) for ast in self.ALLOWED_STRING_TYPES]))) + "got %s as pattern, must be one" + " of: %s, pexpect.EOFException, pexpect.TIMEOUTException" + % ( + type(pattern), + ", ".join([str(ast) for ast in self.ALLOWED_STRING_TYPES]), + ) + ) def compile_pattern_list(self, pattern_list): if not pattern_list: @@ -1023,7 +1100,7 @@ class Execution(object): pattern_list_temp.append(EOFException) elif pattern is TimeoutException: pattern_list_temp.append(TimeoutException) - elif isinstance(pattern, type(re.compile(''))): + elif isinstance(pattern, type(re.compile(""))): pattern_list_temp.append(pattern) else: self.raise_pattern_type_error(pattern) @@ -1048,15 +1125,17 @@ class Execution(object): while True: match_index = re_searcher.search(context_buffer) if match_index > -1: - self.context_buffer = context_buffer[re_searcher.end:] + self.context_buffer = context_buffer[re_searcher.end :] self.context_before = context_buffer[: re_searcher.start] - self.context_after = context_buffer[re_searcher.start: re_searcher.end] + self.context_after = context_buffer[ + re_searcher.start : re_searcher.end + ] self.context_match = re_searcher.context_match self.matchIndex = match_index return self.matchIndex # no match at this point if (timeout is not None) and (timeout < 0): - raise TimeoutException('Timeout exceeded in loop_expect().') + raise TimeoutException("Timeout exceeded in loop_expect().") # not timed out, continue read more_context = self.read_non_block(self.max_read_size, timeout) time.sleep(0.0001) @@ -1091,8 +1170,8 @@ class Execution(object): raise def set_win_size(self, rows, cols): - win_size = getattr(termios, 'TIOCSWINSZ', -2146929561) - s_size = struct.pack('HHHH', rows, cols, 0, 0) + win_size = getattr(termios, "TIOCSWINSZ", -2146929561) + s_size = struct.pack("HHHH", rows, cols, 0, 0) fcntl.ioctl(self.fileno(), win_size, s_size) def select(self, inputs, outputs, errputs, timeout=None): @@ -1124,8 +1203,7 @@ class RESearcher(object): self.start = None self.context_match = None self.end = None - for index, pattern_item in zip(list(range(len(pattern_list))), - pattern_list): + for index, pattern_item in zip(list(range(len(pattern_list))), pattern_list): if pattern_item is EOFException: self.eof_index = index continue @@ -1138,21 +1216,23 @@ class RESearcher(object): result_list = list() for index, pattern_item in self._searches: try: - result_list.append((index, ' %d: re.compile("%s")' % - (index, pattern_item.pattern))) + result_list.append( + (index, ' %d: re.compile("%s")' % (index, pattern_item.pattern)) + ) except UnicodeEncodeError: - result_list.append((index, ' %d: re.compile(%r)' % - (index, pattern_item.pattern))) - result_list.append((-1, 'RESearcher:')) + result_list.append( + (index, " %d: re.compile(%r)" % (index, pattern_item.pattern)) + ) + result_list.append((-1, "RESearcher:")) if self.eof_index >= 0: - result_list.append((self.eof_index, ' %d: EOF' % - self.eof_index)) + result_list.append((self.eof_index, " %d: EOF" % self.eof_index)) if self.timeout_index >= 0: - result_list.append((self.timeout_index, ' %d: TIMEOUT' % - self.timeout_index)) + result_list.append( + (self.timeout_index, " %d: TIMEOUT" % self.timeout_index) + ) result_list.sort() s_result_list = list(zip(*result_list))[1] - return '\n'.join(s_result_list) + return "\n".join(s_result_list) def search(self, content): first_match_index = None diff --git a/pkg/deploy/action/cantian/cantian_install.py b/pkg/deploy/action/cantian/cantian_install.py index b8f854e2..50cd85ce 100644 --- a/pkg/deploy/action/cantian/cantian_install.py +++ b/pkg/deploy/action/cantian/cantian_install.py @@ -28,13 +28,21 @@ try: import collections from datetime import datetime from get_config_info import get_value - from cantian_funclib import CommonValue, SingleNodeConfig, ClusterNode0Config, \ - ClusterNode1Config, DefaultConfigValue + from cantian_funclib import ( + CommonValue, + SingleNodeConfig, + ClusterNode0Config, + ClusterNode1Config, + DefaultConfigValue, + ) from Common import CommonPrint from options import Options from exception import NormalException from log import LOGGER - sys.path.append(os.path.join(os.path.dirname(os.path.abspath(__file__)), "..", "dbstor")) + + sys.path.append( + os.path.join(os.path.dirname(os.path.abspath(__file__)), "..", "dbstor") + ) from kmc_adapter import CApiWrapper PYTHON242 = "2.4.2" @@ -43,10 +51,12 @@ try: if gPyVersion < PYTHON3: print_str = CommonPrint() - print_str.common_log("This install script can not support python version: %s" - % gPyVersion) - raise Exception("This install script can not support python version: %s" - % gPyVersion) + print_str.common_log( + "This install script can not support python version: %s" % gPyVersion + ) + raise Exception( + "This install script can not support python version: %s" % gPyVersion + ) sys.path.append(os.path.split(os.path.realpath(__file__))[0]) sys.dont_write_bytecode = True @@ -64,12 +74,24 @@ MYSQLD = "mysqld" USE_DBSTOR = ["combined", "dbstor"] USE_LUN = ["dss"] -INSTALL_SCRIPT = os.path.join(os.path.dirname(os.path.abspath(__file__)), "installdb.sh") - -VALID_RUNNING_MODE = {CANTIAND, CANTIAND_WITH_MYSQL, CANTIAND_WITH_MYSQL_ST, CANTIAND_IN_CLUSTER, - CANTIAND_WITH_MYSQL_IN_CLUSTER, MYSQLD} - -VALID_SINGLE_MYSQL_RUNNING_MODE = {CANTIAND_WITH_MYSQL_IN_CLUSTER, CANTIAND_WITH_MYSQL_ST, CANTIAND_WITH_MYSQL} +INSTALL_SCRIPT = os.path.join( + os.path.dirname(os.path.abspath(__file__)), "installdb.sh" +) + +VALID_RUNNING_MODE = { + CANTIAND, + CANTIAND_WITH_MYSQL, + CANTIAND_WITH_MYSQL_ST, + CANTIAND_IN_CLUSTER, + CANTIAND_WITH_MYSQL_IN_CLUSTER, + MYSQLD, +} + +VALID_SINGLE_MYSQL_RUNNING_MODE = { + CANTIAND_WITH_MYSQL_IN_CLUSTER, + CANTIAND_WITH_MYSQL_ST, + CANTIAND_WITH_MYSQL, +} CLUSTER_SIZE = 2 # default to 2, 4 node cluster mode need add parameter to specify this @@ -79,10 +101,12 @@ PKG_DIR = os.path.abspath(os.path.join(INSTALL_SCPRIT_DIR, "../..")) JS_CONF_FILE = os.path.join(PKG_DIR, "action", "cantian", "install_config.json") CANTIAN_CONF_FILE = os.path.join("/opt/cantian/cantian", "cfg", "cantian_config.json") CONFIG_PARAMS_FILE = os.path.join(PKG_DIR, "config", "deploy_param.json") -CANTIAN_START_STATUS_FILE = os.path.join("/opt/cantian/cantian", "cfg", "start_status.json") +CANTIAN_START_STATUS_FILE = os.path.join( + "/opt/cantian/cantian", "cfg", "start_status.json" +) CANTIAN_INSTALL_LOG_FILE = "/opt/cantian/log/cantian/cantian_deploy.log" CANTIAND_INI_FILE = "/mnt/dbdata/local/cantian/tmp/data/cfg/cantiand.ini" -CTSQL_INI_FILE = '/mnt/dbdata/local/cantian/tmp/data/cfg/*sql.ini' +CTSQL_INI_FILE = "/mnt/dbdata/local/cantian/tmp/data/cfg/*sql.ini" MYSQL_CODE_DIR = "/opt/cantian/image/cantian_connector/cantian-connector-mysql" MYSQL_BIN_DIR = "/opt/cantian/mysql/install/mysql" @@ -91,14 +115,36 @@ MYSQL_LOG_FILE = "" DEPLOY_MODE = "" g_opts = Options() -CheckPathsInfo = collections.namedtuple('CheckPathsInfo', ['path_len', 'path_type_in', 'a_ascii', - 'z_ascii', 'a_cap_ascii', 'z_cap_ascii', - 'num0_ascii', 'num9_ascii', 'char_check_list']) - -UnitConversionInfo = collections.namedtuple('UnitConversionInfo', ['tmp_gb', 'tmp_mb', 'tmp_kb', 'key', 'value', - 'sga_buff_size', 'temp_buffer_size', - 'data_buffer_size', 'shared_pool_size', - 'log_buffer_size']) +CheckPathsInfo = collections.namedtuple( + "CheckPathsInfo", + [ + "path_len", + "path_type_in", + "a_ascii", + "z_ascii", + "a_cap_ascii", + "z_cap_ascii", + "num0_ascii", + "num9_ascii", + "char_check_list", + ], +) + +UnitConversionInfo = collections.namedtuple( + "UnitConversionInfo", + [ + "tmp_gb", + "tmp_mb", + "tmp_kb", + "key", + "value", + "sga_buff_size", + "temp_buffer_size", + "data_buffer_size", + "shared_pool_size", + "log_buffer_size", + ], +) def check_kernel_parameter(para): @@ -116,12 +162,14 @@ def check_invalid_symbol(para): :param para: parameter's value :return: NA """ - symbols = ["|", "&", "$", ">", "<", "\"", "'", "`"] + symbols = ["|", "&", "$", ">", "<", '"', "'", "`"] for symbol in symbols: if para.find(symbol) > -1: print_str_1 = CommonPrint() - print_str_1.common_log("There is invalid symbol \"%s\" in %s" % (symbol, para)) - raise Exception("There is invalid symbol \"%s\" in %s" % (symbol, para)) + print_str_1.common_log( + 'There is invalid symbol "%s" in %s' % (symbol, para) + ) + raise Exception('There is invalid symbol "%s" in %s' % (symbol, para)) def all_zero_addr_after_ping(node_ip): @@ -132,7 +180,7 @@ def all_zero_addr_after_ping(node_ip): """ if not node_ip: return False - allowed_chars = set('0:.') + allowed_chars = set("0:.") if set(node_ip).issubset(allowed_chars): return True else: @@ -146,38 +194,46 @@ def check_path(path_type_in): :return: weather validity """ path_len = len(path_type_in) - a_ascii = ord('a') - z_ascii = ord('z') - a_cap_ascii = ord('A') - z_cap_ascii = ord('Z') - num0_ascii = ord('0') - num9_ascii = ord('9') - blank_ascii = ord(' ') + a_ascii = ord("a") + z_ascii = ord("z") + a_cap_ascii = ord("A") + z_cap_ascii = ord("Z") + num0_ascii = ord("0") + num9_ascii = ord("9") + blank_ascii = ord(" ") sep1_ascii = ord(os.sep) - sep2_ascii = ord('_') - sep3_ascii = ord(':') - sep4_ascii = ord('-') - sep5_ascii = ord('.') - char_check_list1 = [blank_ascii, - sep1_ascii, - sep2_ascii, - sep4_ascii, - sep5_ascii - ] - - char_check_list2 = [blank_ascii, - sep1_ascii, - sep2_ascii, - sep3_ascii, - sep4_ascii - ] + sep2_ascii = ord("_") + sep3_ascii = ord(":") + sep4_ascii = ord("-") + sep5_ascii = ord(".") + char_check_list1 = [blank_ascii, sep1_ascii, sep2_ascii, sep4_ascii, sep5_ascii] + + char_check_list2 = [blank_ascii, sep1_ascii, sep2_ascii, sep3_ascii, sep4_ascii] if CURRENT_OS == "Linux": - get_check_path_linux_info = CheckPathsInfo(path_len, path_type_in, a_ascii, z_ascii, a_cap_ascii, - z_cap_ascii, num0_ascii, num9_ascii, char_check_list1) + get_check_path_linux_info = CheckPathsInfo( + path_len, + path_type_in, + a_ascii, + z_ascii, + a_cap_ascii, + z_cap_ascii, + num0_ascii, + num9_ascii, + char_check_list1, + ) return check_path_linux(get_check_path_linux_info) elif CURRENT_OS == "Windows": - get_check_path_windows_info = CheckPathsInfo(path_len, path_type_in, a_ascii, z_ascii, a_cap_ascii, - z_cap_ascii, num0_ascii, num9_ascii, char_check_list2) + get_check_path_windows_info = CheckPathsInfo( + path_len, + path_type_in, + a_ascii, + z_ascii, + a_cap_ascii, + z_cap_ascii, + num0_ascii, + num9_ascii, + char_check_list2, + ) return check_path_windows(get_check_path_windows_info) else: print_str_1 = CommonPrint() @@ -186,27 +242,49 @@ def check_path(path_type_in): def check_path_linux(get_check_path_linux_info): - path_len, path_type_in, a_ascii, z_ascii, a_cap_ascii, \ - z_cap_ascii, num0_ascii, num9_ascii, char_check_list1 = get_check_path_linux_info + ( + path_len, + path_type_in, + a_ascii, + z_ascii, + a_cap_ascii, + z_cap_ascii, + num0_ascii, + num9_ascii, + char_check_list1, + ) = get_check_path_linux_info for i in range(0, path_len): char_check = ord(path_type_in[i]) - if (not (a_ascii <= char_check <= z_ascii - or a_cap_ascii <= char_check <= z_cap_ascii - or num0_ascii <= char_check <= num9_ascii - or char_check in char_check_list1)): + if not ( + a_ascii <= char_check <= z_ascii + or a_cap_ascii <= char_check <= z_cap_ascii + or num0_ascii <= char_check <= num9_ascii + or char_check in char_check_list1 + ): return False return True def check_path_windows(get_check_path_windows_info): - path_len, path_type_in, a_ascii, z_ascii, a_cap_ascii, \ - z_cap_ascii, num0_ascii, num9_ascii, char_check_list2 = get_check_path_windows_info + ( + path_len, + path_type_in, + a_ascii, + z_ascii, + a_cap_ascii, + z_cap_ascii, + num0_ascii, + num9_ascii, + char_check_list2, + ) = get_check_path_windows_info for i in range(0, path_len): char_check = ord(path_type_in[i]) - if (not (a_ascii <= char_check <= z_ascii - or a_cap_ascii <= char_check <= z_cap_ascii - or num0_ascii <= char_check <= num9_ascii - or char_check in char_check_list2)): + if not ( + a_ascii <= char_check <= z_ascii + or a_cap_ascii <= char_check <= z_cap_ascii + or num0_ascii <= char_check <= num9_ascii + or char_check in char_check_list2 + ): return False return True @@ -220,8 +298,13 @@ def _exec_popen(cmd, values=None): if not values: values = [] bash_cmd = ["bash"] - pobj = subprocess.Popen(bash_cmd, shell=False, stdin=subprocess.PIPE, - stdout=subprocess.PIPE, stderr=subprocess.PIPE) + pobj = subprocess.Popen( + bash_cmd, + shell=False, + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + ) if gPyVersion[0] == "3": pobj.stdin.write(cmd.encode()) @@ -281,7 +364,9 @@ def check_platform(): pass else: print_str_2 = CommonPrint() - print_str_2.common_log("This install script can not support %s platform." % CURRENT_OS) + print_str_2.common_log( + "This install script can not support %s platform." % CURRENT_OS + ) raise Exception("This install script can not support %s platform." % CURRENT_OS) @@ -291,28 +376,32 @@ def usage(): def load_config_param(json_data): - g_opts.node_id = int(json_data.get('node_id')) - if json_data.get('link_type', '0').strip() == '0': + g_opts.node_id = int(json_data.get("node_id")) + if json_data.get("link_type", "0").strip() == "0": g_opts.link_type = "TCP" - elif json_data.get('link_type', '0').strip() == '1': + elif json_data.get("link_type", "0").strip() == "1": g_opts.link_type = "RDMA" - elif json_data.get('link_type', '0').strip() == '2': + elif json_data.get("link_type", "0").strip() == "2": g_opts.link_type = "RDMA_1823" - if json_data.get('cantian_in_container', 0) == '1': + if json_data.get("cantian_in_container", 0) == "1": g_opts.cantian_in_container = True global DEPLOY_MODE DEPLOY_MODE = get_value("deploy_mode") - g_opts.db_type = json_data.get('db_type', '').strip() + g_opts.db_type = json_data.get("db_type", "").strip() g_opts.storage_dbstor_fs = json_data.get("storage_dbstor_fs", "").strip() - g_opts.storage_share_fs = json_data.get('storage_share_fs', "").strip() - g_opts.namespace = json_data.get('cluster_name', 'test1').strip() - g_opts.share_logic_ip = json_data.get('share_logic_ip', '127.0.0.1').strip() if DEPLOY_MODE == "file" else None - g_opts.archive_logic_ip = json_data.get('archive_logic_ip', '127.0.0.1').strip() + g_opts.storage_share_fs = json_data.get("storage_share_fs", "").strip() + g_opts.namespace = json_data.get("cluster_name", "test1").strip() + g_opts.share_logic_ip = ( + json_data.get("share_logic_ip", "127.0.0.1").strip() + if DEPLOY_MODE == "file" + else None + ) + g_opts.archive_logic_ip = json_data.get("archive_logic_ip", "127.0.0.1").strip() g_opts.mes_type = json_data.get("mes_type", "UC").strip() if DEPLOY_MODE == "file": g_opts.mes_type = "TCP" g_opts.mes_ssl_switch = json_data.get("mes_ssl_switch", False) - storage_archive_fs = json_data.get('storage_archive_fs', "").strip() + storage_archive_fs = json_data.get("storage_archive_fs", "").strip() g_opts.use_dbstor = DEPLOY_MODE in USE_DBSTOR g_opts.use_gss = DEPLOY_MODE in USE_LUN g_opts.archive_location = f"""location=/{f'mnt/dbdata/remote/archive_{storage_archive_fs}' @@ -320,13 +409,13 @@ def load_config_param(json_data): if DEPLOY_MODE in USE_LUN: g_opts.archive_location = "location=+vg3/archive" g_opts.dbstor_deploy_mode = DEPLOY_MODE == "dbstor" - metadata_str = "metadata_" + json_data.get('storage_metadata_fs', '').strip() + metadata_str = "metadata_" + json_data.get("storage_metadata_fs", "").strip() node_str = "node" + str(g_opts.node_id) global MYSQL_DATA_DIR MYSQL_DATA_DIR = os.path.join("/mnt/dbdata/remote", metadata_str, node_str) global MYSQL_LOG_FILE MYSQL_LOG_FILE = os.path.join(MYSQL_DATA_DIR, "mysql.log") - g_opts.max_arch_files_size = json_data['MAX_ARCH_FILES_SIZE'].strip() + g_opts.max_arch_files_size = json_data["MAX_ARCH_FILES_SIZE"].strip() g_opts.cluster_id = json_data.get("cluster_id", "0").strip() @@ -334,8 +423,9 @@ def parse_parameter(): try: # Parameters are passed into argv. After parsing, they are stored # in opts as binary tuples. Unresolved parameters are stored in args. - opts, args = getopt.getopt(sys.argv[1:], - "h:s:t:", ["help", "sys_password=", "isencrept="]) + opts, args = getopt.getopt( + sys.argv[1:], "h:s:t:", ["help", "sys_password=", "isencrept="] + ) if args: print_str_1 = CommonPrint() print_str_1.common_log("Parameter input error: " + str(args[0])) @@ -344,22 +434,24 @@ def parse_parameter(): for _key, _value in opts: if _key == "-s": g_opts.password = _get_input("please input pwd: ").strip() - g_opts.cert_encrypt_pwd = _get_input("please input cert_encrypt_pwd:").strip() + g_opts.cert_encrypt_pwd = _get_input( + "please input cert_encrypt_pwd:" + ).strip() if _key == "-t": - if _value.strip() == 'reserve': + if _value.strip() == "reserve": g_opts.isencrept = False flags = os.O_RDONLY modes = stat.S_IWUSR | stat.S_IRUSR - with os.fdopen(os.open(JS_CONF_FILE, flags, modes), 'r') as fp: + with os.fdopen(os.open(JS_CONF_FILE, flags, modes), "r") as fp: json_data = json.load(fp) - g_opts.log_file = json_data['l_LOG_FILE'].strip() # -I - g_opts.running_mode = json_data['M_RUNING_MODE'].strip() # -M + g_opts.log_file = json_data["l_LOG_FILE"].strip() # -I + g_opts.running_mode = json_data["M_RUNING_MODE"].strip() # -M g_opts.ignore_pkg_check = True # -p flags = os.O_RDONLY modes = stat.S_IWUSR | stat.S_IRUSR - with os.fdopen(os.open(CONFIG_PARAMS_FILE, flags, modes), 'r') as fp: + with os.fdopen(os.open(CONFIG_PARAMS_FILE, flags, modes), "r") as fp: json_data = json.load(fp) load_config_param(json_data) g_opts.opts = opts @@ -375,27 +467,39 @@ def is_mlnx(): """ ret_code, stdout, stderr = _exec_popen("which ofed_info") if ret_code: - log("no ofed_info cmd found." - "ret_code : %s, stdout : %s, stderr : %s" % (ret_code, stdout, stderr)) + log( + "no ofed_info cmd found." + "ret_code : %s, stdout : %s, stderr : %s" % (ret_code, stdout, stderr) + ) return False ret_code, stdout, stderr = _exec_popen("ofed_info -s") if ret_code: - log("exec ofed_info cmd failed." - "ret_code : %s, stdout : %s, stderr : %s" % (ret_code, stdout, stderr)) + log( + "exec ofed_info cmd failed." + "ret_code : %s, stdout : %s, stderr : %s" % (ret_code, stdout, stderr) + ) return False - if 'MLNX_OFED_LINUX-5.5' in stdout: + if "MLNX_OFED_LINUX-5.5" in stdout: LOGGER.info("Is mlnx 5.5") return True ret_code, os_arch, stderr = _exec_popen("uname -i") - aarch_mlnx_version_list = ['OFED-internal-5.8-2.0.3', 'MLNX_OFED_LINUX-5.8', 'MLNX_OFED_LINUX-5.9'] - aarch_version_check_result = any(mlnx_version if mlnx_version in stdout else False - for mlnx_version in aarch_mlnx_version_list) + aarch_mlnx_version_list = [ + "OFED-internal-5.8-2.0.3", + "MLNX_OFED_LINUX-5.8", + "MLNX_OFED_LINUX-5.9", + ] + aarch_version_check_result = any( + mlnx_version if mlnx_version in stdout else False + for mlnx_version in aarch_mlnx_version_list + ) if os_arch == "aarch64" and aarch_version_check_result == True: - log("Is mlnx 5.8~5.9" - " ret_code : %s, stdout : %s, stderr : %s" % (ret_code, os_arch, stderr)) + log( + "Is mlnx 5.8~5.9" + " ret_code : %s, stdout : %s, stderr : %s" % (ret_code, os_arch, stderr) + ) return True LOGGER.info("Not mlnx 5.5") @@ -436,21 +540,38 @@ def check_parameter(): def check_running_mode(print_str_7): # Check running mode - if len(g_opts.running_mode) == 0 or g_opts.running_mode.lower() not in VALID_RUNNING_MODE: + if ( + len(g_opts.running_mode) == 0 + or g_opts.running_mode.lower() not in VALID_RUNNING_MODE + ): print_str_7.common_log("Invalid running mode: " + g_opts.running_mode) raise Exception("Invalid running mode: " + g_opts.running_mode) if g_opts.node_id not in [0, 1]: print_str_7.common_log("Invalid node id: " + g_opts.node_id) raise Exception("Invalid node id: " + g_opts.node_id) - if g_opts.running_mode.lower() in [CANTIAND, CANTIAND_WITH_MYSQL, CANTIAND_WITH_MYSQL_ST] and g_opts.node_id == 1: - print_str_7.common_log("Invalid node id: " + g_opts.node_id + ", this node id can only run in cluster mode") - raise Exception("Invalid node id: " + g_opts.node_id + ", this node id can only run in cluster mode") + if ( + g_opts.running_mode.lower() + in [CANTIAND, CANTIAND_WITH_MYSQL, CANTIAND_WITH_MYSQL_ST] + and g_opts.node_id == 1 + ): + print_str_7.common_log( + "Invalid node id: " + + g_opts.node_id + + ", this node id can only run in cluster mode" + ) + raise Exception( + "Invalid node id: " + + g_opts.node_id + + ", this node id can only run in cluster mode" + ) def check_logfile_path(print_str_7): # Check the legitimacy of the path logfile if not check_path(g_opts.log_file): - print_str_7.common_log("Error: There is invalid character in specified log file.") + print_str_7.common_log( + "Error: There is invalid character in specified log file." + ) raise Exception("Error: There is invalid character in specified log file.") @@ -459,11 +580,20 @@ def init_start_status_file(): try: flags = os.O_WRONLY | os.O_TRUNC | os.O_CREAT modes = stat.S_IWUSR | stat.S_IRUSR - start_parameters = {'start_status': 'default', 'db_create_status': 'default', 'ever_started': False , 'mysql_init': 'default'} - with os.fdopen(os.open(CANTIAN_START_STATUS_FILE, flags, modes), 'w') as load_fp: + start_parameters = { + "start_status": "default", + "db_create_status": "default", + "ever_started": False, + "mysql_init": "default", + } + with os.fdopen( + os.open(CANTIAN_START_STATUS_FILE, flags, modes), "w" + ) as load_fp: json.dump(start_parameters, load_fp) except IOError as ex: - print_str_7.common_log("Error: Can not create or write file: " + CANTIAN_START_STATUS_FILE) + print_str_7.common_log( + "Error: Can not create or write file: " + CANTIAN_START_STATUS_FILE + ) print_str_7.common_log(str(ex)) raise Exception(str(ex)) from ex @@ -477,13 +607,17 @@ def check_log_path(): if not os.path.isdir(dir_path): g_opts.log_file = "" print_str_1 = CommonPrint() - print_str_1.common_log("Specified log path: \"%s\" does not exist, " - "choose the default path instead." % dir_path) + print_str_1.common_log( + 'Specified log path: "%s" does not exist, ' + "choose the default path instead." % dir_path + ) elif not base_name: g_opts.log_file = "" print_str_2 = CommonPrint() - print_str_2.common_log("Log file does not been specified, " - "choose the default logfile instead.") + print_str_2.common_log( + "Log file does not been specified, " + "choose the default logfile instead." + ) def use_default_log_path(): @@ -526,7 +660,7 @@ def cantian_check_share_logic_ip_isvalid(ipname, nodeip): def ping_execute(p_cmd): cmd = "%s %s -i 1 -c 3 | grep ttl | wc -l" % (p_cmd, nodeip) ret_code, stdout, stderr = _exec_popen(cmd) - if ret_code or stdout != '3': + if ret_code or stdout != "3": return False return True @@ -537,15 +671,20 @@ def cantian_check_share_logic_ip_isvalid(ipname, nodeip): LOGGER.info("check nfs logic ip address or domain name.") if not ping_execute("ping") and not ping_execute("ping6"): - err_msg = "checked the node %s IP address or domain name failed: %s" % (ipname, nodeip) + err_msg = "checked the node %s IP address or domain name failed: %s" % ( + ipname, + nodeip, + ) LOGGER.error(err_msg) raise Exception(err_msg) - LOGGER.info("checked the node %s IP address or domain name success: %s" % (ipname, nodeip)) + LOGGER.info( + "checked the node %s IP address or domain name success: %s" % (ipname, nodeip) + ) def file_reader(file_path): - with open(file_path, 'r') as file: + with open(file_path, "r") as file: return file.read() @@ -557,29 +696,42 @@ class Platform(object): def __init__(self): pass - SUPPORTED_DISTS = ('suse', 'debian', 'fedora', 'redhat', 'centos', - 'mandrake', 'mandriva', 'rocks', 'slackware', - 'yellowdog', 'gentoo', 'unitedlinux', - 'turbolinux', 'arch', 'mageia', 'openeuler', - 'neokylin', 'euleros', 'kylin') - UNIXCONFDIR = '/etc' + SUPPORTED_DISTS = ( + "suse", + "debian", + "fedora", + "redhat", + "centos", + "mandrake", + "mandriva", + "rocks", + "slackware", + "yellowdog", + "gentoo", + "unitedlinux", + "turbolinux", + "arch", + "mageia", + "openeuler", + "neokylin", + "euleros", + "kylin", + ) + UNIXCONFDIR = "/etc" @staticmethod - def _parse_release_file(firstline, version='', dst_id=''): + def _parse_release_file(firstline, version="", dst_id=""): """ function: parse first line of /etc/*release input: string output: tuple(string, string, string) """ - lsb_release_version_re = r'(.+) release ([\d.]+)[^(]*(?:\((.+)\))?' - release_version_re = (r'([^0-9]+)(?: release )?([\d.]+)[^(]*' - r'(?:\((.+)\))?') + lsb_release_version_re = r"(.+) release ([\d.]+)[^(]*(?:\((.+)\))?" + release_version_re = r"([^0-9]+)(?: release )?([\d.]+)[^(]*" r"(?:\((.+)\))?" try: - lsb_release_version = re.compile(lsb_release_version_re, - re.ASCII) - release_version = re.compile(release_version_re, - re.ASCII) + lsb_release_version = re.compile(lsb_release_version_re, re.ASCII) + release_version = re.compile(release_version_re, re.ASCII) except AttributeError: lsb_release_version = re.compile(lsb_release_version_re) release_version = re.compile(release_version_re) @@ -602,7 +754,7 @@ class Platform(object): if len(line_list) > 1: dst_id = line_list[1] - return '', version, dst_id + return "", version, dst_id @staticmethod def dist(): @@ -621,9 +773,9 @@ class Platform(object): return "", "", "" try: - release_re = re.compile(r'(\w+)[-_](release|version)', re.ASCII) + release_re = re.compile(r"(\w+)[-_](release|version)", re.ASCII) except AttributeError: - release_re = re.compile(r'(\w+)[-_](release|version)') + release_re = re.compile(r"(\w+)[-_](release|version)") for etc_file in etc: tmp_m = release_re.match(etc_file) @@ -637,15 +789,17 @@ class Platform(object): etc_file_name = os.path.join(Platform.UNIXCONFDIR, etc_file) flags = os.O_RDONLY modes = stat.S_IWUSR | stat.S_IRUSR - with os.fdopen(os.open(etc_file_name, flags, modes), 'r') as f: + with os.fdopen(os.open(etc_file_name, flags, modes), "r") as f: firstline = f.readline() except Exception as error: LOGGER.info("read first line exception: %s." % str(error)) continue # when euler, has centos-release - if (_distname.lower() == "centos" and - _distname.lower() not in firstline.lower()): + if ( + _distname.lower() == "centos" + and _distname.lower() not in firstline.lower() + ): continue if _distname.lower() in Platform.SUPPORTED_DISTS: @@ -686,8 +840,7 @@ class ParameterContainer(object): index = self.ifiles.index(value) para_index = 0 for _ in range(index + 1): - para_index = self.parameters.index(self.IFILE, - para_index + 1) + para_index = self.parameters.index(self.IFILE, para_index + 1) self.parameters.pop(para_index) self.ifiles.pop(index) self.parameters.append(key) @@ -726,7 +879,10 @@ class ParameterContainer(object): def skip_execute_in_node_1(): - if g_opts.running_mode in [CANTIAND_IN_CLUSTER, CANTIAND_WITH_MYSQL_IN_CLUSTER] and g_opts.node_id == 1: + if ( + g_opts.running_mode in [CANTIAND_IN_CLUSTER, CANTIAND_WITH_MYSQL_IN_CLUSTER] + and g_opts.node_id == 1 + ): return True return False @@ -738,7 +894,7 @@ def create_dir_if_needed(condition, directory_addr): class Installer: - """ This is Cantiand installer. """ + """This is Cantiand installer.""" # Defining a constant identifies which step the installer failed to take. # For roll back. @@ -778,7 +934,7 @@ class Installer: ipv_type = "ipv4" def __init__(self, user, group): - """ Constructor for the Installer class. """ + """Constructor for the Installer class.""" LOGGER.info("Begin init...") LOGGER.info("Installer runs on python version : " + gPyVersion) @@ -885,14 +1041,14 @@ class Installer: data = sql_content % (node0_redo, node1_redo) modes = stat.S_IWRITE | stat.S_IRUSR flags = os.O_WRONLY | os.O_TRUNC | os.O_CREAT - with os.fdopen(os.open(sql_file, flags, modes), 'w', encoding='utf-8') as file: + with os.fdopen(os.open(sql_file, flags, modes), "w", encoding="utf-8") as file: file.write(data) def get_decompress_tarname(self, tar_file): - ''' + """ decompress a.tar.gz, then get file name :return: - ''' + """ # get real directory name in tar file tars = tarfile.open(tar_file) basename = tars.getnames()[0] @@ -900,52 +1056,54 @@ class Installer: return basename def is_readable(self, file_name, user): - ''' + """ :param path: :param user: :return: - ''' + """ user_info = pwd.getpwnam(user) uid = user_info.pw_uid gid = user_info.pw_gid s = os.stat(file_name) mode = s[stat.ST_MODE] return ( - ((s[stat.ST_UID] == uid) and (mode & stat.S_IRUSR > 0)) or - ((s[stat.ST_GID] == gid) and (mode & stat.S_IRGRP > 0)) or - (mode & stat.S_IROTH > 0) + ((s[stat.ST_UID] == uid) and (mode & stat.S_IRUSR > 0)) + or ((s[stat.ST_GID] == gid) and (mode & stat.S_IRGRP > 0)) + or (mode & stat.S_IROTH > 0) ) def check_createdb_file(self): - ''' + """ check it is a file; user has read permission, :return: - ''' + """ # check -f parameter if self.option != self.INS_ALL: - raise Exception("Error: -f parameter should be used without" - " -O parameter ") + raise Exception( + "Error: -f parameter should be used without" " -O parameter " + ) # check it is a file if not os.path.isfile(self.create_db_file): - raise Exception("Error: %s does not exists or is not a file" - " or permission is not right." - % self.create_db_file) + raise Exception( + "Error: %s does not exists or is not a file" + " or permission is not right." % self.create_db_file + ) if not check_path(self.create_db_file): - raise Exception("Error: %s file path invalid: " - % self.create_db_file) + raise Exception("Error: %s file path invalid: " % self.create_db_file) # if execute user is root, check common user has read permission file_path = os.path.dirname(self.create_db_file) # check path of create db sql file that user can cd permission_ok, _ = self.check_permission(file_path, True) if not permission_ok: - raise Exception("Error: %s can not access %s" - % (self.user, file_path)) + raise Exception("Error: %s can not access %s" % (self.user, file_path)) # check create db file is readable for user if not self.is_readable(self.create_db_file, self.user): - raise Exception("Error: %s is not readable for user %s" - % (self.create_db_file, self.user)) + raise Exception( + "Error: %s is not readable for user %s" + % (self.create_db_file, self.user) + ) # change file to a realpath file self.create_db_file = os.path.realpath(self.create_db_file) @@ -955,11 +1113,21 @@ class Installer: :return: cantiand config, cms config, gss config """ # 获取默认值 - if g_opts.running_mode in [CANTIAND, CANTIAND_WITH_MYSQL, CANTIAND_WITH_MYSQL_ST]: + if g_opts.running_mode in [ + CANTIAND, + CANTIAND_WITH_MYSQL, + CANTIAND_WITH_MYSQL_ST, + ]: self.cantiand_configs = SingleNodeConfig.get_config(False) - if g_opts.running_mode in [CANTIAND_IN_CLUSTER, CANTIAND_WITH_MYSQL_IN_CLUSTER] and g_opts.node_id == 0: + if ( + g_opts.running_mode in [CANTIAND_IN_CLUSTER, CANTIAND_WITH_MYSQL_IN_CLUSTER] + and g_opts.node_id == 0 + ): self.cantiand_configs = ClusterNode0Config.get_config(False) - if g_opts.running_mode in [CANTIAND_IN_CLUSTER, CANTIAND_WITH_MYSQL_IN_CLUSTER] and g_opts.node_id == 1: + if ( + g_opts.running_mode in [CANTIAND_IN_CLUSTER, CANTIAND_WITH_MYSQL_IN_CLUSTER] + and g_opts.node_id == 1 + ): self.cantiand_configs = ClusterNode1Config.get_config(False) def check_parameter(self): @@ -995,12 +1163,15 @@ class Installer: err_msg = "Parameter input error, need -U parameter." LOGGER.error(err_msg) raise Exception(err_msg) - os.environ['cantiand_user'] = str(self.user) + os.environ["cantiand_user"] = str(self.user) # User must be exist. str_cmd = "id -u ${cantiand_user}" ret_code, stdout, stderr = _exec_popen(str_cmd) if ret_code: - err_msg = "%s : no such user, command: %s ret_code : %s, stdout : %s, stderr : %s" % (self.user, str_cmd, ret_code, stdout, stderr) + err_msg = ( + "%s : no such user, command: %s ret_code : %s, stdout : %s, stderr : %s" + % (self.user, str_cmd, ret_code, stdout, stderr) + ) LOGGER.error(err_msg) raise Exception(err_msg) if self.option == self.INS_ALL: @@ -1032,12 +1203,14 @@ class Installer: LOGGER.error(err_msg) raise Exception(err_msg) if len(self.cantiand_configs.get("LOG_HOME")) == 0: - log_path=os.path.dirname(g_opts.log_file) + log_path = os.path.dirname(g_opts.log_file) self.cantiand_configs["LOG_HOME"] = log_path if len(self.cantiand_configs.get("SHARED_PATH")) == 0: self.cantiand_configs["SHARED_PATH"] = os.path.join(self.data, "data") if g_opts.use_dbstor: - self.cantiand_configs["DBSTOR_DEPLOY_MODE"] = "1" if g_opts.dbstor_deploy_mode else "0" + self.cantiand_configs["DBSTOR_DEPLOY_MODE"] = ( + "1" if g_opts.dbstor_deploy_mode else "0" + ) self.cantiand_configs["ARCHIVE_DEST_1"] = g_opts.archive_location self.cantiand_configs["MAX_ARCH_FILES_SIZE"] = g_opts.max_arch_files_size self.cantiand_configs["CLUSTER_ID"] = g_opts.cluster_id @@ -1064,52 +1237,59 @@ class Installer: LOGGER.info("End check parameters.") def add_config_for_dbstor(self): - self.cantiand_configs["CONTROL_FILES"] = "{0}, {1}, {2}".format(os.path.join(self.data, "data/ctrl1"), - os.path.join(self.data, "data/ctrl2"), - os.path.join(self.data, "data/ctrl3")) + self.cantiand_configs["CONTROL_FILES"] = "{0}, {1}, {2}".format( + os.path.join(self.data, "data/ctrl1"), + os.path.join(self.data, "data/ctrl2"), + os.path.join(self.data, "data/ctrl3"), + ) if g_opts.use_dbstor: self.cantiand_configs["CONTROL_FILES"] = "(-ctrl1, -ctrl2, -ctrl3)" self.cantiand_configs["SHARED_PATH"] = "-" self.cantiand_configs["ENABLE_DBSTOR"] = "TRUE" self.cantiand_configs["DBSTOR_NAMESPACE"] = g_opts.namespace elif g_opts.use_gss: - self.cantiand_configs["CONTROL_FILES"] = "(+vg1/ctrl1, +vg1/ctrl2, +vg1/ctrl3)" + self.cantiand_configs["CONTROL_FILES"] = ( + "(+vg1/ctrl1, +vg1/ctrl2, +vg1/ctrl3)" + ) self.cantiand_configs["ENABLE_DBSTOR"] = "FALSE" self.cantiand_configs["ENABLE_DSS"] = "TRUE" self.cantiand_configs["SHARED_PATH"] = "+vg1" else: self.cantiand_configs["ENABLE_DBSTOR"] = "FALSE" - self.cantiand_configs["SHARED_PATH"] = \ - '/mnt/dbdata/remote/storage_{}/data'.format(g_opts.storage_dbstor_fs) + self.cantiand_configs["SHARED_PATH"] = ( + "/mnt/dbdata/remote/storage_{}/data".format(g_opts.storage_dbstor_fs) + ) def parse_key_and_value(self): flags = os.O_RDONLY modes = stat.S_IWUSR | stat.S_IRUSR - with os.fdopen(os.open(CONFIG_PARAMS_FILE, flags, modes), 'r') as fp: + with os.fdopen(os.open(CONFIG_PARAMS_FILE, flags, modes), "r") as fp: json_data = json.load(fp) - _value = json_data.get('cms_ip', '0').strip() - self.cantiand_configs['INTERCONNECT_ADDR'] = _value + _value = json_data.get("cms_ip", "0").strip() + self.cantiand_configs["INTERCONNECT_ADDR"] = _value node_addr = _value.split(";")[g_opts.node_id] - self.cantiand_configs['LSNR_ADDR'] += "," + node_addr - _value = json_data.get('mysql_metadata_in_cantian', 'True') - self.cantiand_configs['MYSQL_METADATA_IN_CANTIAN'] = str(_value).upper() - _value = json_data.get('mes_ssl_switch', 'False') - self.cantiand_configs['MES_SSL_SWITCH'] = str(_value).upper() + self.cantiand_configs["LSNR_ADDR"] += "," + node_addr + _value = json_data.get("mysql_metadata_in_cantian", "True") + self.cantiand_configs["MYSQL_METADATA_IN_CANTIAN"] = str(_value).upper() + _value = json_data.get("mes_ssl_switch", "False") + self.cantiand_configs["MES_SSL_SWITCH"] = str(_value).upper() flags = os.O_RDONLY modes = stat.S_IWUSR | stat.S_IRUSR - with os.fdopen(os.open(JS_CONF_FILE, flags, modes), 'r') as fp: + with os.fdopen(os.open(JS_CONF_FILE, flags, modes), "r") as fp: json_data = json.load(fp) - self.install_path = json_data['R_INSTALL_PATH'].strip() # -R - self.data = json_data['D_DATA_PATH'].strip() # -D + self.install_path = json_data["R_INSTALL_PATH"].strip() # -R + self.data = json_data["D_DATA_PATH"].strip() # -D self.option = self.INS_ALL for tmp_num in range(100): - tmp_word = 'Z_KERNEL_PARAMETER' + str(tmp_num) - if json_data.get(tmp_word, '') != '': - _value = json_data.get(tmp_word, '').strip().split('=') + tmp_word = "Z_KERNEL_PARAMETER" + str(tmp_num) + if json_data.get(tmp_word, "") != "": + _value = json_data.get(tmp_word, "").strip().split("=") self.cantiand_configs[_value[0].strip().upper()] = _value[1].strip() if json_data["CT_CLUSTER_STRICT_CHECK"] in ["FALSE", "TRUE"]: - self.cantiand_configs["CT_CLUSTER_STRICT_CHECK"] = json_data["CT_CLUSTER_STRICT_CHECK"] + self.cantiand_configs["CT_CLUSTER_STRICT_CHECK"] = json_data[ + "CT_CLUSTER_STRICT_CHECK" + ] if g_opts.password: self.cantiand_configs["_SYS_PASSWORD"] = g_opts.password if g_opts.cert_encrypt_pwd: @@ -1126,9 +1306,8 @@ class Installer: LOGGER.info("Checking runner.") gid = os.getgid() uid = os.getuid() - log("Check runner user id and group id is : %s, %s" - % (str(uid), str(gid))) - if (gid != 0 and uid != 0): + log("Check runner user id and group id is : %s, %s" % (str(uid), str(gid))) + if gid != 0 and uid != 0: err_msg = "Only user with root privilege can run this script" LOGGER.error(err_msg) raise Exception(err_msg) @@ -1155,12 +1334,14 @@ class Installer: chown data and gcc dirs :return: """ - cmd = "chown %s:%s -hR \"%s\";" % (self.user, self.group, self.data) + cmd = 'chown %s:%s -hR "%s";' % (self.user, self.group, self.data) LOGGER.info("Change owner cmd: %s" % cmd) ret_code, _, stderr = _exec_popen(cmd) if ret_code: raise Exception( - "chown to %s:%s return: %s%s%s" % (self.user, self.group, str(ret_code), os.linesep, stderr)) + "chown to %s:%s return: %s%s%s" + % (self.user, self.group, str(ret_code), os.linesep, stderr) + ) ########################################################################### # Is there a database installed by the user? If right, raise error @@ -1179,12 +1360,15 @@ class Installer: str_cmd = "echo ~" ret_code, stdout, stderr = _exec_popen(str_cmd) if ret_code: - err_msg = "Can not get user home. ret_code : %s, stdout : %s, stderr : %s" % (ret_code, stdout, stderr) + err_msg = ( + "Can not get user home. ret_code : %s, stdout : %s, stderr : %s" + % (ret_code, stdout, stderr) + ) LOGGER.error(err_msg) raise Exception(err_msg) # Get the profile of user. output = os.path.realpath(os.path.normpath(stdout)) - if (not check_path(output)): + if not check_path(output): err_msg = "The user home directory is invalid." LOGGER.error(err_msg) raise Exception(err_msg) @@ -1196,7 +1380,7 @@ class Installer: try: flags = os.O_RDONLY modes = stat.S_IWUSR | stat.S_IRUSR - with os.fdopen(os.open(self.user_profile, flags, modes), 'r') as _file: + with os.fdopen(os.open(self.user_profile, flags, modes), "r") as _file: is_find = self.dealwith_gsdb(is_find, _file) except IOError as ex: err_msg = "Can not read user profile: " + str(ex) @@ -1217,15 +1401,18 @@ class Installer: def dealwith_gsdb(self, is_find, _file): while True: str_line = _file.readline() - if (not str_line): + if not str_line: break str_line = str_line.strip() - if (str_line.startswith("#")): + if str_line.startswith("#"): continue user_info = str_line.split() self.dealwith_gsdb_data(user_info, str_line) - if (len(user_info) >= 2 and user_info[0] == "export" - and user_info[1].startswith("CTDB_HOME_BAK=") > 0): + if ( + len(user_info) >= 2 + and user_info[0] == "export" + and user_info[1].startswith("CTDB_HOME_BAK=") > 0 + ): is_find = True break else: @@ -1234,9 +1421,12 @@ class Installer: def dealwith_gsdb_data(self, user_info, str_line): # deal with the CTDB_DATA with """ - if (len(user_info) >= 2 and user_info[0] == "export" - and user_info[1].startswith('CTDB_DATA="') > 0): - self.old_data_path = str_line[str_line.find("=") + 2:-1] + if ( + len(user_info) >= 2 + and user_info[0] == "export" + and user_info[1].startswith('CTDB_DATA="') > 0 + ): + self.old_data_path = str_line[str_line.find("=") + 2 : -1] self.old_data_path = os.path.normpath(self.old_data_path) real_path = os.path.realpath(self.old_data_path) if not check_path(real_path): @@ -1249,12 +1439,15 @@ class Installer: LOGGER.error(err_msg) raise Exception(err_msg) # deal with the CTDB_DATA path without """ - elif (len(user_info) >= 2 and user_info[0] == "export" - and user_info[1].startswith("CTDB_DATA=") > 0): - self.old_data_path = str_line[str_line.find("=") + 1:] + elif ( + len(user_info) >= 2 + and user_info[0] == "export" + and user_info[1].startswith("CTDB_DATA=") > 0 + ): + self.old_data_path = str_line[str_line.find("=") + 1 :] self.old_data_path = os.path.normpath(self.old_data_path) real_path = os.path.realpath(self.old_data_path) - if (not check_path(real_path)): + if not check_path(real_path): err_msg = "The Path specified by CTDB_DATA is invalid." LOGGER.error(err_msg) raise Exception(err_msg) @@ -1288,8 +1481,11 @@ class Installer: # check the user has write permission or not test_file = os.path.join(original_path, "touch.tst") - cmd = ("touch %s && chmod %s %s " - % (test_file, CommonValue.KEY_FILE_MODE, test_file)) + cmd = "touch %s && chmod %s %s " % ( + test_file, + CommonValue.KEY_FILE_MODE, + test_file, + ) status, _, stderr = _exec_popen(cmd) if status != 0: @@ -1345,14 +1541,22 @@ class Installer: # 98: Address already in use # 95: Operation not supported # 13: Permission denied - if (int(error.errno) == 98 or int(error.errno) == 95 - or int(error.errno) == 13): - log("Error: port %s has been used,the detail" - " information is as follows:" % value) + if ( + int(error.errno) == 98 + or int(error.errno) == 95 + or int(error.errno) == 13 + ): + log( + "Error: port %s has been used,the detail" + " information is as follows:" % value + ) str_cmd = "netstat -unltp | grep %s" % value ret_code, stdout, stderr = _exec_popen(str_cmd) if ret_code: - err_msg = "can not get detail information of the port, command: %s. ret_code : %s, stdout : %s, stderr : %s" % (str_cmd, ret_code, stdout, stderr) + err_msg = ( + "can not get detail information of the port, command: %s. ret_code : %s, stdout : %s, stderr : %s" + % (str_cmd, ret_code, stdout, stderr) + ) LOGGER.error(err_msg) raise Exception(err_msg) LOGGER.error(str(stdout)) @@ -1361,7 +1565,9 @@ class Installer: LOGGER.error("check port failed: " + str(ex)) raise Exception("check port failed: " + str(ex)) else: - err_msg = "This install script can not support python version : " + gPyVersion + err_msg = ( + "This install script can not support python version : " + gPyVersion + ) LOGGER.error(err_msg) raise Exception(err_msg) @@ -1401,7 +1607,7 @@ class Installer: output: NA """ LOGGER.info("check the node IP address.") - if get_value("cantian_in_container") == '0': + if get_value("cantian_in_container") == "0": try: socket.inet_aton(node_ip) self.ipv_type = "ipv4" @@ -1410,7 +1616,10 @@ class Installer: socket.inet_pton(socket.AF_INET6, node_ip) self.ipv_type = "ipv6" except socket.error: - err_msg = "The invalid IP address : %s is not ipv4 or ipv6 format." % node_ip + err_msg = ( + "The invalid IP address : %s is not ipv4 or ipv6 format." + % node_ip + ) LOGGER.error(err_msg) raise Exception(err_msg) @@ -1423,8 +1632,11 @@ class Installer: cmd = "%s %s -i 1 -c 3 | grep ttl | wc -l" % (ping_cmd, node_ip) ret_code, stdout, stderr = _exec_popen(cmd) - if ret_code or stdout != '3': - err_msg = "The invalid IP address is %s. ret_code : %s, stdout : %s, stderr : %s" % (node_ip, ret_code, stdout, stderr) + if ret_code or stdout != "3": + err_msg = ( + "The invalid IP address is %s. ret_code : %s, stdout : %s, stderr : %s" + % (node_ip, ret_code, stdout, stderr) + ) LOGGER.error(err_msg) raise Exception(err_msg) @@ -1439,7 +1651,10 @@ class Installer: ip_is_found = 0 if ret_code or not int(ip_is_found): - err_msg = "The invalid IP address is %s. ret_code : %s, ip_is_found : %s, stderr : %s" % (node_ip, ret_code, ip_is_found, stderr) + err_msg = ( + "The invalid IP address is %s. ret_code : %s, ip_is_found : %s, stderr : %s" + % (node_ip, ret_code, ip_is_found, stderr) + ) LOGGER.error(err_msg) raise Exception(err_msg) @@ -1450,16 +1665,18 @@ class Installer: # not meet the requirements of the database。 ######################################################################### def set_numa_config(self): - if not os.path.exists('/usr/bin/lscpu'): + if not os.path.exists("/usr/bin/lscpu"): LOGGER.info("Warning: lscpu path get error") return - ret_code, result, stderr = _exec_popen('/usr/bin/lscpu | grep -i "NUMA node(s)"') + ret_code, result, stderr = _exec_popen( + '/usr/bin/lscpu | grep -i "NUMA node(s)"' + ) if ret_code: err_msg = "can not get numa node parameters, err: %s" % stderr LOGGER.error(err_msg) raise Exception(err_msg) - _result = result.strip().split(':') + _result = result.strip().split(":") if len(_result) != 2: LOGGER.info("Warning: numa get error, result:%s" % result) @@ -1474,14 +1691,16 @@ class Installer: LOGGER.info("Warning: numa(s) size get error, result:%s" % result) return while numa_num < int(_result[1].strip()): - err_code, ans, err_msg = _exec_popen('/usr/bin/lscpu | grep -i "NUMA node%s"' % numa_num) - _ans = ans.strip().split(':') + err_code, ans, err_msg = _exec_popen( + '/usr/bin/lscpu | grep -i "NUMA node%s"' % numa_num + ) + _ans = ans.strip().split(":") if len(_ans) != 2: LOGGER.info("Warning: numa node get error, ans:%s" % ans) return numa_str = _ans[1].strip() - if platform.machine() == 'aarch64' and numa_num == 0: - numa_id_str = _ans[1].strip().split('-') + if platform.machine() == "aarch64" and numa_num == 0: + numa_id_str = _ans[1].strip().split("-") last_numa_id = numa_id_str[-1] if int(last_numa_id) >= 16 and g_opts.use_dbstor: numa_str = "0-1,6-11,16-" + str(last_numa_id) @@ -1490,7 +1709,9 @@ class Installer: if not numa_info.isspace(): shm_mysql_cpu_group_info = (numa_info[:-1] + ";") * numa_cpu_num - self.cantiand_configs["SHM_MYSQL_CPU_GROUP_INFO"] = shm_mysql_cpu_group_info[:-1] + self.cantiand_configs["SHM_MYSQL_CPU_GROUP_INFO"] = ( + shm_mysql_cpu_group_info[:-1] + ) self.cantiand_configs["SHM_CPU_GROUP_INFO"] = numa_info def check_config_options(self): @@ -1509,8 +1730,9 @@ class Installer: shared_pool_size = 128 * tmp_mb data_buffer_size = 128 * tmp_mb temp_buffer_size = 32 * tmp_mb - sga_buff_size = (log_buffer_size + shared_pool_size + data_buffer_size - + temp_buffer_size) + sga_buff_size = ( + log_buffer_size + shared_pool_size + data_buffer_size + temp_buffer_size + ) self.set_numa_config() # parse the value of kernel parameters @@ -1522,9 +1744,18 @@ class Installer: check_kernel_parameter(key) check_invalid_symbol(value) # Unit conversion - get_unit_conversion_info = UnitConversionInfo(tmp_gb, tmp_mb, tmp_kb, key, value, - sga_buff_size, temp_buffer_size, data_buffer_size, - shared_pool_size, log_buffer_size) + get_unit_conversion_info = UnitConversionInfo( + tmp_gb, + tmp_mb, + tmp_kb, + key, + value, + sga_buff_size, + temp_buffer_size, + data_buffer_size, + shared_pool_size, + log_buffer_size, + ) sga_buff_size = self.do_unit_conversion(get_unit_conversion_info) except ValueError as ex: LOGGER.error("check kernel parameter failed: " + str(ex)) @@ -1536,7 +1767,9 @@ class Installer: # Check the ip address for item in _list: if len(_list) != 1 and all_zero_addr_after_ping(item): - err_msg = "lsnr_addr contains all-zero ip, can not specify other ip." + err_msg = ( + "lsnr_addr contains all-zero ip, can not specify other ip." + ) LOGGER.error(err_msg) raise Exception(err_msg) self.check_ip_is_vaild(item) @@ -1559,15 +1792,22 @@ class Installer: self.check_port(self.lsnr_port) self.lsnr_port = int(self.lsnr_port) # check sga_buff_size - cmd = ("cat /proc/meminfo |grep -wE 'MemFree:|Buffers:|Cached:" - "|SwapCached' |awk '{sum += $2};END {print sum}'") + cmd = ( + "cat /proc/meminfo |grep -wE 'MemFree:|Buffers:|Cached:" + "|SwapCached' |awk '{sum += $2};END {print sum}'" + ) ret_code, cur_avi_memory, stderr = _exec_popen(cmd) if ret_code: - err_msg = "can not get shmmax parameters, command: %s, err: %s" % (cmd, stderr) + err_msg = "can not get shmmax parameters, command: %s, err: %s" % ( + cmd, + stderr, + ) LOGGER.error(err_msg) raise Exception(err_msg) if sga_buff_size < 114 * tmp_mb: - err_msg = "sga_buff_size should bigger than or equal to 114*MB, please check it!" + err_msg = ( + "sga_buff_size should bigger than or equal to 114*MB, please check it!" + ) LOGGER.error(err_msg) raise Exception(err_msg) try: @@ -1604,11 +1844,17 @@ class Installer: if mode == "decrypted": ret_pwd = kmc_adapter.decrypt(plain_text) except Exception as error: - raise Exception("Failed to %s password of user [sys]. Error: %s" % (mode, str(error))) from error + raise Exception( + "Failed to %s password of user [sys]. Error: %s" % (mode, str(error)) + ) from error - split_env = os.environ['LD_LIBRARY_PATH'].split(":") - filtered_env = [single_env for single_env in split_env if "/opt/cantian/dbstor/lib" not in single_env] - os.environ['LD_LIBRARY_PATH'] = ":".join(filtered_env) + split_env = os.environ["LD_LIBRARY_PATH"].split(":") + filtered_env = [ + single_env + for single_env in split_env + if "/opt/cantian/dbstor/lib" not in single_env + ] + os.environ["LD_LIBRARY_PATH"] = ":".join(filtered_env) return ret_pwd @@ -1616,21 +1862,27 @@ class Installer: def decrypt_db_passwd(): file_list = glob.glob(CTSQL_INI_FILE) ctsql_ini_data = file_reader(file_list[0]) - encrypt_pwd = ctsql_ini_data[ctsql_ini_data.find('=') + 1:].strip() + encrypt_pwd = ctsql_ini_data[ctsql_ini_data.find("=") + 1 :].strip() mode = "decrypted" g_opts.db_passwd = Installer.kmc_resovle_password(mode, encrypt_pwd) @staticmethod def set_cms_ini(passwd): cms_conf = "/opt/cantian/cms/cfg/cms.ini" - str_cmd = f"sed -i '/_CMS_MES_SSL_KEY_PWD = None/d' {cms_conf}" \ - f"&& echo '_CMS_MES_SSL_KEY_PWD = {passwd}' >> {cms_conf}" + str_cmd = ( + f"sed -i '/_CMS_MES_SSL_KEY_PWD = None/d' {cms_conf}" + f"&& echo '_CMS_MES_SSL_KEY_PWD = {passwd}' >> {cms_conf}" + ) LOGGER.info("Copy config files cmd: " + str_cmd) ret_code, _, stderr = _exec_popen(str_cmd) if ret_code: - LOGGER.error("update cms.ini failed: " + str(ret_code) + os.linesep + stderr) - raise Exception("update cms.ini failed: " + str(ret_code) + os.linesep + stderr) - + LOGGER.error( + "update cms.ini failed: " + str(ret_code) + os.linesep + stderr + ) + raise Exception( + "update cms.ini failed: " + str(ret_code) + os.linesep + stderr + ) + @staticmethod def set_mes_passwd(passwd): file_path = "/opt/cantian/common/config/certificates/mes.pass" @@ -1648,29 +1900,37 @@ class Installer: elif key == "LSNR_PORT": self.lsnr_port = value elif key == "ENABLE_SYSDBA_LOGIN": - self.enable_sysdba_login = Installer.check_pare_bool_value( - key, value) + self.enable_sysdba_login = Installer.check_pare_bool_value(key, value) elif key == "REPL_AUTH": - self.repl_auth = Installer.check_pare_bool_value( - key, value) + self.repl_auth = Installer.check_pare_bool_value(key, value) elif key == "REPL_SCRAM_AUTH": - self.repl_scram_auth = Installer.check_pare_bool_value( - key, value) + self.repl_scram_auth = Installer.check_pare_bool_value(key, value) elif key == "ENABLE_ACCESS_DC": - self.enable_access_dc = Installer.check_pare_bool_value( - key, value) + self.enable_access_dc = Installer.check_pare_bool_value(key, value) elif key == "REPLACE_VERIFY_PASSWORD": - self.replace_password_verify = Installer.check_pare_bool_value( - key, value) + self.replace_password_verify = Installer.check_pare_bool_value(key, value) else: return def do_unit_conversion(self, get_unit_conversion_info): - tmp_gb, tmp_mb, tmp_kb, key, value, \ - sga_buff_size, temp_buffer_size, data_buffer_size, \ - shared_pool_size, log_buffer_size = get_unit_conversion_info - if key in ["TEMP_BUFFER_SIZE", "DATA_BUFFER_SIZE", - "SHARED_POOL_SIZE", "LOG_BUFFER_SIZE"]: + ( + tmp_gb, + tmp_mb, + tmp_kb, + key, + value, + sga_buff_size, + temp_buffer_size, + data_buffer_size, + shared_pool_size, + log_buffer_size, + ) = get_unit_conversion_info + if key in [ + "TEMP_BUFFER_SIZE", + "DATA_BUFFER_SIZE", + "SHARED_POOL_SIZE", + "LOG_BUFFER_SIZE", + ]: if value[0:-1].isdigit() and value[-1:] in ["G", "M", "K"]: unit_map = { "G": tmp_gb, @@ -1699,49 +1959,75 @@ class Installer: output: NA """ # change install path privilege to 700 - str_cmd = "chmod %s %s -R" % (CommonValue.KEY_DIRECTORY_MODE, - self.install_path) + str_cmd = "chmod %s %s -R" % (CommonValue.KEY_DIRECTORY_MODE, self.install_path) # chmod add-ons/ file 500 - str_cmd += ("&& find '%s'/add-ons -type f | xargs chmod %s " - % (self.install_path, CommonValue.MID_FILE_MODE)) + str_cmd += "&& find '%s'/add-ons -type f | xargs chmod %s " % ( + self.install_path, + CommonValue.MID_FILE_MODE, + ) # chmod admin/ file 600 - str_cmd += ("&& find '%s'/admin -type f | xargs chmod %s " - % (self.install_path, CommonValue.KEY_FILE_MODE)) + str_cmd += "&& find '%s'/admin -type f | xargs chmod %s " % ( + self.install_path, + CommonValue.KEY_FILE_MODE, + ) # chmod admin/scripts/fetch_cls_stat.py file 550 - str_cmd += ("&& find '%s'/admin -type f | grep fetch_cls_stat.py | xargs chmod %s " - % (self.install_path, CommonValue.MAX_FILE_MODE)) + str_cmd += ( + "&& find '%s'/admin -type f | grep fetch_cls_stat.py | xargs chmod %s " + % (self.install_path, CommonValue.MAX_FILE_MODE) + ) # chmod lib/ file 500 - str_cmd += ("&& find '%s'/lib -type f | xargs chmod %s" - % (self.install_path, CommonValue.MID_FILE_MODE)) + str_cmd += "&& find '%s'/lib -type f | xargs chmod %s" % ( + self.install_path, + CommonValue.MID_FILE_MODE, + ) # chmod bin/ file 500 - str_cmd += ("&& find '%s'/bin -type f | xargs chmod %s " - % (self.install_path, CommonValue.MID_FILE_MODE)) - str_cmd += ("&& find '%s'/cfg -type f | xargs chmod %s " - % (self.install_path, CommonValue.KEY_FILE_MODE)) + str_cmd += "&& find '%s'/bin -type f | xargs chmod %s " % ( + self.install_path, + CommonValue.MID_FILE_MODE, + ) + str_cmd += "&& find '%s'/cfg -type f | xargs chmod %s " % ( + self.install_path, + CommonValue.KEY_FILE_MODE, + ) package_xml = os.path.join(self.install_path, "package.xml") if os.path.exists(package_xml): - str_cmd += ("&& chmod %s '%s'/package.xml" - % (CommonValue.MIN_FILE_MODE, self.install_path)) + str_cmd += "&& chmod %s '%s'/package.xml" % ( + CommonValue.MIN_FILE_MODE, + self.install_path, + ) LOGGER.info("Change app permission cmd: %s" % str_cmd) ret_code, _, stderr = _exec_popen(str_cmd) if ret_code: self.failed_pos = self.DECOMPRESS_BIN_FAILED - err_msg = "chmod %s return: " % CommonValue.KEY_DIRECTORY_MODE + str(ret_code) + os.linesep + stderr + err_msg = ( + "chmod %s return: " % CommonValue.KEY_DIRECTORY_MODE + + str(ret_code) + + os.linesep + + stderr + ) LOGGER.error(err_msg) raise Exception(err_msg) # 管控面使用 - str_cmd = "chmod %s %s " % (CommonValue.MAX_DIRECTORY_MODE, - self.install_path) - str_cmd += "&& chmod %s %s " % (CommonValue.MAX_DIRECTORY_MODE, - os.path.join(self.install_path, "admin")) - str_cmd += "&& chmod %s %s" % (CommonValue.MAX_DIRECTORY_MODE, - os.path.join(self.install_path, "admin", "scripts")) + str_cmd = "chmod %s %s " % (CommonValue.MAX_DIRECTORY_MODE, self.install_path) + str_cmd += "&& chmod %s %s " % ( + CommonValue.MAX_DIRECTORY_MODE, + os.path.join(self.install_path, "admin"), + ) + str_cmd += "&& chmod %s %s" % ( + CommonValue.MAX_DIRECTORY_MODE, + os.path.join(self.install_path, "admin", "scripts"), + ) LOGGER.info("Change app server/admin/scripts dir for om. cmd: %s" % str_cmd) ret_code, _, stderr = _exec_popen(str_cmd) if ret_code: self.failed_pos = self.DECOMPRESS_BIN_FAILED - err_msg = "chmod %s return: " % CommonValue.KEY_DIRECTORY_MODE + str(ret_code) + os.linesep + stderr + err_msg = ( + "chmod %s return: " % CommonValue.KEY_DIRECTORY_MODE + + str(ret_code) + + os.linesep + + stderr + ) LOGGER.error(err_msg) raise Exception(err_msg) @@ -1752,15 +2038,15 @@ class Installer: """ # eg 'length in [8-64]' if len(passwd) < shortest_len or len(passwd) > 64: - raise Exception("The length of password must be %s to 64." - % shortest_len) + raise Exception("The length of password must be %s to 64." % shortest_len) # Can't save with user name if passwd == self.user: raise Exception("Error: Password can't be the same as username.") elif passwd == self.user[::-1]: - raise Exception("Error: Password cannot be the same as username " - "in reverse order") + raise Exception( + "Error: Password cannot be the same as username " "in reverse order" + ) upper_cases = set("ABCDEFGHIJKLMNOPQRSTUVWXYZ") lower_cases = set("abcdefghijklmnopqrstuvwxyz") @@ -1774,16 +2060,19 @@ class Installer: if passwd_set & cases: types += 1 if types < 3: - raise Exception("Error: Password must contains at least three" - " different types of characters.") + raise Exception( + "Error: Password must contains at least three" + " different types of characters." + ) # Only can contains enumerated cases all_cases = upper_cases | lower_cases | digits | special_cases un_cases = passwd_set - all_cases if un_cases: - raise Exception("Error: There are characters that are not" - " allowed in the password: '%s'" - % "".join(un_cases)) + raise Exception( + "Error: There are characters that are not" + " allowed in the password: '%s'" % "".join(un_cases) + ) def get_new_passwd(self, pw_prompt, user_prompt, shortest_len): """Get new passwd""" @@ -1799,7 +2088,9 @@ class Installer: continue print_str_3 = CommonPrint() - print_str_3.common_log("Please enter %s of %s again: " % (pw_prompt, user_prompt)) + print_str_3.common_log( + "Please enter %s of %s again: " % (pw_prompt, user_prompt) + ) new_passwd2 = getpass.getpass() if new_passwd == new_passwd2: @@ -1821,9 +2112,11 @@ class Installer: # 0. "_SYS_PASSWORD" can't be set when ENABLE_SYSDBA_LOGIN is False sys_password = self.cantiand_configs["_SYS_PASSWORD"] if not self.enable_sysdba_login and len(sys_password) != 0: - raise Exception("Can't use _SYS_PASSWORD to set the password of " - "user [SYS] in the installation script when " - "ENABLE_SYSDBA_LOGIN is False.") + raise Exception( + "Can't use _SYS_PASSWORD to set the password of " + "user [SYS] in the installation script when " + "ENABLE_SYSDBA_LOGIN is False." + ) # 1. Get passed from parameter -C # Set passwd of SYS in cantiand.ini by parameter -C @@ -1834,9 +2127,8 @@ class Installer: if sys.stdin.isatty(): # If not pipe content, get passwd by interactive input g_opts.db_passwd = self.get_new_passwd( - pw_prompt="database password", - user_prompt="user [SYS]", - shortest_len=8) + pw_prompt="database password", user_prompt="user [SYS]", shortest_len=8 + ) else: try: # Get passwd from pipe @@ -1848,25 +2140,45 @@ class Installer: def install_xnet_lib(self): if is_rdma_startup(): - str_cmd = "cp -rf %s/add-ons/mlnx/lib* %s/add-ons/" % (self.install_path, self.install_path) + str_cmd = "cp -rf %s/add-ons/mlnx/lib* %s/add-ons/" % ( + self.install_path, + self.install_path, + ) elif is_rdma_1823_startup(): - str_cmd = "cp -rf %s/add-ons/1823/lib* %s/add-ons/" % (self.install_path, self.install_path) + str_cmd = "cp -rf %s/add-ons/1823/lib* %s/add-ons/" % ( + self.install_path, + self.install_path, + ) else: - str_cmd = "cp -rf %s/add-ons/nomlnx/lib* %s/add-ons/" % (self.install_path, self.install_path) + str_cmd = "cp -rf %s/add-ons/nomlnx/lib* %s/add-ons/" % ( + self.install_path, + self.install_path, + ) LOGGER.info("Install xnet lib cmd: " + str_cmd) ret_code, _, stderr = _exec_popen(str_cmd) if ret_code: - LOGGER.error("Install xnet lib return: " + str(ret_code) + os.linesep + stderr) - raise Exception("Install xnet lib return: " + str(ret_code) + os.linesep + stderr) + LOGGER.error( + "Install xnet lib return: " + str(ret_code) + os.linesep + stderr + ) + raise Exception( + "Install xnet lib return: " + str(ret_code) + os.linesep + stderr + ) def install_kmc_lib(self): - str_cmd = "cp -rf %s/add-ons/kmc_shared/lib* %s/add-ons/" % (self.install_path, self.install_path) + str_cmd = "cp -rf %s/add-ons/kmc_shared/lib* %s/add-ons/" % ( + self.install_path, + self.install_path, + ) LOGGER.info("install kmc lib cmd:" + str_cmd) ret_code, _, stderr = _exec_popen(str_cmd) if ret_code: - LOGGER.error("Install kmc lib return: " + str(ret_code) + os.linesep + stderr) - raise Exception("Install kmc lib return: " + str(ret_code) + os.linesep + stderr) + LOGGER.error( + "Install kmc lib return: " + str(ret_code) + os.linesep + stderr + ) + raise Exception( + "Install kmc lib return: " + str(ret_code) + os.linesep + stderr + ) ######################################################################### # Unzip the installation files to the installation directory. @@ -1876,36 +2188,72 @@ class Installer: Unzip the installation files to the installation directory. :return: NA """ - self.run_file = "/opt/cantian/image/cantian_connector/CantianKernel/Cantian-DATABASE-CENTOS-64bit/" \ - "Cantian-RUN-CENTOS-64bit.tar.gz" + self.run_file = ( + "/opt/cantian/image/cantian_connector/CantianKernel/Cantian-DATABASE-CENTOS-64bit/" + "Cantian-RUN-CENTOS-64bit.tar.gz" + ) self.run_pkg_name = self.get_decompress_tarname(self.run_file) LOGGER.info("Decompressing run file.") if g_opts.use_dbstor: - os.makedirs("%s/dbstor/conf/dbs" % self.data, CommonValue.KEY_DIRECTORY_PERMISSION) - os.makedirs("%s/dbstor/conf/infra/config" % self.data, CommonValue.KEY_DIRECTORY_PERMISSION) - os.makedirs("%s/dbstor/data/logs" % self.data, CommonValue.KEY_DIRECTORY_PERMISSION) - os.makedirs("%s/dbstor/data/ftds" % self.data, CommonValue.KEY_DIRECTORY_PERMISSION) + os.makedirs( + "%s/dbstor/conf/dbs" % self.data, CommonValue.KEY_DIRECTORY_PERMISSION + ) + os.makedirs( + "%s/dbstor/conf/infra/config" % self.data, + CommonValue.KEY_DIRECTORY_PERMISSION, + ) + os.makedirs( + "%s/dbstor/data/logs" % self.data, CommonValue.KEY_DIRECTORY_PERMISSION + ) + os.makedirs( + "%s/dbstor/data/ftds" % self.data, CommonValue.KEY_DIRECTORY_PERMISSION + ) if is_rdma_startup() or is_rdma_1823_startup(): - str_cmd = "cp %s/cfg/node_config_rdma.xml %s/dbstor/conf/infra/config/node_config.xml" % ( - self.install_path, self.data) + str_cmd = ( + "cp %s/cfg/node_config_rdma.xml %s/dbstor/conf/infra/config/node_config.xml" + % (self.install_path, self.data) + ) else: - str_cmd = "cp %s/cfg/node_config_tcp.xml %s/dbstor/conf/infra/config/node_config.xml" % ( - self.install_path, self.data) - - str_cmd += " && cp %s/cfg/osd.cfg %s/dbstor/conf/infra/config/osd.cfg" % (self.install_path, self.data) - str_cmd += " && cp /opt/cantian/dbstor/tools/dbstor_config.ini %s/dbstor/conf/dbs/" % (self.data) - str_cmd += " && echo 'DBSTOR_OWNER_NAME = cantian' >> %s/dbstor/conf/dbs/dbstor_config.ini" % (self.data) - str_cmd += " && sed -i '/^\s*$/d' %s/dbstor/conf/dbs/dbstor_config.ini" % (self.data) - str_cmd += " && chown -R %s:%s %s/dbstor" % (self.user, self.group, self.data) - str_cmd += " && chmod 640 %s/dbstor/conf/dbs/dbstor_config.ini" % (self.data) + str_cmd = ( + "cp %s/cfg/node_config_tcp.xml %s/dbstor/conf/infra/config/node_config.xml" + % (self.install_path, self.data) + ) + + str_cmd += " && cp %s/cfg/osd.cfg %s/dbstor/conf/infra/config/osd.cfg" % ( + self.install_path, + self.data, + ) + str_cmd += ( + " && cp /opt/cantian/dbstor/tools/dbstor_config.ini %s/dbstor/conf/dbs/" + % (self.data) + ) + str_cmd += ( + " && echo 'DBSTOR_OWNER_NAME = cantian' >> %s/dbstor/conf/dbs/dbstor_config.ini" + % (self.data) + ) + str_cmd += " && sed -i '/^\s*$/d' %s/dbstor/conf/dbs/dbstor_config.ini" % ( + self.data + ) + str_cmd += " && chown -R %s:%s %s/dbstor" % ( + self.user, + self.group, + self.data, + ) + str_cmd += " && chmod 640 %s/dbstor/conf/dbs/dbstor_config.ini" % ( + self.data + ) LOGGER.info("Copy config files cmd: " + str_cmd) ret_code, _, stderr = _exec_popen(str_cmd) if ret_code: self.failed_pos = self.DECOMPRESS_BIN_FAILED - LOGGER.error("Decompress bin return: " + str(ret_code) + os.linesep + stderr) - raise Exception("Decompress bin return: " + str(ret_code) + os.linesep + stderr) + LOGGER.error( + "Decompress bin return: " + str(ret_code) + os.linesep + stderr + ) + raise Exception( + "Decompress bin return: " + str(ret_code) + os.linesep + stderr + ) if not g_opts.cantian_in_container: cantian_check_share_logic_ip_isvalid("share", g_opts.share_logic_ip) @@ -1917,14 +2265,19 @@ class Installer: self.change_app_permission() # change owner to user:group - str_cmd = "chown %s:%s -hR %s " % (self.user, self.group, - self.install_path) + str_cmd = "chown %s:%s -hR %s " % (self.user, self.group, self.install_path) # Change the owner LOGGER.info("Change owner cmd: %s" % str_cmd) ret_code, _, stderr = _exec_popen(str_cmd) if ret_code: self.failed_pos = self.DECOMPRESS_BIN_FAILED - err_msg = "chown to %s: %s return: %s%s%s" % (self.user, self.group, str(ret_code), os.linesep, stderr) + err_msg = "chown to %s: %s return: %s%s%s" % ( + self.user, + self.group, + str(ret_code), + os.linesep, + stderr, + ) LOGGER.error(err_msg) raise Exception(err_msg) @@ -1934,30 +2287,40 @@ class Installer: try: flags = os.O_RDWR modes = stat.S_IWUSR | stat.S_IRUSR - with os.fdopen(os.open(self.user_profile, flags, modes), 'a') as _file: - _file.write("export CTDB_HOME=\"%s\"" % self.install_path) + with os.fdopen(os.open(self.user_profile, flags, modes), "a") as _file: + _file.write('export CTDB_HOME="%s"' % self.install_path) _file.write(os.linesep) - _file.write("export PATH=\"%s\":$PATH" - % os.path.join(MYSQL_BIN_DIR, "bin")) + _file.write( + 'export PATH="%s":$PATH' % os.path.join(MYSQL_BIN_DIR, "bin") + ) _file.write(os.linesep) - _file.write("export PATH=\"%s\":$PATH" - % os.path.join(self.install_path, "bin")) + _file.write( + 'export PATH="%s":$PATH' % os.path.join(self.install_path, "bin") + ) _file.write(os.linesep) if "LD_LIBRARY_PATH" in os.environ: - _file.write("export LD_LIBRARY_PATH=\"%s\":\"%s\":\"%s\"" - ":$LD_LIBRARY_PATH" - % (os.path.join(self.install_path, "lib"), - os.path.join(self.install_path, "add-ons"), - os.path.join(MYSQL_BIN_DIR, "lib/plugin"))) + _file.write( + 'export LD_LIBRARY_PATH="%s":"%s":"%s"' + ":$LD_LIBRARY_PATH" + % ( + os.path.join(self.install_path, "lib"), + os.path.join(self.install_path, "add-ons"), + os.path.join(MYSQL_BIN_DIR, "lib/plugin"), + ) + ) else: - _file.write("export LD_LIBRARY_PATH=\"%s\":\"%s\":\"%s\"" - % (os.path.join(self.install_path, "lib"), - os.path.join(self.install_path, "add-ons"), - os.path.join(MYSQL_BIN_DIR, "lib/plugin"))) + _file.write( + 'export LD_LIBRARY_PATH="%s":"%s":"%s"' + % ( + os.path.join(self.install_path, "lib"), + os.path.join(self.install_path, "add-ons"), + os.path.join(MYSQL_BIN_DIR, "lib/plugin"), + ) + ) _file.write(os.linesep) if self.old_data_path == "": # set CTDB_DATA - _file.write("export CTDB_DATA=\"%s\"" % self.data) + _file.write('export CTDB_DATA="%s"' % self.data) _file.write(os.linesep) _file.flush() except IOError as ex: @@ -1983,19 +2346,25 @@ class Installer: # Avoid create database failed by the value of CTSQL_SSL_KEY_PASSWD self.clean_ssl_env() - os.environ['PATH'] = (os.path.join(self.install_path, "bin") - + ":" + os.environ['PATH']) + os.environ["PATH"] = ( + os.path.join(self.install_path, "bin") + ":" + os.environ["PATH"] + ) # in some system LD_LIBRARY_PATH is not set, # so must check it, or excetion will be raise - if 'LD_LIBRARY_PATH' in os.environ: - os.environ['LD_LIBRARY_PATH'] = ("%s:%s:%s" % ( - os.path.join(self.install_path, "lib"), os.path.join( - self.install_path, "add-ons", ), - os.environ['LD_LIBRARY_PATH'])) + if "LD_LIBRARY_PATH" in os.environ: + os.environ["LD_LIBRARY_PATH"] = "%s:%s:%s" % ( + os.path.join(self.install_path, "lib"), + os.path.join( + self.install_path, + "add-ons", + ), + os.environ["LD_LIBRARY_PATH"], + ) else: - os.environ['LD_LIBRARY_PATH'] = ("%s:%s" % ( + os.environ["LD_LIBRARY_PATH"] = "%s:%s" % ( os.path.join(self.install_path, "lib"), - os.path.join(self.install_path, "add-ons"),)) + os.path.join(self.install_path, "add-ons"), + ) os.environ["CTDB_HOME"] = self.install_path os.environ["CTDB_DATA"] = self.data os.environ["CANTIANLOG"] = self.cantiand_configs["LOG_HOME"] @@ -2025,8 +2394,10 @@ class Installer: cmd = cmd.strip(";") ret_code, _, stderr = _exec_popen(cmd) if ret_code: - raise Exception("Can not write the %s, command: %s," - " output: %s" % (conf_file, cmd, stderr)) + raise Exception( + "Can not write the %s, command: %s," + " output: %s" % (conf_file, cmd, stderr) + ) def clean_old_conf(self, param_list, conf_file): """ @@ -2037,25 +2408,33 @@ class Installer: cmd = "" # make the command of delete the parameter for parameter in param_list: - cmd += "sed -i '/^%s/d' %s;" % (parameter.replace('[', '\[').replace(']', '\]'), conf_file) + cmd += "sed -i '/^%s/d' %s;" % ( + parameter.replace("[", "\[").replace("]", "\]"), + conf_file, + ) if cmd: cmd = cmd.strip(";") ret_code, _, stderr = _exec_popen(cmd) if ret_code: - raise Exception("Can not write the %s, command: %s," - " output: %s" - % (conf_file, cmd, stderr)) + raise Exception( + "Can not write the %s, command: %s," + " output: %s" % (conf_file, cmd, stderr) + ) def generate_nomount_passwd(self, plain_passwd=""): cmd = "source ~/.bashrc && %s/bin/ctencrypt -e PBKDF2" % self.install_path - g_opts.db_passwd = g_opts.db_passwd if len(plain_passwd.strip()) == 0 else plain_passwd.strip() + g_opts.db_passwd = ( + g_opts.db_passwd if len(plain_passwd.strip()) == 0 else plain_passwd.strip() + ) values = [g_opts.db_passwd, g_opts.db_passwd] ret_code, stdout, stderr = _exec_popen(cmd, values) if ret_code: - raise OSError("Failed to encrypt password of user [sys]." - " Error: %s" % (stderr + os.linesep + stderr)) + raise OSError( + "Failed to encrypt password of user [sys]." + " Error: %s" % (stderr + os.linesep + stderr) + ) # Example of output: # Please enter password to encrypt: @@ -2080,18 +2459,26 @@ class Installer: cmd = "echo >> %s" % conf_file ret_code, _, stderr = _exec_popen(cmd) if ret_code: - raise Exception("Can not write the %s, command: %s," - " output: %s" % (file, cmd, stderr)) + raise Exception( + "Can not write the %s, command: %s," + " output: %s" % (file, cmd, stderr) + ) # Generate new kernel parameters common_parameters = copy.deepcopy(config) # Set password of NOMOUNT mode before create database. if encrypt_passwd and not g_opts.cantian_in_container: mode = "encrypted" - self.ctsql_conf["SYS_PASSWORD"] = Installer.kmc_resovle_password(mode, common_parameters["_SYS_PASSWORD"]) - common_parameters["_SYS_PASSWORD"] = self.generate_nomount_passwd(common_parameters["_SYS_PASSWORD"]) + self.ctsql_conf["SYS_PASSWORD"] = Installer.kmc_resovle_password( + mode, common_parameters["_SYS_PASSWORD"] + ) + common_parameters["_SYS_PASSWORD"] = self.generate_nomount_passwd( + common_parameters["_SYS_PASSWORD"] + ) if g_opts.mes_ssl_switch == True: - common_parameters["MES_SSL_KEY_PWD"] = Installer.kmc_resovle_password(mode, g_opts.cert_encrypt_pwd) + common_parameters["MES_SSL_KEY_PWD"] = Installer.kmc_resovle_password( + mode, g_opts.cert_encrypt_pwd + ) self.set_cms_ini(common_parameters["MES_SSL_KEY_PWD"]) self.set_mes_passwd(common_parameters["MES_SSL_KEY_PWD"]) g_opts.password = common_parameters["_SYS_PASSWORD"] @@ -2118,8 +2505,10 @@ class Installer: cmd = cmd.strip(";") ret_code, _, stderr = _exec_popen(cmd) if ret_code: - raise Exception("Can not write the %s, command: %s," - " output: %s" % (conf_file, cmd, stderr)) + raise Exception( + "Can not write the %s, command: %s," + " output: %s" % (conf_file, cmd, stderr) + ) def set_cluster_conf(self): """ @@ -2131,9 +2520,16 @@ class Installer: cmd = "echo >> %s" % conf_file ret_code, _, stderr = _exec_popen(cmd) if ret_code: - raise Exception("Can not write the %s, command: %s, output: %s" % (conf_file, cmd, stderr)) + raise Exception( + "Can not write the %s, command: %s, output: %s" + % (conf_file, cmd, stderr) + ) size = CLUSTER_SIZE - if g_opts.running_mode in [CANTIAND, CANTIAND_WITH_MYSQL, CANTIAND_WITH_MYSQL_ST]: + if g_opts.running_mode in [ + CANTIAND, + CANTIAND_WITH_MYSQL, + CANTIAND_WITH_MYSQL_ST, + ]: size = 1 if g_opts.node_id == 0 or g_opts.node_id == 1: node_ip = self.cantiand_configs["INTERCONNECT_ADDR"].split(",") @@ -2153,7 +2549,7 @@ class Installer: common_parameters = { "REPORT_FILE": g_opts.log_file, "STATUS_LOG": os.path.join(self.data, "log", "cantianstatus.log"), - "LD_LIBRARY_PATH": os.environ['LD_LIBRARY_PATH'], + "LD_LIBRARY_PATH": os.environ["LD_LIBRARY_PATH"], "USER_HOME": self.user_home_path, "USE_GSS": g_opts.use_gss, "USE_DBSTOR": g_opts.use_dbstor, @@ -2179,14 +2575,16 @@ class Installer: if g_opts.use_dbstor: common_parameters["CONTROL_FILES"] = self.cantiand_configs["CONTROL_FILES"] - common_parameters["SHARED_PATH"] = self.cantiand_configs["SHARED_PATH"], - common_parameters["DBSTOR_NAMESPACE"] = self.cantiand_configs["DBSTOR_NAMESPACE"] + common_parameters["SHARED_PATH"] = (self.cantiand_configs["SHARED_PATH"],) + common_parameters["DBSTOR_NAMESPACE"] = self.cantiand_configs[ + "DBSTOR_NAMESPACE" + ] common_parameters["ENABLE_DBSTOR"] = self.cantiand_configs["ENABLE_DBSTOR"] cantian_config_data = common_parameters flags = os.O_WRONLY | os.O_CREAT | os.O_TRUNC modes = stat.S_IWUSR | stat.S_IRUSR - with os.fdopen(os.open(CANTIAN_CONF_FILE, flags, modes), 'w') as fp: + with os.fdopen(os.open(CANTIAN_CONF_FILE, flags, modes), "w") as fp: json.dump(cantian_config_data, fp) return common_parameters @@ -2227,8 +2625,10 @@ class Installer: cmd = cmd.strip(";") ret_code, _, stderr = _exec_popen(cmd) if ret_code: - raise Exception("Can not write the %s, command: %s," - " output: %s" % (self.CANTIAND_HBA_FILE, cmd, stderr)) + raise Exception( + "Can not write the %s, command: %s," + " output: %s" % (self.CANTIAND_HBA_FILE, cmd, stderr) + ) def init_db_instance(self): """ @@ -2245,7 +2645,9 @@ class Installer: self.set_cthba_ssl() self.add_cthba_item() # g_opts.isencrept默认加密 - self.set_conf(self.cantiand_configs, self.CANTIAND_CONF_FILE, g_opts.isencrept) + self.set_conf( + self.cantiand_configs, self.CANTIAND_CONF_FILE, g_opts.isencrept + ) self.set_cluster_conf() except Exception as error: LOGGER.error(str(error)) @@ -2267,7 +2669,7 @@ class Installer: ins_list = ins_str.split(os.sep) reg_string = "" for i in ins_list: - if (i == ""): + if i == "": continue else: reg_string += r"\/" + i @@ -2281,8 +2683,7 @@ class Installer: LOGGER.info("Clean environment variables cmd: %s" % cmd) ret_code, _, stderr = _exec_popen(cmd) if ret_code: - log("Failed to clean environment variables." - " Error: %s" % stderr) + log("Failed to clean environment variables." " Error: %s" % stderr) err_msg = "Failed to clean environment variables." LOGGER.error(err_msg) raise Exception(err_msg) @@ -2315,16 +2716,17 @@ class Installer: # Clear environment variable CTDB_DATA data_cmd = r"/^\s*export\s*CTDB_DATA=.*$/d" # Clear environment variable PATH about database - path_cmd = (r"/^\s*export\s*PATH=.*%s\/bin.*:\$PATH$/d" - % self.genregstring(self.install_path)) + path_cmd = r"/^\s*export\s*PATH=.*%s\/bin.*:\$PATH$/d" % self.genregstring( + self.install_path + ) # Clear environment variable LD_LIBRARY_PATH about database - lib_cmd = (r"/^\s*export\s*LD_LIBRARY_PATH=.*%s\/lib.*" - r":.*%s\/add-ons*$/d" - % (self.genregstring(self.install_path), - self.genregstring(self.install_path))) + lib_cmd = r"/^\s*export\s*LD_LIBRARY_PATH=.*%s\/lib.*" r":.*%s\/add-ons*$/d" % ( + self.genregstring(self.install_path), + self.genregstring(self.install_path), + ) # Clear environment variable CTDB_HOME home_cmd = r"/^\s*export\s*CTDB_HOME=.*$/d" - # Clear environment variable CANTIANLOG + # Clear environment variable CANTIANLOG cantianlog_cmd = r"/^\s*export\s*CANTIANLOG=.*$/d" # Clear environment ssl cert @@ -2334,8 +2736,17 @@ class Installer: mode_cmd = r"/^\s*export\s*CTSQL_SSL_MODE=.*$/d" cipher_cmd = r"/^\s*export\s*CTSQL_SSL_KEY_PASSWD=.*$/d" - cmds = [path_cmd, lib_cmd, home_cmd, cantianlog_cmd, - ca_cmd, cert_cmd, key_cmd, mode_cmd, cipher_cmd] + cmds = [ + path_cmd, + lib_cmd, + home_cmd, + cantianlog_cmd, + ca_cmd, + cert_cmd, + key_cmd, + mode_cmd, + cipher_cmd, + ] if self.option == self.INS_ALL: cmds.insert(0, data_cmd) @@ -2373,14 +2784,21 @@ class Installer: def __kill_process(self, process_name): # root do install, need su - user kill process - kill_cmd = (r"proc_pid_list=`ps ux | grep %s | grep -v grep" - r"|awk '{print $2}'` && " % process_name) - kill_cmd += (r"(if [ X\"$proc_pid_list\" != X\"\" ];then echo " - r"$proc_pid_list | xargs kill -9; fi)") + kill_cmd = ( + r"proc_pid_list=`ps ux | grep %s | grep -v grep" + r"|awk '{print $2}'` && " % process_name + ) + kill_cmd += ( + r"(if [ X\"$proc_pid_list\" != X\"\" ];then echo " + r"$proc_pid_list | xargs kill -9; fi)" + ) LOGGER.info("kill process cmd: %s" % kill_cmd) ret_code, stdout, stderr = _exec_popen(kill_cmd) if ret_code: - err_msg = "kill process %s faild. ret_code : %s, stdout : %s, stderr : %s" % (process_name, ret_code, stdout, stderr) + err_msg = ( + "kill process %s faild. ret_code : %s, stdout : %s, stderr : %s" + % (process_name, ret_code, stdout, stderr) + ) LOGGER.error(err_msg) raise Exception(err_msg) @@ -2405,20 +2823,30 @@ class Installer: ret_code, _, stderr = _exec_popen(str_cmd) if ret_code: raise Exception( - "chmod %s return: " % CommonValue.KEY_DIRECTORY_MODE + str(ret_code) + os.linesep + stderr) + "chmod %s return: " % CommonValue.KEY_DIRECTORY_MODE + + str(ret_code) + + os.linesep + + stderr + ) # create data, cfg, log dir, trc data_dir = "%s/data" % self.data if not g_opts.use_dbstor: - mount_storage_data = f"/mnt/dbdata/remote/storage_{g_opts.storage_dbstor_fs}/data" + mount_storage_data = ( + f"/mnt/dbdata/remote/storage_{g_opts.storage_dbstor_fs}/data" + ) cmd = "ln -s %s %s;" % (mount_storage_data, self.data) ret_code, _, stderr = _exec_popen(cmd) if ret_code: - raise Exception("Can not link data dir, command: %s, output: %s" % (cmd, stderr)) + raise Exception( + "Can not link data dir, command: %s, output: %s" % (cmd, stderr) + ) else: os.makedirs(data_dir, CommonValue.KEY_DIRECTORY_PERMISSION) os.makedirs("%s/log" % self.data, CommonValue.KEY_DIRECTORY_PERMISSION) - os.makedirs("%s/archive_log" % self.data, CommonValue.KEY_DIRECTORY_PERMISSION) + os.makedirs( + "%s/archive_log" % self.data, CommonValue.KEY_DIRECTORY_PERMISSION + ) os.makedirs("%s/trc" % self.data, CommonValue.KEY_DIRECTORY_PERMISSION) os.makedirs("%s/tmp" % self.data, CommonValue.KEY_DIRECTORY_PERMISSION) @@ -2426,18 +2854,36 @@ class Installer: os.makedirs("%s/dbs" % self.data, CommonValue.KEY_DIRECTORY_PERMISSION) # move the config files about database. - cmd = "mv -i %s/cfg %s;touch %s/cfg/%s" % (self.install_path, self.data, self.data, self.CTSQL_CONF_FILE) + cmd = "mv -i %s/cfg %s;touch %s/cfg/%s" % ( + self.install_path, + self.data, + self.data, + self.CTSQL_CONF_FILE, + ) ret_code, _, stderr = _exec_popen(cmd) if ret_code: - raise Exception("Can not create prepare data dir, command: %s, output: %s" % (cmd, stderr)) + raise Exception( + "Can not create prepare data dir, command: %s, output: %s" + % (cmd, stderr) + ) # Change the mode of config files to 600 cmd = "chmod {0} {1}/cfg/{2} {1}/cfg/{3} {1}/cfg/{4} {1}/cfg/{5}".format( - CommonValue.KEY_FILE_MODE, self.data, self.CANTIAND_CONF_FILE, - self.CMS_CONF_FILE, self.CANTIAND_HBA_FILE, self.CTSQL_CONF_FILE) + CommonValue.KEY_FILE_MODE, + self.data, + self.CANTIAND_CONF_FILE, + self.CMS_CONF_FILE, + self.CANTIAND_HBA_FILE, + self.CTSQL_CONF_FILE, + ) ret_code, _, stderr = _exec_popen(cmd) if ret_code: - raise Exception("chmod %s return: " % CommonValue.KEY_FILE_MODE + str(ret_code) + os.linesep + stderr) + raise Exception( + "chmod %s return: " % CommonValue.KEY_FILE_MODE + + str(ret_code) + + os.linesep + + stderr + ) # Change the owner of config files self.chown_data_dir() @@ -2474,7 +2920,9 @@ class Installer: # start cantian dn flags = os.O_RDONLY modes = stat.S_IWUSR | stat.S_IRUSR - with os.fdopen(os.open(CANTIAN_START_STATUS_FILE, flags, modes), 'r') as load_fp: + with os.fdopen( + os.open(CANTIAN_START_STATUS_FILE, flags, modes), "r" + ) as load_fp: start_parameters = json.load(load_fp) self.chown_data_dir() status_success = False @@ -2484,23 +2932,43 @@ class Installer: start_mode = self.OPEN_MODE if g_opts.install_type == "reserve": start_mode = self.OPEN_MODE - if start_parameters.setdefault('db_create_status', "default") == "done" and g_opts.node_id == 0: + if ( + start_parameters.setdefault("db_create_status", "default") == "done" + and g_opts.node_id == 0 + ): start_mode = self.OPEN_MODE - if start_parameters.setdefault('mysql_init', "default") == "done" or no_init: + if start_parameters.setdefault("mysql_init", "default") == "done" or no_init: mysql_init = False # Start instance, according to running mode can point to cantiand or cantiand with mysql cmd = "echo -e '%s' | sh %s -P cantiand -M %s -T %s -C %s >> %s 2>&1" % ( - g_opts.db_passwd, INSTALL_SCRIPT, start_mode, g_opts.running_mode.lower(), - g_opts.mysql_config_file_path, g_opts.log_file) + g_opts.db_passwd, + INSTALL_SCRIPT, + start_mode, + g_opts.running_mode.lower(), + g_opts.mysql_config_file_path, + g_opts.log_file, + ) install_log_file = self.status_log - if g_opts.running_mode.lower() in [CANTIAND_WITH_MYSQL, CANTIAND_WITH_MYSQL_IN_CLUSTER, CANTIAND_WITH_MYSQL_ST]: + if g_opts.running_mode.lower() in [ + CANTIAND_WITH_MYSQL, + CANTIAND_WITH_MYSQL_IN_CLUSTER, + CANTIAND_WITH_MYSQL_ST, + ]: log_home = self.cantiand_configs["LOG_HOME"] install_log_file = os.path.join(log_home, "run", "cantiand.rlog") if not mysql_init: - cmd = "echo -e '%s' | sh %s -P cantiand -M %s -T %s -C %s -R >> %s 2>&1" % ( - g_opts.db_passwd, INSTALL_SCRIPT, start_mode, g_opts.running_mode.lower(), - g_opts.mysql_config_file_path, g_opts.log_file) + cmd = ( + "echo -e '%s' | sh %s -P cantiand -M %s -T %s -C %s -R >> %s 2>&1" + % ( + g_opts.db_passwd, + INSTALL_SCRIPT, + start_mode, + g_opts.running_mode.lower(), + g_opts.mysql_config_file_path, + g_opts.log_file, + ) + ) now_time = datetime.now() begin_time = str(now_time).split(".")[0] else: @@ -2511,13 +2979,17 @@ class Installer: output = stdout + stderr if g_opts.db_passwd in output: output = "installdb.sh was killed" - raise Exception("Can not start instance %s.\nOutput: %s" % (self.data, output)) + raise Exception( + "Can not start instance %s.\nOutput: %s" % (self.data, output) + ) # In some condition cantian will take some time to start, so wait # it by checking the process cyclically after the start command # returned. If the cantiand process can't be found within the # expected time, it is considered that the startup failed. - tem_log_info, status_success = self.init_some_condition(status_success, install_log_file, begin_time) + tem_log_info, status_success = self.init_some_condition( + status_success, install_log_file, begin_time + ) # the log file's permission is 600, change it if os.path.exists(self.status_log): @@ -2527,14 +2999,18 @@ class Installer: os.chmod(self.status_log, CommonValue.KEY_FILE_PERMISSION) if not status_success: - raise Exception("Can not get instance '%s' process pid," - "The detailed information: '%s' " % (self.data, tem_log_info)) + raise Exception( + "Can not get instance '%s' process pid," + "The detailed information: '%s' " % (self.data, tem_log_info) + ) log("cantiand has started") flags = os.O_RDWR | os.O_CREAT modes = stat.S_IWUSR | stat.S_IRUSR - with os.fdopen(os.open(CANTIAN_START_STATUS_FILE, flags, modes), 'w+') as load_fp: + with os.fdopen( + os.open(CANTIAN_START_STATUS_FILE, flags, modes), "w+" + ) as load_fp: start_parameters = json.load(load_fp) - start_status_item = {'mysql_init': "done"} + start_status_item = {"mysql_init": "done"} start_parameters.update(start_status_item) load_fp.seek(0) load_fp.truncate() @@ -2547,7 +3023,10 @@ class Installer: ret_code, stdout, stderr = _exec_popen(cmd) output = stdout + stderr if ret_code: - LOGGER.info("Failed to get the error message from '%s'. Output: %s" % (run_log, output)) + LOGGER.info( + "Failed to get the error message from '%s'. Output: %s" + % (run_log, output) + ) return "" else: return output @@ -2558,40 +3037,61 @@ class Installer: for i in range(0, start_time): time.sleep(3) - cmd = ("ps aux | grep -v grep | grep %s | grep $ " - "|awk '{print $2}'" % (self.data)) - if g_opts.running_mode.lower() in [CANTIAND_WITH_MYSQL, CANTIAND_WITH_MYSQL_ST, - CANTIAND_WITH_MYSQL_IN_CLUSTER]: + cmd = "ps aux | grep -v grep | grep %s | grep $ " "|awk '{print $2}'" % ( + self.data + ) + if g_opts.running_mode.lower() in [ + CANTIAND_WITH_MYSQL, + CANTIAND_WITH_MYSQL_ST, + CANTIAND_WITH_MYSQL_IN_CLUSTER, + ]: cmd = "ps -ef | grep /opt/cantian/mysql/install/mysql/bin/mysqld | grep -v grep | awk '{print $2}'" ret_code, stdout, stderr = _exec_popen(cmd) if ret_code: status_success = False - tem_log_info = ("Failed to execute cmd: %s.output:%s" - % (str(cmd), str(stderr))) + tem_log_info = "Failed to execute cmd: %s.output:%s" % ( + str(cmd), + str(stderr), + ) break else: - all_the_text = open(status_log, errors='ignore').read() + all_the_text = open(status_log, errors="ignore").read() is_instance_started = False is_instance_failed = False - if g_opts.running_mode.lower() in [CANTIAND_WITH_MYSQL, CANTIAND_WITH_MYSQL_ST, - CANTIAND_WITH_MYSQL_IN_CLUSTER] : - succ_pattern = re.compile(r'.{10}(\d{4}\-\d{2}\-\d{2} \d{2}\:\d{2}\:\d{2}).*?instance started.*?', re.IGNORECASE) - fail_pattern = re.compile(r'.{10}(\d{4}\-\d{2}\-\d{2} \d{2}\:\d{2}\:\d{2}).*?instance startup failed.*?', re.IGNORECASE) + if g_opts.running_mode.lower() in [ + CANTIAND_WITH_MYSQL, + CANTIAND_WITH_MYSQL_ST, + CANTIAND_WITH_MYSQL_IN_CLUSTER, + ]: + succ_pattern = re.compile( + r".{10}(\d{4}\-\d{2}\-\d{2} \d{2}\:\d{2}\:\d{2}).*?instance started.*?", + re.IGNORECASE, + ) + fail_pattern = re.compile( + r".{10}(\d{4}\-\d{2}\-\d{2} \d{2}\:\d{2}\:\d{2}).*?instance startup failed.*?", + re.IGNORECASE, + ) succ_timestamps = re.findall(succ_pattern, all_the_text) fail_timestamps = re.findall(fail_pattern, all_the_text) - is_instance_started = len(succ_timestamps) != 0 and max(succ_timestamps) >= begin_time - is_instance_failed = len(fail_timestamps) != 0 and max(fail_timestamps) >= begin_time + is_instance_started = ( + len(succ_timestamps) != 0 and max(succ_timestamps) >= begin_time + ) + is_instance_failed = ( + len(fail_timestamps) != 0 and max(fail_timestamps) >= begin_time + ) else: is_instance_started = all_the_text.find("instance started") >= 0 - is_instance_failed = all_the_text.find("instance startup failed") > 0 - if (is_instance_started): + is_instance_failed = ( + all_the_text.find("instance startup failed") > 0 + ) + if is_instance_started: if stdout: status_success = True self.pid = stdout.strip() LOGGER.info("start instance successfully, pid = %s" % stdout) break - elif (is_instance_failed): + elif is_instance_failed: status_success = False tem_log_info = all_the_text.strip() # Get the error message from run log. After roll_back, @@ -2600,7 +3100,10 @@ class Installer: run_log_info = self.get_invalid_parameter() if run_log_info: tem_log_info += os.linesep - tem_log_info += ("The run log error: %s%s" % (os.linesep, run_log_info)) + tem_log_info += "The run log error: %s%s" % ( + os.linesep, + run_log_info, + ) break if (i + 1) == start_time: status_success = False @@ -2632,30 +3135,44 @@ class Installer: return "Install type is reserve." if self.enable_sysdba_login: - cmd = "source ~/.bashrc && %s/bin/ctsql / as sysdba -q -D %s -f %s" % (self.install_path, - self.data, sql_file) + cmd = "source ~/.bashrc && %s/bin/ctsql / as sysdba -q -D %s -f %s" % ( + self.install_path, + self.data, + sql_file, + ) return_code, stdout_data, stderr_data = _exec_popen(cmd) else: - cmd = ("source ~/.bashrc && echo -e '%s' | %s/bin/ctsql %s@%s:%s -q -f %s" % ( - g_opts.db_passwd, - self.install_path, - g_opts.db_user, - self.login_ip, - self.lsnr_port, - sql_file)) + cmd = ( + "source ~/.bashrc && echo -e '%s' | %s/bin/ctsql %s@%s:%s -q -f %s" + % ( + g_opts.db_passwd, + self.install_path, + g_opts.db_user, + self.login_ip, + self.lsnr_port, + sql_file, + ) + ) return_code, stdout_data, stderr_data = _exec_popen(cmd) - output = "%s%s".replace("password", "***") % (str(stdout_data), str(stderr_data)) + output = "%s%s".replace("password", "***") % ( + str(stdout_data), + str(stderr_data), + ) if g_opts.db_passwd in output: output = "execute ctsql file failed" LOGGER.info("Execute sql file %s output: %s" % (sql_file, output)) if return_code: - raise Exception("Failed to execute sql file %s, output:%s" % (sql_file, output)) + raise Exception( + "Failed to execute sql file %s, output:%s" % (sql_file, output) + ) # return code is 0, but output has error info, CT-xxx, ZS-xxx result = output.replace("\n", "") if re.match(".*CT-\d{5}.*", result) or re.match(".*ZS-\d{5}.*", result): - raise Exception("Failed to execute sql file %s, output:%s" % (sql_file, output)) + raise Exception( + "Failed to execute sql file %s, output:%s" % (sql_file, output) + ) return stdout_data def execute_sql(self, sql, message): @@ -2673,26 +3190,27 @@ class Installer: ctsql_path = "%s/bin/*sql" % self.install_path ctsql_path = glob.glob(ctsql_path)[0] if self.enable_sysdba_login: - cmd = ("source ~/.bashrc && %s / as sysdba " - "-q -D %s -c \"%s\"" - % (ctsql_path, self.data, sql)) + cmd = "source ~/.bashrc && %s / as sysdba " '-q -D %s -c "%s"' % ( + ctsql_path, + self.data, + sql, + ) return_code, stdout_data, stderr_data = _exec_popen(cmd) else: - cmd = ("source ~/.bashrc && echo -e '%s' | %s %s@%s:%s -q" - " -c \"%s\"" % ( - g_opts.db_passwd, - ctsql_path, - g_opts.db_user, - self.login_ip, - self.lsnr_port, - sql)) + cmd = "source ~/.bashrc && echo -e '%s' | %s %s@%s:%s -q" ' -c "%s"' % ( + g_opts.db_passwd, + ctsql_path, + g_opts.db_user, + self.login_ip, + self.lsnr_port, + sql, + ) return_code, stdout_data, stderr_data = _exec_popen(cmd) output = "%s%s" % (str(stdout_data), str(stderr_data)) output.replace(g_opts.db_passwd, "*****") if return_code: - raise Exception("Failed to %s by sql, output:%s" - % (message, output)) + raise Exception("Failed to %s by sql, output:%s" % (message, output)) # return code is 0, but output has error info, CT-xxx, ZS-xxx result = output.replace("\n", "") @@ -2715,26 +3233,37 @@ class Installer: flags = os.O_RDONLY modes = stat.S_IWUSR | stat.S_IRUSR - with os.fdopen(os.open(CANTIAN_CONF_FILE, flags, modes), 'r') as fp: + with os.fdopen(os.open(CANTIAN_CONF_FILE, flags, modes), "r") as fp: json_data = json.load(fp) - self.user = json_data['USER'].strip() - self.group = json_data['GROUP'].strip() - self.data = json_data['DATA'].strip() - self.create_db_file = json_data['CREAT_DB_FILE'].strip() - self.install_path = json_data['INSTALL_PATH'].strip() - g_opts.running_mode = json_data['RUNNING_MODE'].strip() - self.cantiand_configs["LOG_HOME"] = json_data.get('LOG_HOME', '').strip() - self.cantiand_configs["ENABLE_DBSTOR"] = json_data.get('ENABLE_DBSTOR', '').strip() + self.user = json_data["USER"].strip() + self.group = json_data["GROUP"].strip() + self.data = json_data["DATA"].strip() + self.create_db_file = json_data["CREAT_DB_FILE"].strip() + self.install_path = json_data["INSTALL_PATH"].strip() + g_opts.running_mode = json_data["RUNNING_MODE"].strip() + self.cantiand_configs["LOG_HOME"] = json_data.get("LOG_HOME", "").strip() + self.cantiand_configs["ENABLE_DBSTOR"] = json_data.get( + "ENABLE_DBSTOR", "" + ).strip() if g_opts.use_dbstor: - self.cantiand_configs["SHARED_PATH"] = json_data.get('SHARED_PATH', '') - self.cantiand_configs["CONTROL_FILES"] = json_data.get('CONTROL_FILES', '').strip() - self.cantiand_configs["DBSTOR_NAMESPACE"] = json_data.get('DBSTOR_NAMESPACE', '').strip() + self.cantiand_configs["SHARED_PATH"] = json_data.get("SHARED_PATH", "") + self.cantiand_configs["CONTROL_FILES"] = json_data.get( + "CONTROL_FILES", "" + ).strip() + self.cantiand_configs["DBSTOR_NAMESPACE"] = json_data.get( + "DBSTOR_NAMESPACE", "" + ).strip() elif g_opts.use_gss: - self.cantiand_configs["CONTROL_FILES"] = "(+vg1/ctrl1, +vg1/ctrl2, +vg1/ctrl3)" + self.cantiand_configs["CONTROL_FILES"] = ( + "(+vg1/ctrl1, +vg1/ctrl2, +vg1/ctrl3)" + ) self.cantiand_configs["SHARED_PATH"] = "+vg1" else: - self.cantiand_configs["SHARED_PATH"] = '/mnt/dbdata/remote/storage_{}/data'.format( - g_opts.storage_dbstor_fs) + self.cantiand_configs["SHARED_PATH"] = ( + "/mnt/dbdata/remote/storage_{}/data".format( + g_opts.storage_dbstor_fs + ) + ) # clean old backup log # backup log file before rm data @@ -2752,33 +3281,41 @@ class Installer: cmd = "chown %s:%s %s;" % (self.user, self.group, INSTALL_SCRIPT) ret_code, _, stderr = _exec_popen(cmd) if ret_code: - raise Exception("chown to %s:%s return: %s%s%s" - % (self.user, self.group, str(ret_code), - os.linesep, stderr)) + raise Exception( + "chown to %s:%s return: %s%s%s" + % (self.user, self.group, str(ret_code), os.linesep, stderr) + ) try: # 准备拉起cantiand或mysqld self.failed_pos = self.CREATE_DB_FAILED mysql_init = True - with os.fdopen(os.open(CANTIAN_START_STATUS_FILE, flags, modes), 'r') as load_fp: + with os.fdopen( + os.open(CANTIAN_START_STATUS_FILE, flags, modes), "r" + ) as load_fp: start_parameters = json.load(load_fp) - if start_parameters.setdefault('mysql_init', "default") == "done": + if start_parameters.setdefault("mysql_init", "default") == "done": mysql_init = False if g_opts.node_id != 0: # node 1 self.failed_pos = self.CREATE_DB_FAILED cantian_check_share_logic_ip_isvalid("share", g_opts.share_logic_ip) - LOGGER.info('Begin to start node1') - with os.fdopen(os.open(CONFIG_PARAMS_FILE, flags, modes), 'r') as fp: + LOGGER.info("Begin to start node1") + with os.fdopen(os.open(CONFIG_PARAMS_FILE, flags, modes), "r") as fp: json_data = json.load(fp) - is_metadata_in_cantian = str(json_data.get('mysql_metadata_in_cantian', 'True')) - if is_metadata_in_cantian != 'True': + is_metadata_in_cantian = str( + json_data.get("mysql_metadata_in_cantian", "True") + ) + if is_metadata_in_cantian != "True": # 非归一，第一次是初始化，需要第二次start - log('mysql initialize...........') + log("mysql initialize...........") self.start_cantiand() - if g_opts.running_mode in VALID_SINGLE_MYSQL_RUNNING_MODE and mysql_init: + if ( + g_opts.running_mode in VALID_SINGLE_MYSQL_RUNNING_MODE + and mysql_init + ): # 若是首次拉起且单进程，还需二次启动mysql；否则退出 cmd = "pidof mysqld" - log('wait for mysqld init complete.....................') + log("wait for mysqld init complete.....................") while True: return_code, stdout_data, _ = _exec_popen(cmd) if return_code or not stdout_data: @@ -2794,14 +3331,17 @@ class Installer: cantian_check_share_logic_ip_isvalid("share", g_opts.share_logic_ip) self.start_cantiand() log("Creating cantian database...") - log('wait for cantiand thread startup') + log("wait for cantiand thread startup") time.sleep(20) self.create_db() - if g_opts.running_mode in VALID_SINGLE_MYSQL_RUNNING_MODE and mysql_init: + if ( + g_opts.running_mode in VALID_SINGLE_MYSQL_RUNNING_MODE + and mysql_init + ): # 第二次拉起mysqld。 cmd = "pidof mysqld" - log('wait for mysqld init complete.....................') + log("wait for mysqld init complete.....................") while True: return_code, stdout_data, _ = _exec_popen(cmd) if return_code or not stdout_data: @@ -2819,19 +3359,25 @@ class Installer: return flags = os.O_RDONLY modes = stat.S_IWUSR | stat.S_IRUSR - with os.fdopen(os.open(CANTIAN_START_STATUS_FILE, flags, modes), 'r') as load_fp: + with os.fdopen( + os.open(CANTIAN_START_STATUS_FILE, flags, modes), "r" + ) as load_fp: start_parameters = json.load(load_fp) - if (start_parameters.setdefault('db_create_status', "default") == "default"): + if start_parameters.setdefault("db_create_status", "default") == "default": self.update_factor_key() flags = os.O_WRONLY | os.O_TRUNC - db_create_status_item = {'db_create_status': "creating"} + db_create_status_item = {"db_create_status": "creating"} start_parameters.update(db_create_status_item) - with os.fdopen(os.open(CANTIAN_START_STATUS_FILE, flags, modes), 'w') as load_fp: + with os.fdopen( + os.open(CANTIAN_START_STATUS_FILE, flags, modes), "w" + ) as load_fp: json.dump(start_parameters, load_fp) self.execute_sql_file(self.get_database_file()) - db_create_status_item = {'db_create_status': "done"} + db_create_status_item = {"db_create_status": "done"} start_parameters.update(db_create_status_item) - with os.fdopen(os.open(CANTIAN_START_STATUS_FILE, flags, modes), 'w') as load_fp: + with os.fdopen( + os.open(CANTIAN_START_STATUS_FILE, flags, modes), "w" + ) as load_fp: json.dump(start_parameters, load_fp) def check_db_status(self): @@ -2866,7 +3412,9 @@ class Installer: continue if "1 rows fetched" not in res: continue - db_status = re.split(r"\s+", re.split(r"\n+", res.strip())[-2].strip())[1].strip() + db_status = re.split(r"\s+", re.split(r"\n+", res.strip())[-2].strip())[ + 1 + ].strip() LOGGER.info("ctsql db status: %s" % db_status) if db_status == "OPEN": LOGGER.info("Cantiand start success, db status: %s" % db_status) @@ -2891,31 +3439,38 @@ class Installer: create_database_sql = os.path.join(sql_file_path, file_name) if g_opts.use_dbstor: file_name = "create_dbstor_database.sample.sql" - if g_opts.running_mode in [CANTIAND_IN_CLUSTER, CANTIAND_WITH_MYSQL_IN_CLUSTER]: + if g_opts.running_mode in [ + CANTIAND_IN_CLUSTER, + CANTIAND_WITH_MYSQL_IN_CLUSTER, + ]: file_name = "create_dbstor_cluster_database.sample.sql" create_database_sql = os.path.join(sql_file_path, file_name) file_name_0 = "create_dbstor_cluster_database.lun.sql" file_name_1 = "create_dbstor_cluster_database.sample.sql" file_name_2 = "create_dbstor_database.sample.sql" - if g_opts.db_type in ['0', '1', '2']: + if g_opts.db_type in ["0", "1", "2"]: create_database_sql_dic = {} - create_database_sql_dic['0'] = file_name_0 - create_database_sql_dic['1'] = file_name_1 - create_database_sql_dic['2'] = file_name_2 - create_database_sql = os.path.join(sql_file_path, create_database_sql_dic.get(g_opts.db_type)) + create_database_sql_dic["0"] = file_name_0 + create_database_sql_dic["1"] = file_name_1 + create_database_sql_dic["2"] = file_name_2 + create_database_sql = os.path.join( + sql_file_path, create_database_sql_dic.get(g_opts.db_type) + ) else: create_database_sql = os.path.join(sql_file_path, file_name) file_name_0 = "create_cluster_database.lun.sql" file_name_1 = "create_cluster_database.sample.sql" file_name_2 = "create_database.sample.sql" - if g_opts.db_type in ['0', '1', '2']: + if g_opts.db_type in ["0", "1", "2"]: create_database_sql_dic = {} - create_database_sql_dic['0'] = file_name_0 - create_database_sql_dic['1'] = file_name_1 - create_database_sql_dic['2'] = file_name_2 - create_database_sql = os.path.join(sql_file_path, create_database_sql_dic.get(g_opts.db_type)) - - db_data_path = os.path.join(self.data, "data").replace('/', '\/') + create_database_sql_dic["0"] = file_name_0 + create_database_sql_dic["1"] = file_name_1 + create_database_sql_dic["2"] = file_name_2 + create_database_sql = os.path.join( + sql_file_path, create_database_sql_dic.get(g_opts.db_type) + ) + + db_data_path = os.path.join(self.data, "data").replace("/", "\/") self.set_sql_redo_size_and_num(db_data_path, create_database_sql) if g_opts.use_gss: self._sed_file("dbfiles1", "+vg1", create_database_sql) @@ -2929,7 +3484,7 @@ class Installer: return create_database_sql def _sed_file(self, prefix, replace, file_name): - fix_sql_file_cmd = ("sed -i 's/%s/%s/g' %s" % (prefix, replace, file_name)) + fix_sql_file_cmd = "sed -i 's/%s/%s/g' %s" % (prefix, replace, file_name) ret_code, _, _ = _exec_popen(fix_sql_file_cmd) if ret_code: raise Exception("sed %s failed, replace %s" % (file_name, replace)) @@ -2938,8 +3493,10 @@ class Installer: cmd = "chown -hR %s:%s %s; " % (self.user, self.group, self.ssl_path) ret_code, _, stderr = _exec_popen(cmd) if ret_code: - raise Exception("chown to %s:%s return: %s%s%s" - % (self.user, self.group, str(ret_code), os.linesep, stderr)) + raise Exception( + "chown to %s:%s return: %s%s%s" + % (self.user, self.group, str(ret_code), os.linesep, stderr) + ) def get_ctencrypt_keys(self, skip_execute_sql=False): """Set the config about _FACTOR_KEY and LOCAL_KEY.""" @@ -2948,8 +3505,10 @@ class Installer: cmd = "%s/bin/ctencrypt -g" % self.install_path ret_code, stdout, stderr = _exec_popen(cmd) if ret_code: - raise OSError("Failed to generate encrypted keys. Error: %s" - % (stderr + os.linesep + stderr)) + raise OSError( + "Failed to generate encrypted keys. Error: %s" + % (stderr + os.linesep + stderr) + ) # Example of output: # eg'Key: XXXXXXXXXXXXXXXXXXXXXXX' @@ -2981,8 +3540,10 @@ class Installer: cmd = "%s/bin/ctencrypt -g -o '%s' " % (self.install_path, f_factor1) ret_code, stdout, stderr = _exec_popen(cmd) if ret_code: - raise OSError("Failed to generate encrypted keys. Error: %s" - % (stderr + os.linesep + stderr)) + raise OSError( + "Failed to generate encrypted keys. Error: %s" + % (stderr + os.linesep + stderr) + ) # Example of output: # eg'Key: XXXXXXXXXXXXXXXXXXXXXXX' @@ -3005,18 +3566,25 @@ class Installer: LOGGER.info("Generate encrypted keys successfully.") return key_, work_key - def encrypt_ssl_key_passwd(self, key_, work_key, ssl_passwd, skip_execute_sql=False): + def encrypt_ssl_key_passwd( + self, key_, work_key, ssl_passwd, skip_execute_sql=False + ): """Encrypt ssl key password with _FACTOR_KEY and LOCAL_KEY.""" LOGGER.info("Encrypt ssl key password.") - cmd = ("%s/bin/ctencrypt -e AES256 -f %s -k %s" - % (self.install_path, key_, work_key)) + cmd = "%s/bin/ctencrypt -e AES256 -f %s -k %s" % ( + self.install_path, + key_, + work_key, + ) values = [ssl_passwd, ssl_passwd] ret_code, stdout, stderr = _exec_popen(cmd, values) if ret_code: - raise OSError("Failed to encrypt ssl key password. Error: %s" - % (stderr + os.linesep + stderr)) + raise OSError( + "Failed to encrypt ssl key password. Error: %s" + % (stderr + os.linesep + stderr) + ) # Example of output: # Please enter password to encrypt: @@ -3038,8 +3606,7 @@ class Installer: # Don't set SSL_CA and CTSQL_SSL_CA. # Avoid the need to copy files, env and kernel parameter # from the primary dn when installing the backup dn. - cantian_conf_file = os.path.join(self.data, "cfg", - self.CANTIAND_CONF_FILE) + cantian_conf_file = os.path.join(self.data, "cfg", self.CANTIAND_CONF_FILE) ssl_map = { "SSL_CERT": os.path.join(self.ssl_path, "server.crt"), "SSL_KEY": os.path.join(self.ssl_path, "server.key"), @@ -3069,21 +3636,26 @@ class Installer: try: flags = os.O_RDWR modes = stat.S_IWUSR | stat.S_IRUSR - with os.fdopen(os.open(self.user_profile, flags, modes), 'a') as _file: - _file.write("export CTSQL_SSL_CERT=\"%s\"" - % os.path.join(self.ssl_path, "client.crt")) + with os.fdopen(os.open(self.user_profile, flags, modes), "a") as _file: + _file.write( + 'export CTSQL_SSL_CERT="%s"' + % os.path.join(self.ssl_path, "client.crt") + ) _file.write(os.linesep) - _file.write("export CTSQL_SSL_KEY=\"%s\"" - % os.path.join(self.ssl_path, "client.key")) + _file.write( + 'export CTSQL_SSL_KEY="%s"' + % os.path.join(self.ssl_path, "client.key") + ) _file.write(os.linesep) - _file.write("export CTSQL_SSL_MODE=\"required\"") + _file.write('export CTSQL_SSL_MODE="required"') _file.write(os.linesep) - _file.write("export CTSQL_SSL_KEY_PASSWD=\"%s\"" % cipher) + _file.write('export CTSQL_SSL_KEY_PASSWD="%s"' % cipher) _file.write(os.linesep) _file.flush() except IOError as ex: - raise IOError("Failed Set user environment variables about ssl: %s" - % str(ex)) from ex + raise IOError( + "Failed Set user environment variables about ssl: %s" % str(ex) + ) from ex os.environ["CTSQL_SSL_CERT"] = os.path.join(self.ssl_path, "client.crt") os.environ["CTSQL_SSL_KEY"] = os.path.join(self.ssl_path, "client.key") @@ -3099,22 +3671,32 @@ class Installer: if not g_opts.db_passwd: # connect database by sysdba - cmd = ("%s/bin/shutdowndb.sh -h %s -p %s -w -m %s -D %s -T %d" - % (self.install_path, host_ip, self.lsnr_port, - "immediate", self.data, timeout)) + cmd = "%s/bin/shutdowndb.sh -h %s -p %s -w -m %s -D %s -T %d" % ( + self.install_path, + host_ip, + self.lsnr_port, + "immediate", + self.data, + timeout, + ) ret_code, _, stderr = _exec_popen(cmd) else: # connect database by username and password - cmd = ("%s/bin/shutdowndb.sh -h" - " %s -p %s -U %s -m %s -W -D %s -T %d" % - (self.install_path, - host_ip, self.lsnr_port, g_opts.db_user, "immediate", - self.data, timeout)) + cmd = "%s/bin/shutdowndb.sh -h" " %s -p %s -U %s -m %s -W -D %s -T %d" % ( + self.install_path, + host_ip, + self.lsnr_port, + g_opts.db_user, + "immediate", + self.data, + timeout, + ) ret_code, _, stderr = _exec_popen(cmd, [g_opts.db_passwd]) if ret_code: - raise Exception("Failed to stop database. Error: %s" - % (stderr + os.linesep + stderr)) + raise Exception( + "Failed to stop database. Error: %s" % (stderr + os.linesep + stderr) + ) LOGGER.info("stop cantian instance successfully.") def chmod_install_sqlfile(self): @@ -3124,15 +3706,22 @@ class Installer: output: NA """ try: - str_cmd = ("find '%s'/admin -type f | xargs chmod %s " - % (self.install_path, CommonValue.MIN_FILE_MODE)) + str_cmd = "find '%s'/admin -type f | xargs chmod %s " % ( + self.install_path, + CommonValue.MIN_FILE_MODE, + ) ret_code, _, _ = _exec_popen(str_cmd) if ret_code: print_str_1 = CommonPrint() - print_str_1.common_log("Change file permission to %s failed." - " Please chmod %s filein directory %s/admin manually." - % (CommonValue.MIN_FILE_MODE, - CommonValue.MIN_FILE_MODE, self.install_path)) + print_str_1.common_log( + "Change file permission to %s failed." + " Please chmod %s filein directory %s/admin manually." + % ( + CommonValue.MIN_FILE_MODE, + CommonValue.MIN_FILE_MODE, + self.install_path, + ) + ) except Exception as error: LOGGER.error(str(error)) raise Exception(str(error)) @@ -3187,15 +3776,16 @@ class Installer: cantiand_ini = {} modes = stat.S_IWUSR | stat.S_IRUSR flags = os.O_RDONLY - with os.fdopen(os.open(CANTIAND_INI_FILE, flags, modes), 'r') as fp: + with os.fdopen(os.open(CANTIAND_INI_FILE, flags, modes), "r") as fp: for line in fp: if line == "\n": continue (key, val) = line.split(" = ") - val = val.replace('\n', '') + val = val.replace("\n", "") cantiand_ini[key] = val - self.enable_sysdba_login = Installer.check_pare_bool_value("ENABLE_SYSDBA_LOGIN", - cantiand_ini.get("ENABLE_SYSDBA_LOGIN", "FALSE")) + self.enable_sysdba_login = Installer.check_pare_bool_value( + "ENABLE_SYSDBA_LOGIN", cantiand_ini.get("ENABLE_SYSDBA_LOGIN", "FALSE") + ) def install_start(self): self.parse_cantiand_ini() @@ -3206,15 +3796,24 @@ class Installer: def check_parameter_mysql(self): if g_opts.mysql_config_file_path == "unset": - g_opts.mysql_config_file_path = os.path.join(MYSQL_CODE_DIR, "scripts/my.cnf") - LOGGER.info("no mysql config file assigned, set to %s" % g_opts.mysql_config_file_path) + g_opts.mysql_config_file_path = os.path.join( + MYSQL_CODE_DIR, "scripts/my.cnf" + ) + LOGGER.info( + "no mysql config file assigned, set to %s" + % g_opts.mysql_config_file_path + ) real_path = os.path.realpath(g_opts.mysql_config_file_path) if not os.path.isfile(real_path): - err_msg = "mysql config file {} not existed or it is not a file".format(real_path) + err_msg = "mysql config file {} not existed or it is not a file".format( + real_path + ) LOGGER.error(err_msg) raise Exception(err_msg) if not self.is_readable(real_path, self.user): - err_msg = "mysql config file {} is not readable by {}".format(real_path, self.user) + err_msg = "mysql config file {} is not readable by {}".format( + real_path, self.user + ) LOGGER.error(err_msg) raise Exception(err_msg) g_opts.mysql_config_file_path = real_path @@ -3222,7 +3821,10 @@ class Installer: cmd += "chmod {} {}".format(CommonValue.MAX_FILE_MODE, real_path) ret_code, _, stderr = _exec_popen(cmd) if ret_code: - err_msg = "Can not set mysql config mode, command: %s, output: %s" % (cmd, stderr) + err_msg = "Can not set mysql config mode, command: %s, output: %s" % ( + cmd, + stderr, + ) LOGGER.error(err_msg) raise Exception(err_msg) @@ -3269,10 +3871,10 @@ class Installer: """ LOGGER.info("Preparing path [%s]." % one_path) owner_path = one_path - if (os.path.exists(one_path)): - if (check_empty): + if os.path.exists(one_path): + if check_empty: file_list = os.listdir(one_path) - if (len(file_list) != 0): + if len(file_list) != 0: err_msg = "Database path %s should be empty." % one_path LOGGER.error(err_msg) raise Exception(err_msg) @@ -3290,8 +3892,7 @@ class Installer: # will cause an error if owner_path != one_path: cmd = "chown -hR %s:%s %s; " % (self.user, self.group, owner_path) - cmd += "chmod -R %s %s" % (CommonValue.KEY_DIRECTORY_MODE, - owner_path) + cmd += "chmod -R %s %s" % (CommonValue.KEY_DIRECTORY_MODE, owner_path) else: cmd = "chown %s:%s %s; " % (self.user, self.group, owner_path) cmd += "chmod %s %s" % (CommonValue.KEY_DIRECTORY_MODE, owner_path) @@ -3306,7 +3907,11 @@ class Installer: LOGGER.info("check [%s] user permission" % one_path) permission_ok, stderr = self.check_permission(one_path) if not permission_ok: - err_msg = "Failed to check user [%s] path [%s] permission. Error: %s" % (self.user, one_path, stderr) + err_msg = "Failed to check user [%s] path [%s] permission. Error: %s" % ( + self.user, + one_path, + stderr, + ) LOGGER.error(err_msg) raise Exception(err_msg) @@ -3319,52 +3924,71 @@ class Installer: LOGGER.info("Preparing mysql bin dir...") self.clean_dir(MYSQL_BIN_DIR) self.prepare_given_path(MYSQL_BIN_DIR, True) - cmd = "cp -arf {} {};".format(os.path.join(MYSQL_CODE_DIR, "mysql_bin/mysql/*"), MYSQL_BIN_DIR) + cmd = "cp -arf {} {};".format( + os.path.join(MYSQL_CODE_DIR, "mysql_bin/mysql/*"), MYSQL_BIN_DIR + ) cmd += "cp %s/cfg/osd.cfg %s/bin/osd.cfg;" % (self.data, MYSQL_BIN_DIR) cmd += "chown -hR {}:{} {};".format(self.user, self.group, MYSQL_BIN_DIR) ret_code, _, stderr = _exec_popen(cmd) if ret_code: - LOGGER.error("Can not copy mysql bin, command: %s, output: %s" % (cmd, stderr)) - raise Exception("Can not copy mysql bin, command: %s, output: %s" % (cmd, stderr)) + LOGGER.error( + "Can not copy mysql bin, command: %s, output: %s" % (cmd, stderr) + ) + raise Exception( + "Can not copy mysql bin, command: %s, output: %s" % (cmd, stderr) + ) LOGGER.info("end mysql bin dir...") + def set_mysql_env(self): LOGGER.info("Preparing mysql running env...") - if 'LD_LIBRARY_PATH' in os.environ: - os.environ['LD_LIBRARY_PATH'] = ("%s:%s:%s" % ( + if "LD_LIBRARY_PATH" in os.environ: + os.environ["LD_LIBRARY_PATH"] = "%s:%s:%s" % ( os.path.join(MYSQL_BIN_DIR, "lib"), os.path.join(MYSQL_CODE_DIR, "cantian_lib"), - os.environ['LD_LIBRARY_PATH'])) + os.environ["LD_LIBRARY_PATH"], + ) else: - os.environ['LD_LIBRARY_PATH'] = ("%s:%s" % ( - os.path.join(MYSQL_BIN_DIR, "lib"), os.path.join( - MYSQL_CODE_DIR, "cantian_lib"))) - cmd = "ldconfig -N %s %s" % (os.path.join(MYSQL_CODE_DIR, "cantian_lib"), os.path.join(MYSQL_BIN_DIR, "lib")) + os.environ["LD_LIBRARY_PATH"] = "%s:%s" % ( + os.path.join(MYSQL_BIN_DIR, "lib"), + os.path.join(MYSQL_CODE_DIR, "cantian_lib"), + ) + cmd = "ldconfig -N %s %s" % ( + os.path.join(MYSQL_CODE_DIR, "cantian_lib"), + os.path.join(MYSQL_BIN_DIR, "lib"), + ) ret_code, _, stderr = _exec_popen(cmd) if ret_code: - LOGGER.error("Can not link mysql lib, command: %s, output: %s" % (cmd, stderr)) - raise Exception("Can not link mysql lib, command: %s, output: %s" % (cmd, stderr)) + LOGGER.error( + "Can not link mysql lib, command: %s, output: %s" % (cmd, stderr) + ) + raise Exception( + "Can not link mysql lib, command: %s, output: %s" % (cmd, stderr) + ) def export_mysql_env(self): try: flags = os.O_RDWR modes = stat.S_IWUSR | stat.S_IRUSR - with os.fdopen(os.open(self.user_profile, flags, modes), 'a') as _file: - _file.write("export MYSQL_BIN_DIR=\"%s\"" % MYSQL_BIN_DIR) + with os.fdopen(os.open(self.user_profile, flags, modes), "a") as _file: + _file.write('export MYSQL_BIN_DIR="%s"' % MYSQL_BIN_DIR) _file.write(os.linesep) - _file.write("export MYSQL_CODE_DIR=\"%s\"" % MYSQL_CODE_DIR) + _file.write('export MYSQL_CODE_DIR="%s"' % MYSQL_CODE_DIR) _file.write(os.linesep) - _file.write("export MYSQL_DATA_DIR=\"%s\"" % MYSQL_DATA_DIR) + _file.write('export MYSQL_DATA_DIR="%s"' % MYSQL_DATA_DIR) _file.write(os.linesep) - _file.write("export MYSQL_LOG_FILE=\"%s\"" % MYSQL_LOG_FILE) + _file.write('export MYSQL_LOG_FILE="%s"' % MYSQL_LOG_FILE) _file.write(os.linesep) _file.flush() except IOError as ex: self.failed_pos = self.SET_ENV_FAILED log_exit("Can not set user environment variables: %s" % str(ex)) - + def prepare_mysql_for_single(self): - if g_opts.running_mode.lower() not in VALID_SINGLE_MYSQL_RUNNING_MODE or int(g_opts.cantian_in_container)==1: + if ( + g_opts.running_mode.lower() not in VALID_SINGLE_MYSQL_RUNNING_MODE + or int(g_opts.cantian_in_container) == 1 + ): return LOGGER.info("prepare_mysql_for_single") self.check_parameter_mysql() @@ -3375,20 +3999,20 @@ class Installer: def check_archive_dir(): - if g_opts.db_type not in ['0', '1', '2']: + if g_opts.db_type not in ["0", "1", "2"]: err_msg = "Invalid db_type : %s." % g_opts.db_type LOGGER.error(err_msg) raise Exception(err_msg) - if g_opts.db_type == '0' or g_opts.install_type == "reserve": + if g_opts.db_type == "0" or g_opts.install_type == "reserve": return if g_opts.node_id == 1: return flags = os.O_RDONLY modes = stat.S_IWUSR | stat.S_IRUSR - with os.fdopen(os.open(CANTIAN_START_STATUS_FILE, flags, modes), 'r') as load_fp: + with os.fdopen(os.open(CANTIAN_START_STATUS_FILE, flags, modes), "r") as load_fp: start_parameters = json.load(load_fp) - if start_parameters.setdefault('db_create_status', "default") == "done": + if start_parameters.setdefault("db_create_status", "default") == "done": return if DEPLOY_MODE == "dss": return @@ -3398,9 +4022,13 @@ def check_archive_dir(): if os.path.exists(archive_dir): files = os.listdir(archive_dir) for file in files: - if (file[-4:] == ".arc" and file[:4] == "arch") or ("arch_file.tmp" in file): - err_msg = "archive dir %s is not empty, history archive file or archive tmp file : %s." % ( - archive_dir, file) + if (file[-4:] == ".arc" and file[:4] == "arch") or ( + "arch_file.tmp" in file + ): + err_msg = ( + "archive dir %s is not empty, history archive file or archive tmp file : %s." + % (archive_dir, file) + ) LOGGER.error(err_msg) raise Exception(err_msg) else: @@ -3415,11 +4043,17 @@ def check_archive_dir(): if "the archive dir does not exist" in str(output): log("INFO: %s" % output.strip()) else: - err_msg = "Failed to execute command '%s', error: %s" % (arch_query_cmd, stderr) + err_msg = "Failed to execute command '%s', error: %s" % ( + arch_query_cmd, + stderr, + ) LOGGER.error(err_msg) raise Exception(err_msg) else: - if any("arch" in line and (".arc" in line or "arch_file.tmp" in line) for line in output.splitlines()): + if any( + "arch" in line and (".arc" in line or "arch_file.tmp" in line) + for line in output.splitlines() + ): err_msg = "Archive files found in dbstor: %s" % output LOGGER.error(err_msg) raise Exception(err_msg) @@ -3427,8 +4061,12 @@ def check_archive_dir(): class CanTian(object): - g_opts.os_user, g_opts.os_group = get_value("deploy_user"), get_value("deploy_group") - g_opts.install_type = get_value('install_type') if get_value('install_type') else "0" + g_opts.os_user, g_opts.os_group = get_value("deploy_user"), get_value( + "deploy_group" + ) + g_opts.install_type = ( + get_value("install_type") if get_value("install_type") else "0" + ) def cantian_pre_install(self): check_platform() @@ -3441,7 +4079,10 @@ class CanTian(object): try: installer = Installer(g_opts.os_user, g_opts.os_group) installer.install() - LOGGER.info("Install successfully, for more detail information see %s." % g_opts.log_file) + LOGGER.info( + "Install successfully, for more detail information see %s." + % g_opts.log_file + ) except Exception as error: LOGGER.error("Install failed: " + str(error)) raise Exception(str(error)) from error @@ -3450,9 +4091,11 @@ class CanTian(object): try: flags = os.O_RDWR | os.O_CREAT modes = stat.S_IWUSR | stat.S_IRUSR - with os.fdopen(os.open(CANTIAN_START_STATUS_FILE, flags, modes), 'w+') as load_fp: + with os.fdopen( + os.open(CANTIAN_START_STATUS_FILE, flags, modes), "w+" + ) as load_fp: start_parameters = json.load(load_fp) - start_status_item = {'start_status': "starting"} + start_status_item = {"start_status": "starting"} start_parameters.update(start_status_item) load_fp.seek(0) load_fp.truncate() @@ -3462,14 +4105,19 @@ class CanTian(object): check_archive_dir() installer = Installer(g_opts.os_user, g_opts.os_group) installer.install_start() - LOGGER.info("Start successfully, for more detail information see %s." % g_opts.log_file) + LOGGER.info( + "Start successfully, for more detail information see %s." + % g_opts.log_file + ) flags = os.O_RDWR | os.O_CREAT - with os.fdopen(os.open(CANTIAN_START_STATUS_FILE, flags, modes), 'w+') as load_fp: + with os.fdopen( + os.open(CANTIAN_START_STATUS_FILE, flags, modes), "w+" + ) as load_fp: start_parameters = json.load(load_fp) - start_status_item = {'start_status': "started"} + start_status_item = {"start_status": "started"} start_parameters.update(start_status_item) - ever_started_item = {'ever_started': True} + ever_started_item = {"ever_started": True} start_parameters.update(ever_started_item) load_fp.seek(0) load_fp.truncate() @@ -3477,35 +4125,57 @@ class CanTian(object): sep_mark = os.path.sep cmd = "pidof cantiand" - if g_opts.running_mode.lower() in [CANTIAND_WITH_MYSQL, CANTIAND_WITH_MYSQL_ST, - CANTIAND_WITH_MYSQL_IN_CLUSTER]: + if g_opts.running_mode.lower() in [ + CANTIAND_WITH_MYSQL, + CANTIAND_WITH_MYSQL_ST, + CANTIAND_WITH_MYSQL_IN_CLUSTER, + ]: cmd = "pidof mysqld" ret_code, cantiand_pid, stderr = _exec_popen(cmd) if ret_code: - LOGGER.error("can not get pid of cantiand or mysqld, command: %s, err: %s" % (cmd, stderr)) - raise Exception("can not get pid of cantiand or mysqld, command: %s, err: %s" % (cmd, stderr)) + LOGGER.error( + "can not get pid of cantiand or mysqld, command: %s, err: %s" + % (cmd, stderr) + ) + raise Exception( + "can not get pid of cantiand or mysqld, command: %s, err: %s" + % (cmd, stderr) + ) cantiand_pids = cantiand_pid.strip().split() if len(cantiand_pids) > 1: - error_message = (f"Detected multiple cantiand/mysqld processes ({', '.join(cantiand_pids)}). " - f"Please manually clean up the abnormal processes and retry.") + error_message = ( + f"Detected multiple cantiand/mysqld processes ({', '.join(cantiand_pids)}). " + f"Please manually clean up the abnormal processes and retry." + ) LOGGER.error(error_message) raise Exception(error_message) cantiand_pid = cantiand_pids[0] - coredump_filter_path = os.path.join(sep_mark, "proc", str(cantiand_pid), "coredump_filter") + coredump_filter_path = os.path.join( + sep_mark, "proc", str(cantiand_pid), "coredump_filter" + ) if cantiand_pid: cmd = f"echo 0x6f > {coredump_filter_path}" ret_code, _, stderr = _exec_popen(cmd) if ret_code: - LOGGER.error("can not set coredump_filter, command: %s, err: %s" % (cmd, stderr)) - raise Exception("can not set coredump_filter, command: %s, err: %s" % (cmd, stderr)) + LOGGER.error( + "can not set coredump_filter, command: %s, err: %s" + % (cmd, stderr) + ) + raise Exception( + "can not set coredump_filter, command: %s, err: %s" + % (cmd, stderr) + ) LOGGER.info("Set coredump_filter successfully") except Exception as error: LOGGER.info("Start failed: " + str(error)) - LOGGER.info("Please refer to install log \"%s\" for more detailed information." % g_opts.log_file) + LOGGER.info( + 'Please refer to install log "%s" for more detailed information.' + % g_opts.log_file + ) raise Exception(str(error)) from error def post_check(self): @@ -3514,9 +4184,9 @@ class CanTian(object): installer.decrypt_db_passwd() flags = os.O_RDONLY modes = stat.S_IWUSR | stat.S_IRUSR - with os.fdopen(os.open(JS_CONF_FILE, flags, modes), 'r') as fp: + with os.fdopen(os.open(JS_CONF_FILE, flags, modes), "r") as fp: json_data = json.load(fp) - installer.install_path = json_data['R_INSTALL_PATH'].strip() + installer.install_path = json_data["R_INSTALL_PATH"].strip() installer.check_db_status() LOGGER.info("Post upgrade check success.") diff --git a/pkg/deploy/action/cantian/cantian_post_upgrade.py b/pkg/deploy/action/cantian/cantian_post_upgrade.py index c2793ba9..5d044003 100644 --- a/pkg/deploy/action/cantian/cantian_post_upgrade.py +++ b/pkg/deploy/action/cantian/cantian_post_upgrade.py @@ -10,4 +10,4 @@ if __name__ == "__main__": exit(str(err)) except Exception as err: exit(str(err)) - exit(0) \ No newline at end of file + exit(0) diff --git a/pkg/deploy/action/cantian/cantian_start.py b/pkg/deploy/action/cantian/cantian_start.py index 5db575cc..de6b48ed 100644 --- a/pkg/deploy/action/cantian/cantian_start.py +++ b/pkg/deploy/action/cantian/cantian_start.py @@ -13,4 +13,4 @@ if __name__ == "__main__": exit(str(err)) except Exception as err: exit(str(err)) - exit(0) \ No newline at end of file + exit(0) diff --git a/pkg/deploy/action/cantian/cantian_uninstall.py b/pkg/deploy/action/cantian/cantian_uninstall.py index 87b04bf8..acad8fdd 100644 --- a/pkg/deploy/action/cantian/cantian_uninstall.py +++ b/pkg/deploy/action/cantian/cantian_uninstall.py @@ -42,9 +42,13 @@ PKG_DIR = os.path.abspath(os.path.join(INSTALL_SCPRIT_DIR, "../..")) JS_CONF_FILE = os.path.join(PKG_DIR, "action", "cantian", "install_config.json") UNINSTALL_PATH = os.path.join(PKG_DIR, "action", "cantian") -CANTIAN_UNINSTALL_CONF_FILE = os.path.join(PKG_DIR, "action", "cantian", "cantian_uninstall_config.json") +CANTIAN_UNINSTALL_CONF_FILE = os.path.join( + PKG_DIR, "action", "cantian", "cantian_uninstall_config.json" +) CANTIAN_CONF_FILE = os.path.join("/opt/cantian/cantian", "cfg", "cantian_config.json") -CANTIAN_START_STATUS_FILE = os.path.join("/opt/cantian/cantian", "cfg", "start_status.json") +CANTIAN_START_STATUS_FILE = os.path.join( + "/opt/cantian/cantian", "cfg", "start_status.json" +) CANTIAN_UNINSTALL_LOG_FILE = "/opt/cantian/log/cantian/cantian_deploy.log" CONFIG_PARAMS_FILE = os.path.join(PKG_DIR, "config", "deploy_param.json") FORCE_UNINSTALL = None @@ -52,7 +56,12 @@ CHECK_MAX_TIMES = 60 CANTIAND_WITH_MYSQL = "cantiand_with_mysql" CANTIAND_WITH_MYSQL_ST = "cantiand_with_mysql_st" CANTIAND_WITH_MYSQL_IN_CLUSTER = "cantiand_with_mysql_in_cluster" -VALID_SINGLE_MYSQL_RUNNING_MODE = {CANTIAND_WITH_MYSQL_IN_CLUSTER, CANTIAND_WITH_MYSQL_ST, CANTIAND_WITH_MYSQL} +VALID_SINGLE_MYSQL_RUNNING_MODE = { + CANTIAND_WITH_MYSQL_IN_CLUSTER, + CANTIAND_WITH_MYSQL_ST, + CANTIAND_WITH_MYSQL, +} + def _exec_popen(_cmd, values=None): """ @@ -63,8 +72,13 @@ def _exec_popen(_cmd, values=None): if not values: values = [] bash_cmd = ["bash"] - pobj = subprocess.Popen(bash_cmd, shell=False, stdin=subprocess.PIPE, - stdout=subprocess.PIPE, stderr=subprocess.PIPE) + pobj = subprocess.Popen( + bash_cmd, + shell=False, + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + ) if gPyVersion[0] == "3": pobj.stdin.write(_cmd.encode()) @@ -105,11 +119,11 @@ def cantian_check_share_logic_ip_isvalid(node_ip, deploy_mode): input : ip output: NA """ - + def ping_execute(p_cmd): cmd = "%s %s -i 1 -c 3 | grep ttl | wc -l" % (p_cmd, node_ip) ret_code, stdout, _ = _exec_popen(cmd) - if ret_code or stdout != '3': + if ret_code or stdout != "3": return False return True @@ -198,21 +212,27 @@ def parse_parameter(): flags = os.O_RDONLY modes = stat.S_IWUSR | stat.S_IRUSR - with os.fdopen(os.open(JS_CONF_FILE, flags, modes), 'r') as fp: + with os.fdopen(os.open(JS_CONF_FILE, flags, modes), "r") as fp: json_data = json.load(fp) g_opts.clean_data_dir_on = 0 # -F - g_opts.clean_data_dir = json_data['UNINSTALL_D_LOCATION_DATABASE_AREA'].strip() # -D + g_opts.clean_data_dir = json_data[ + "UNINSTALL_D_LOCATION_DATABASE_AREA" + ].strip() # -D if os.getuid() != 0: # -g - g_opts.install_user_privilege = json_data['UNINSTALL_g_RUN_UNINSTALL_SCRIPT'].strip() + g_opts.install_user_privilege = json_data[ + "UNINSTALL_g_RUN_UNINSTALL_SCRIPT" + ].strip() g_opts.use_gss = True # -s flags = os.O_RDONLY modes = stat.S_IWUSR | stat.S_IRUSR - with os.fdopen(os.open(CONFIG_PARAMS_FILE, flags, modes), 'r') as config_fp: + with os.fdopen(os.open(CONFIG_PARAMS_FILE, flags, modes), "r") as config_fp: json_data = json.load(config_fp) - g_opts.namespace = json_data.get('cluster_name', 'test1').strip() - g_opts.node_id = int(json_data.get('node_id')) - metadata_str = "metadata_" + json_data.get('storage_metadata_fs', '').strip() + g_opts.namespace = json_data.get("cluster_name", "test1").strip() + g_opts.node_id = int(json_data.get("node_id")) + metadata_str = ( + "metadata_" + json_data.get("storage_metadata_fs", "").strip() + ) node_str = "node" + str(g_opts.node_id) global MYSQL_DATA_DIR MYSQL_DATA_DIR = os.path.join("/mnt/dbdata/remote", metadata_str, node_str) @@ -252,10 +272,14 @@ def check_parameter(): if g_opts.install_user_privilege != "withoutroot": print_str = CommonPrint() - print_str.common_log("Error: User has no root privilege, " - "do uninstall, need specify parameter '-g withoutroot'.") - raise ValueError("Error: User has no root privilege, " - "do uninstall, need specify parameter '-g withoutroot'.") + print_str.common_log( + "Error: User has no root privilege, " + "do uninstall, need specify parameter '-g withoutroot'." + ) + raise ValueError( + "Error: User has no root privilege, " + "do uninstall, need specify parameter '-g withoutroot'." + ) else: print_str = CommonPrint() print_str.common_log("Error:Check os failed:current os is not linux") @@ -264,25 +288,28 @@ def check_parameter(): if g_opts.clean_data_dir_on == 1: if g_opts.clean_data_dir: print_str = CommonPrint() - print_str.common_log("Error: Parameter input error: " - "you can not use -D without using -F") - raise ValueError("Error: Parameter input error: " - "you can not use -D without using -F") + print_str.common_log( + "Error: Parameter input error: " "you can not use -D without using -F" + ) + raise ValueError( + "Error: Parameter input error: " "you can not use -D without using -F" + ) if g_opts.clean_data_dir: g_opts.clean_data_dir = os.path.realpath( - os.path.normpath(g_opts.clean_data_dir)) + os.path.normpath(g_opts.clean_data_dir) + ) DefaultValue.check_invalid_path(g_opts.clean_data_dir) def check_log_path(): flags = os.O_RDONLY modes = stat.S_IWUSR | stat.S_IRUSR - with os.fdopen(os.open(CANTIAN_UNINSTALL_CONF_FILE, flags, modes), 'r') as fp: + with os.fdopen(os.open(CANTIAN_UNINSTALL_CONF_FILE, flags, modes), "r") as fp: json_data = json.load(fp) - if json_data.get('LOG_FILE', '').strip() == "": + if json_data.get("LOG_FILE", "").strip() == "": g_opts.log_file = CANTIAN_UNINSTALL_LOG_FILE else: - g_opts.log_file = json_data.get('LOG_FILE', '').strip() + g_opts.log_file = json_data.get("LOG_FILE", "").strip() if not g_opts.log_file: g_opts.log_file = CANTIAN_UNINSTALL_LOG_FILE @@ -322,9 +349,9 @@ def get_install_path(): flags = os.O_RDONLY modes = stat.S_IWUSR | stat.S_IRUSR - with os.fdopen(os.open(JS_CONF_FILE, flags, modes), 'r') as fp: + with os.fdopen(os.open(JS_CONF_FILE, flags, modes), "r") as fp: json_data = json.load(fp) - g_opts.install_path_l = json_data['R_INSTALL_PATH'].strip() + g_opts.install_path_l = json_data["R_INSTALL_PATH"].strip() # Must be exist if not os.path.exists(g_opts.install_path_l): err_msg = "Failed to get install path." @@ -338,7 +365,7 @@ def get_deploy_user(): flags = os.O_RDONLY modes = stat.S_IWUSR | stat.S_IRUSR - with os.fdopen(os.open(CONFIG_PARAMS_FILE, flags, modes), 'r') as fp1: + with os.fdopen(os.open(CONFIG_PARAMS_FILE, flags, modes), "r") as fp1: json_data_deploy = json.load(fp1) deploy_user = get_value("deploy_user") @@ -354,7 +381,8 @@ def get_user_environment_file(): LOGGER.info("Getting user environment variables file path...") home_path = g_opts.user_info.pw_dir g_opts.user_env_path = os.path.realpath( - os.path.normpath(os.path.join(home_path, ".bashrc"))) + os.path.normpath(os.path.join(home_path, ".bashrc")) + ) if not os.path.isfile(os.path.realpath(g_opts.user_env_path)): err_msg = "Can't get the environment variables file." LOGGER.error(err_msg) @@ -371,7 +399,7 @@ def find_before_slice(slice_, str_): find '#' in the head of line """ place = str_.find(slice_) - return str_.find('#', 0, place) + return str_.find("#", 0, place) #################################################################### @@ -400,12 +428,12 @@ def check_environment_install_path(): line = tmp_f.readline() while line: # Obtain 'export CTDB_HOME' - if line.find('export CTDB_HOME') != -1: + if line.find("export CTDB_HOME") != -1: # Determine whether there is "#" before CTDB_HOME, the # function returns a value of -1, indicating that it is # not found, CTDB_HOME is valid. - if find_before_slice(line, 'CTDB_HOME') == -1: - install_env_dic_l = line.split('=') + if find_before_slice(line, "CTDB_HOME") == -1: + install_env_dic_l = line.split("=") install_env_temp_l = install_env_dic_l[1].rstrip() install_env_l = os.path.normpath(install_env_temp_l) install_env_l = os.path.realpath(install_env_l[1:-1]) @@ -415,11 +443,14 @@ def check_environment_install_path(): return 0 line = tmp_f.readline() tmp_f.close() - err_msg = "Check install path in user environment variables failed:\ - can not find install path in user: %s environment variables" % g_opts.user_info.pw_name + err_msg = ( + "Check install path in user environment variables failed:\ + can not find install path in user: %s environment variables" + % g_opts.user_info.pw_name + ) LOGGER.error(err_msg) if FORCE_UNINSTALL != "force": - raise Exception(err_msg) + raise Exception(err_msg) LOGGER.info("End check install path in user environment variables") @@ -455,8 +486,8 @@ def get_gsdata_path_env(): # Determine whether there is "#" before CTDB_DATA, the # function returns a value of -1, indicating that it is # not found, CTDB_DATA is valid. - if find_before_slice('export CTDB_DATA', line) == -1: - gsdata_path_dic_temp = line.split('=') + if find_before_slice("export CTDB_DATA", line) == -1: + gsdata_path_dic_temp = line.split("=") gsdata_path_temp = gsdata_path_dic_temp[1].rstrip() gsdata_path = os.path.normpath(gsdata_path_temp) g_opts.gs_data_path = os.path.realpath(gsdata_path[1:-1]) @@ -474,15 +505,16 @@ def get_gsdata_path_env(): return 0 # deal with the CTDB_HOME with """ # Obtain 'export CTDB_DATA' - elif line.find('export CTDB_DATA') != -1: + elif line.find("export CTDB_DATA") != -1: # Determine whether there is "#" before CTDB_DATA, the # function returns a value of -1, indicating that it is # not found, CTDB_DATA is valid. - if find_before_slice('export CTDB_DATA', line) == -1: - gsdata_path_dic_temp = line.split('=') + if find_before_slice("export CTDB_DATA", line) == -1: + gsdata_path_dic_temp = line.split("=") gsdata_path_temp = gsdata_path_dic_temp[1].rstrip() g_opts.gs_data_path = os.path.realpath( - os.path.normpath(gsdata_path_temp)) + os.path.normpath(gsdata_path_temp) + ) if not os.path.exists(g_opts.gs_data_path): f.close() err_msg = "Get data directory in user environment variables \ @@ -513,12 +545,14 @@ def check_data_dir(): """ LOGGER.info("Begin check data dir...") if g_opts.clean_data_dir: - if os.path.exists(g_opts.clean_data_dir) \ - and os.path.isdir(g_opts.clean_data_dir) \ - and g_opts.clean_data_dir == g_opts.gs_data_path: - LOGGER.info("path: \"%s\" is correct" % g_opts.clean_data_dir) + if ( + os.path.exists(g_opts.clean_data_dir) + and os.path.isdir(g_opts.clean_data_dir) + and g_opts.clean_data_dir == g_opts.gs_data_path + ): + LOGGER.info('path: "%s" is correct' % g_opts.clean_data_dir) else: - err_msg = "path: \"%s\" is incorrect" % g_opts.clean_data_dir + err_msg = 'path: "%s" is incorrect' % g_opts.clean_data_dir LOGGER.error(err_msg) if FORCE_UNINSTALL != "force": raise Exception(err_msg) @@ -537,11 +571,11 @@ def check_uninstall_pos(): output: NA """ LOGGER.info("Checking uninstall.py position...") - bin_path = g_opts.install_path_l + os.sep + 'bin' - addons_path = g_opts.install_path_l + os.sep + 'add-ons' - admin_path = g_opts.install_path_l + os.sep + 'admin' - lib_path = g_opts.install_path_l + os.sep + 'lib' - pkg_file = g_opts.install_path_l + os.sep + 'package.xml' + bin_path = g_opts.install_path_l + os.sep + "bin" + addons_path = g_opts.install_path_l + os.sep + "add-ons" + admin_path = g_opts.install_path_l + os.sep + "admin" + lib_path = g_opts.install_path_l + os.sep + "lib" + pkg_file = g_opts.install_path_l + os.sep + "package.xml" # Check if the install path exists if not os.path.exists(g_opts.install_path_l): @@ -607,11 +641,15 @@ def clean_install_path(): if os.path.exists(g_opts.install_path_l): shutil.rmtree(g_opts.install_path_l) except OSError as error: - LOGGER.error("Clean install path failed:can not delete install path " - "%s\nPlease manually delete it." % str(error)) + LOGGER.error( + "Clean install path failed:can not delete install path " + "%s\nPlease manually delete it." % str(error) + ) if FORCE_UNINSTALL != "force": - raise Exception("Clean install path failed:can not delete install path " - "%s\nPlease manually delete it." % str(error)) + raise Exception( + "Clean install path failed:can not delete install path " + "%s\nPlease manually delete it." % str(error) + ) LOGGER.info("Clean install path success") LOGGER.info("End clean Install path") @@ -620,6 +658,7 @@ def clean_install_path(): # Clear environment variables ########################################################################### + # Resolution path def gen_reg_string(text): """ @@ -633,7 +672,7 @@ def gen_reg_string(text): in_s_list = in_s_str.split(os.sep) reg_string = "" for i in in_s_list: - if (i == ""): + if i == "": continue else: reg_string += r"\/" + i @@ -653,12 +692,14 @@ def clean_environment(): # Clear environment variable CTDB_DATA data_cmd = r"/^\s*export\s*CTDB_DATA=\".*\"$/d" # Clear environment variable PATH about database - path_cmd = (r"/^\s*export\s*PATH=\"%s\/bin\":\$PATH$/d" - % gen_reg_string(g_opts.install_path_l)) + path_cmd = r"/^\s*export\s*PATH=\"%s\/bin\":\$PATH$/d" % gen_reg_string( + g_opts.install_path_l + ) # Clear environment variable LD_LIBRARY_PATH about database - lib_cmd = (r"/^\s*export\s*LD_LIBRARY_PATH=\"%s\/lib\":\"%s\/add-ons\".*$/d" - % (gen_reg_string(g_opts.install_path_l), - gen_reg_string(g_opts.install_path_l))) + lib_cmd = r"/^\s*export\s*LD_LIBRARY_PATH=\"%s\/lib\":\"%s\/add-ons\".*$/d" % ( + gen_reg_string(g_opts.install_path_l), + gen_reg_string(g_opts.install_path_l), + ) # Clear environment variable CTDB_HOME home_cmd = r"/^\s*export\s*CTDB_HOME=\".*\"$/d" # Clear environment variable CANTIANLOG @@ -670,13 +711,22 @@ def clean_environment(): mode_cmd = r"/^\s*export\s*CTSQL_SSL_MODE=.*$/d" cipher_cmd = r"/^\s*export\s*CTSQL_SSL_KEY_PASSWD=.*$/d" - cmds = [path_cmd, lib_cmd, home_cmd, cantianlog_cmd, - ca_cmd, cert_cmd, key_cmd, mode_cmd, cipher_cmd] + cmds = [ + path_cmd, + lib_cmd, + home_cmd, + cantianlog_cmd, + ca_cmd, + cert_cmd, + key_cmd, + mode_cmd, + cipher_cmd, + ] flags = os.O_RDONLY modes = stat.S_IWUSR | stat.S_IRUSR - with os.fdopen(os.open(JS_CONF_FILE, flags, modes), 'r') as fp: + with os.fdopen(os.open(JS_CONF_FILE, flags, modes), "r") as fp: json_data = json.load(fp) - g_opts.running_mode = json_data['M_RUNING_MODE'].strip() + g_opts.running_mode = json_data["M_RUNING_MODE"].strip() if g_opts.running_mode.lower() in VALID_SINGLE_MYSQL_RUNNING_MODE: mysql_bin_cmd = r"/^\s*export\s*MYSQL_BIN_DIR=\".*\"$/d" mysql_code_cmd = r"/^\s*export\s*MYSQL_CODE_DIR=\".*\"$/d" @@ -698,6 +748,7 @@ def clean_environment(): raise Exception(err_msg) LOGGER.info("End clean user environment variables...") + def read_ifile(ifile, keyword): if not os.path.isfile(ifile): LOGGER.error("The value of IFILE '{}' is not exists.".format(ifile)) @@ -705,7 +756,7 @@ def read_ifile(ifile, keyword): raise Exception("The value of IFILE '{}' is not exists.".format(ifile)) flags = os.O_RDONLY modes = stat.S_IWUSR | stat.S_IRUSR - with os.fdopen(os.open(ifile, flags, modes), 'r') as fp: + with os.fdopen(os.open(ifile, flags, modes), "r") as fp: for line in fp: items = line.split("=", 1) if len(items) == 2 and items[0].strip() == keyword: @@ -735,7 +786,7 @@ def read_cantiand_cfg(keyword): flags = os.O_RDONLY modes = stat.S_IWUSR | stat.S_IRUSR - with os.fdopen(os.open(cantiand_cfg_file, flags, modes), 'r') as fp: + with os.fdopen(os.open(cantiand_cfg_file, flags, modes), "r") as fp: for line in fp: items = line.split("=", 1) if len(items) != 2: @@ -757,20 +808,25 @@ def get_instance_id(): """ flags = os.O_RDONLY modes = stat.S_IWUSR | stat.S_IRUSR - _cmd = ("ps ux | grep -v grep | grep cantiand " - "| grep -w '\-D %s' |awk '{print $2}'") % g_opts.gs_data_path - with os.fdopen(os.open(CANTIAN_CONF_FILE, flags, modes), 'r') as fp: - json_data = json.load(fp) - g_opts.running_mode = json_data['RUNNING_MODE'].strip() - if g_opts.running_mode.lower() in [CANTIAND_WITH_MYSQL, CANTIAND_WITH_MYSQL_IN_CLUSTER, CANTIAND_WITH_MYSQL_ST]: - _cmd = ("ps ux | grep -v grep | grep mysqld |awk '{print $2}'") + _cmd = ( + "ps ux | grep -v grep | grep cantiand " "| grep -w '\-D %s' |awk '{print $2}'" + ) % g_opts.gs_data_path + with os.fdopen(os.open(CANTIAN_CONF_FILE, flags, modes), "r") as fp: + json_data = json.load(fp) + g_opts.running_mode = json_data["RUNNING_MODE"].strip() + if g_opts.running_mode.lower() in [ + CANTIAND_WITH_MYSQL, + CANTIAND_WITH_MYSQL_IN_CLUSTER, + CANTIAND_WITH_MYSQL_ST, + ]: + _cmd = "ps ux | grep -v grep | grep mysqld |awk '{print $2}'" status, output, _ = _exec_popen(_cmd) if status: - LOGGER.error("Failed to execute cmd: %s. Error:%s." % (str(_cmd), - str(output))) + LOGGER.error("Failed to execute cmd: %s. Error:%s." % (str(_cmd), str(output))) if FORCE_UNINSTALL != "force": - raise Exception("Failed to execute cmd: %s. Error:%s." % (str(_cmd), - str(output))) + raise Exception( + "Failed to execute cmd: %s. Error:%s." % (str(_cmd), str(output)) + ) # process exists return output @@ -782,18 +838,26 @@ def get_error_id(error_name): output: NA """ if error_name == "installdb.sh": - _cmd = ("ps ux | grep -v grep | grep installdb.sh | grep P | grep cantiand " - "| awk '{print $2}'") + _cmd = ( + "ps ux | grep -v grep | grep installdb.sh | grep P | grep cantiand " + "| awk '{print $2}'" + ) elif error_name == "cantian_start.py": - _cmd = ("ps ux | grep -v grep | grep python | grep cantian_start.py " - "| awk '{print $2}'") + _cmd = ( + "ps ux | grep -v grep | grep python | grep cantian_start.py " + "| awk '{print $2}'" + ) status, output, error = _exec_popen(_cmd) if status: - LOGGER.error("Failed to execute cmd: %s. Output:%s. Error:%s" % (str(_cmd), - str(output), str(error))) + LOGGER.error( + "Failed to execute cmd: %s. Output:%s. Error:%s" + % (str(_cmd), str(output), str(error)) + ) if FORCE_UNINSTALL != "force": - raise Exception("Failed to execute cmd: %s. Output:%s. Error:%s" % (str(_cmd), - str(output), str(error))) + raise Exception( + "Failed to execute cmd: %s. Output:%s. Error:%s" + % (str(_cmd), str(output), str(error)) + ) # process exists return output @@ -808,11 +872,17 @@ def kill_instance(instance_pid): LOGGER.info("kill process cmd: %s" % kill_cmd_tmp) ret_code_1, stdout, stderr = _exec_popen(kill_cmd_tmp) if ret_code_1 and "No such process" not in stderr: - LOGGER.error("kill process %s failed." - "ret_code : %s, stdout : %s, stderr : %s" % (instance_pid, ret_code_1, stdout, stderr)) + LOGGER.error( + "kill process %s failed." + "ret_code : %s, stdout : %s, stderr : %s" + % (instance_pid, ret_code_1, stdout, stderr) + ) if FORCE_UNINSTALL != "force": - raise Exception("kill process %s failed." - "ret_code : %s, stdout : %s, stderr : %s" % (instance_pid, ret_code_1, stdout, stderr)) + raise Exception( + "kill process %s failed." + "ret_code : %s, stdout : %s, stderr : %s" + % (instance_pid, ret_code_1, stdout, stderr) + ) check_process_status("cantiand") LOGGER.info("Kill cantiand instance succeed") @@ -832,9 +902,15 @@ def kill_extra_process(): LOGGER.info("kill process cmd: %s" % kill_cmd_tmp) ret_code, output, error = _exec_popen(kill_cmd_tmp) if ret_code and "No such process" not in error: - LOGGER.error("kill extra process failed. Output:%s. Error:%s." % (str(output), str(error))) + LOGGER.error( + "kill extra process failed. Output:%s. Error:%s." + % (str(output), str(error)) + ) if FORCE_UNINSTALL != "force": - raise Exception("kill extra process failed. Output:%s. Error:%s." % (str(output), str(error))) + raise Exception( + "kill extra process failed. Output:%s. Error:%s." + % (str(output), str(error)) + ) check_process_status(extra_process) LOGGER.info("Kill %s succeed" % extra_process) @@ -865,18 +941,28 @@ def check_process_status(process_name): def kill_process(process_name): - kill_cmd_1 = (r"proc_pid_list=`ps ux | grep %s | grep -v grep" - r"|awk '{print $2}'` && " % process_name) - kill_cmd_1 += (r"(if [ X\"$proc_pid_list\" != X\"\" ];then echo " - r"$proc_pid_list | xargs kill -9; exit 0; fi)") + kill_cmd_1 = ( + r"proc_pid_list=`ps ux | grep %s | grep -v grep" + r"|awk '{print $2}'` && " % process_name + ) + kill_cmd_1 += ( + r"(if [ X\"$proc_pid_list\" != X\"\" ];then echo " + r"$proc_pid_list | xargs kill -9; exit 0; fi)" + ) LOGGER.info("kill process cmd: %s" % kill_cmd_1) ret_code_2, stdout, stderr = _exec_popen(kill_cmd_1) if ret_code_2: - LOGGER.error("kill process %s faild." - "ret_code : %s, stdout : %s, stderr : %s" % (process_name, ret_code_2, stdout, stderr)) + LOGGER.error( + "kill process %s faild." + "ret_code : %s, stdout : %s, stderr : %s" + % (process_name, ret_code_2, stdout, stderr) + ) if FORCE_UNINSTALL != "force": - raise Exception("kill process %s faild." - "ret_code : %s, stdout : %s, stderr : %s" % (process_name, ret_code_2, stdout, stderr)) + raise Exception( + "kill process %s faild." + "ret_code : %s, stdout : %s, stderr : %s" + % (process_name, ret_code_2, stdout, stderr) + ) def stop_instance(): @@ -902,7 +988,7 @@ def stop_instance(): LOGGER.error(err_msg) if FORCE_UNINSTALL != "force": raise Exception(err_msg) - host_ip = lsnr_addr.split(',')[0] + host_ip = lsnr_addr.split(",")[0] # if the cantian process not exists, and disable sysdba user # tell user the user name and password input interactive are @@ -910,11 +996,14 @@ def stop_instance(): instance_pid = get_instance_id() # specify -P parameter, db password is supported if not instance_pid and g_opts.db_passwd: - log("Notice: Instance '%s' has been stopped." % - g_opts.gs_data_path) - log(("Notice: The Database username and password" - " that are interactive entered " - "will not be verified correct and used.")) + log("Notice: Instance '%s' has been stopped." % g_opts.gs_data_path) + log( + ( + "Notice: The Database username and password" + " that are interactive entered " + "will not be verified correct and used." + ) + ) kill_process("mysqld") if g_opts.use_gss: kill_process("gssd") @@ -936,24 +1025,34 @@ def stop_instance(): install_path_l = g_opts.install_path_l gs_data_path = g_opts.gs_data_path tmp_cmd = "%s/bin/shutdowndb.sh -h %s -p %s -w -m immediate -D %s" % ( - install_path_l, host_ip, lsnr_port, gs_data_path) + install_path_l, + host_ip, + lsnr_port, + gs_data_path, + ) else: - tmp_cmd = ("echo '%s' | %s/bin/shutdowndb.sh" - " -h %s -p %s -U %s -m immediate -W -D %s") % ( - g_opts.db_passwd, - g_opts.install_path_l, - host_ip, - lsnr_port, - g_opts.db_user, - g_opts.gs_data_path) + tmp_cmd = ( + "echo '%s' | %s/bin/shutdowndb.sh" + " -h %s -p %s -U %s -m immediate -W -D %s" + ) % ( + g_opts.db_passwd, + g_opts.install_path_l, + host_ip, + lsnr_port, + g_opts.db_user, + g_opts.gs_data_path, + ) return_code_3, stdout_2, stderr_2 = _exec_popen(tmp_cmd) if return_code_3: g_opts.db_passwd = "" stdout_2 = get_error_msg(stdout_2, stderr_2) if (not g_opts.db_passwd) and stdout_2.find( - "login as sysdba is prohibited") >= 0: - stdout_2 += ("\nsysdba login is disabled, please specify -P " - "parameter to input password, refer to --help.") + "login as sysdba is prohibited" + ) >= 0: + stdout_2 += ( + "\nsysdba login is disabled, please specify -P " + "parameter to input password, refer to --help." + ) err_msg = "stop cantian instance failed. Error: %s" % stdout_2 LOGGER.error(err_msg) @@ -980,20 +1079,20 @@ def get_error_msg(outmsg, errmsg): def clean_archive_dir(json_data_deploy): - db_type = json_data_deploy.get('db_type', '') - deploy_mode = json_data_deploy.get('deploy_mode', '') - archive_fs = json_data_deploy.get('storage_archive_fs', '') + db_type = json_data_deploy.get("db_type", "") + deploy_mode = json_data_deploy.get("deploy_mode", "") + archive_fs = json_data_deploy.get("storage_archive_fs", "") uninstall_type = sys.argv[1] - if db_type == '' or db_type == '0' or uninstall_type == "reserve": + if db_type == "" or db_type == "0" or uninstall_type == "reserve": return - node_id = json_data_deploy.get('node_id', '').strip() + node_id = json_data_deploy.get("node_id", "").strip() if node_id == "": err_msg = "node_id is not found." LOGGER.error(err_msg) if FORCE_UNINSTALL != "force": raise Exception(err_msg) - archive_logic_ip = json_data_deploy.get('archive_logic_ip', '').strip() - storage_archive_fs = json_data_deploy.get('storage_archive_fs', '').strip() + archive_logic_ip = json_data_deploy.get("archive_logic_ip", "").strip() + storage_archive_fs = json_data_deploy.get("storage_archive_fs", "").strip() archive_dir = os.path.join("/mnt/dbdata/remote", "archive_" + storage_archive_fs) cantian_check_share_logic_ip_isvalid(archive_logic_ip, deploy_mode) @@ -1001,26 +1100,38 @@ def clean_archive_dir(json_data_deploy): cmd = "timeout 10 ls %s" % archive_dir ret_code, _, stderr = _exec_popen(cmd) if node_id == "0" and (ret_code == 0 or FORCE_UNINSTALL != "force"): - cmd_str = "rm -rf %s/arch*.arc %s/*arch_file.tmp" % (archive_dir, archive_dir) + cmd_str = "rm -rf %s/arch*.arc %s/*arch_file.tmp" % ( + archive_dir, + archive_dir, + ) ret_code, _, stderr = _exec_popen(cmd_str) if ret_code: LOGGER.error( - "can not clean the archive dir %s, command: %s, output: %s" % (archive_dir, cmd_str, stderr)) + "can not clean the archive dir %s, command: %s, output: %s" + % (archive_dir, cmd_str, stderr) + ) if FORCE_UNINSTALL != "force": raise Exception( - "can not clean the archive dir %s, command: %s, output: %s" % (archive_dir, cmd_str, stderr)) + "can not clean the archive dir %s, command: %s, output: %s" + % (archive_dir, cmd_str, stderr) + ) LOGGER.info("cleaned archive files.") else: - arch_clean_cmd = f"dbstor --delete-file --fs-name={archive_fs} --file-name=archive" + arch_clean_cmd = ( + f"dbstor --delete-file --fs-name={archive_fs} --file-name=archive" + ) try: ret_code, output, stderr = _exec_popen(arch_clean_cmd) if ret_code: - LOGGER.info(f"Failed to execute command '{arch_clean_cmd}', error: {stderr}") + LOGGER.info( + f"Failed to execute command '{arch_clean_cmd}', error: {stderr}" + ) else: LOGGER.info("Cleaned archive files using dbstor --delete-file.") except Exception as e: - LOGGER.info(f"Exception occurred while executing command '{arch_clean_cmd}': {str(e)}") - + LOGGER.info( + f"Exception occurred while executing command '{arch_clean_cmd}': {str(e)}" + ) class CanTian(object): @@ -1062,22 +1173,29 @@ class CanTian(object): flags = os.O_WRONLY | os.O_TRUNC modes = stat.S_IWUSR | stat.S_IRUSR | stat.S_IXOTH | stat.S_IWOTH - with os.fdopen(os.open(CANTIAN_UNINSTALL_CONF_FILE, flags, modes), 'w') as fp: + with os.fdopen( + os.open(CANTIAN_UNINSTALL_CONF_FILE, flags, modes), "w" + ) as fp: json.dump(cantian_uninstall_config_data, fp) LOGGER.info("Uninstall cantiand finish ") flags = os.O_RDWR | os.O_CREAT modes = stat.S_IWUSR | stat.S_IRUSR - with os.fdopen(os.open(CANTIAN_START_STATUS_FILE, flags, modes), 'w+') as load_fp: + with os.fdopen( + os.open(CANTIAN_START_STATUS_FILE, flags, modes), "w+" + ) as load_fp: start_parameters = json.load(load_fp) - start_status_item = {'start_status': "default"} + start_status_item = {"start_status": "default"} start_parameters.update(start_status_item) load_fp.seek(0) load_fp.truncate() json.dump(start_parameters, load_fp) except Exception as error: LOGGER.info("Stop failed: " + str(error)) - LOGGER.info("Please refer to uninstall log \"%s\" for more detailed information." % g_opts.log_file) + LOGGER.info( + 'Please refer to uninstall log "%s" for more detailed information.' + % g_opts.log_file + ) raise ValueError(str(error)) from error def cantian_uninstall(self): @@ -1088,16 +1206,16 @@ class CanTian(object): flags = os.O_RDONLY modes = stat.S_IWUSR | stat.S_IRUSR - with os.fdopen(os.open(CANTIAN_UNINSTALL_CONF_FILE, flags, modes), 'r') as fp: + with os.fdopen(os.open(CANTIAN_UNINSTALL_CONF_FILE, flags, modes), "r") as fp: json_data = json.load(fp) - g_opts.clean_data_dir_on = json_data.get('CLEAN_DATA_DIR_ON', '') - if json_data.get('USER_ENV_PATH', '').strip() == "": + g_opts.clean_data_dir_on = json_data.get("CLEAN_DATA_DIR_ON", "") + if json_data.get("USER_ENV_PATH", "").strip() == "": g_opts.user_env_path = os.path.join("/home", user, ".bashrc") else: - g_opts.user_env_path = json_data.get('USER_ENV_PATH', '').strip() - g_opts.install_path_l = json_data.get('INSTALL_PATH_L', '').strip() - stat.S_IRUSR = json_data.get('S_IRUSR', '') + g_opts.user_env_path = json_data.get("USER_ENV_PATH", "").strip() + g_opts.install_path_l = json_data.get("INSTALL_PATH_L", "").strip() + stat.S_IRUSR = json_data.get("S_IRUSR", "") g_opts.gs_data_path = "/mnt/dbdata/local/cantian/tmp/data" @@ -1109,15 +1227,19 @@ class CanTian(object): clean_archive_dir(json_data_deploy) LOGGER.info("uninstall step 4") - start_parameters = {'start_status': 'default', 'db_create_status': 'default'} + start_parameters = {"start_status": "default", "db_create_status": "default"} flags = os.O_WRONLY | os.O_TRUNC | os.O_CREAT - with os.fdopen(os.open(CANTIAN_START_STATUS_FILE, flags, modes), 'w') as load_fp: + with os.fdopen( + os.open(CANTIAN_START_STATUS_FILE, flags, modes), "w" + ) as load_fp: json.dump(start_parameters, load_fp) LOGGER.info("uninstall step 5") clean_install_path() LOGGER.info("uninstall step 6") - log("Cantiand was successfully removed from your computer, " - "for more message please see %s." % g_opts.log_file) + log( + "Cantiand was successfully removed from your computer, " + "for more message please see %s." % g_opts.log_file + ) os.chmod(g_opts.log_file, stat.S_IWUSR + stat.S_IRUSR + stat.S_IRGRP) @@ -1125,13 +1247,19 @@ class CanTian(object): g_opts.tmp_fp.flush() g_opts.tmp_fp.close() - ret_code, cantiand_pid, stderr = _exec_popen('exit') + ret_code, cantiand_pid, stderr = _exec_popen("exit") if ret_code: - LOGGER.error("can not logout, command: exit" - " ret_code : %s, stdout : %s, stderr : %s" % (ret_code, cantiand_pid, stderr)) + LOGGER.error( + "can not logout, command: exit" + " ret_code : %s, stdout : %s, stderr : %s" + % (ret_code, cantiand_pid, stderr) + ) if FORCE_UNINSTALL != "force": - raise Exception("can not logout, command: exit" - " ret_code : %s, stdout : %s, stderr : %s" % (ret_code, cantiand_pid, stderr)) + raise Exception( + "can not logout, command: exit" + " ret_code : %s, stdout : %s, stderr : %s" + % (ret_code, cantiand_pid, stderr) + ) def cantian_check_status(self): g_opts.gs_data_path = "/mnt/dbdata/local/cantian/tmp/data" diff --git a/pkg/deploy/action/cantian/exception.py b/pkg/deploy/action/cantian/exception.py index c7691ab3..1021e1ea 100644 --- a/pkg/deploy/action/cantian/exception.py +++ b/pkg/deploy/action/cantian/exception.py @@ -1,2 +1,2 @@ class NormalException(Exception): - pass \ No newline at end of file + pass diff --git a/pkg/deploy/action/cantian/get_config_info.py b/pkg/deploy/action/cantian/get_config_info.py index 1f836d9b..e843cbd5 100644 --- a/pkg/deploy/action/cantian/get_config_info.py +++ b/pkg/deploy/action/cantian/get_config_info.py @@ -8,19 +8,46 @@ INSTALL_SCPRIT_DIR = os.path.dirname(os.path.abspath(__file__)) PKG_DIR = os.path.abspath(os.path.join(INSTALL_SCPRIT_DIR, "../..")) CONFIG_PARAMS_FILE = os.path.join(PKG_DIR, "config", "deploy_param.json") -CANTIAN_CONFIG_PARAMS_FILE = os.path.join(PKG_DIR, "action", "cantian", "cantian_config.json") -CANTIAN_CONFIG_PARAMS_FILE_BACKUP = "/opt/cantian/backup/files/cantian/cantian_config.json" +CANTIAN_CONFIG_PARAMS_FILE = os.path.join( + PKG_DIR, "action", "cantian", "cantian_config.json" +) +CANTIAN_CONFIG_PARAMS_FILE_BACKUP = ( + "/opt/cantian/backup/files/cantian/cantian_config.json" +) NUMA_CONFIG_FILE = "/opt/cantian/cantian/cfg/cpu_config.json" -CANTIAN_START_STATUS_FILE = os.path.join("/opt/cantian/cantian", "cfg", "start_status.json") -CANTIAN_START_CONFIG_FILE = os.path.join(PKG_DIR, "config", "container_conf", "init_conf", "start_config.json") -CANTIAN_MEM_SPEC_FILE = os.path.join(PKG_DIR, "config", "container_conf", "init_conf", "mem_spec") +CANTIAN_START_STATUS_FILE = os.path.join( + "/opt/cantian/cantian", "cfg", "start_status.json" +) +CANTIAN_START_CONFIG_FILE = os.path.join( + PKG_DIR, "config", "container_conf", "init_conf", "start_config.json" +) +CANTIAN_MEM_SPEC_FILE = os.path.join( + PKG_DIR, "config", "container_conf", "init_conf", "mem_spec" +) ENV_FILE = os.path.join(PKG_DIR, "action", "env.sh") info = {} -kernel_params_list = ['SHM_CPU_GROUP_INFO', 'LARGE_POOL_SIZE', 'CR_POOL_COUNT', 'CR_POOL_SIZE', - 'TEMP_POOL_NUM', 'BUF_POOL_NUM', 'LOG_BUFFER_SIZE', 'LOG_BUFFER_COUNT', - 'SHARED_POOL_SIZE', 'DATA_BUFFER_SIZE', 'TEMP_BUFFER_SIZE', 'SESSIONS', - 'SHM_MEMORY_REDUCTION_RATIO', "VARIANT_MEMORY_AREA_SIZE", "DTC_RCY_PARAL_BUF_LIST_SIZE"] -mysql_kernel_params_list = ['max_connections', 'table_open_cache', 'table_open_cache_instances'] +kernel_params_list = [ + "SHM_CPU_GROUP_INFO", + "LARGE_POOL_SIZE", + "CR_POOL_COUNT", + "CR_POOL_SIZE", + "TEMP_POOL_NUM", + "BUF_POOL_NUM", + "LOG_BUFFER_SIZE", + "LOG_BUFFER_COUNT", + "SHARED_POOL_SIZE", + "DATA_BUFFER_SIZE", + "TEMP_BUFFER_SIZE", + "SESSIONS", + "SHM_MEMORY_REDUCTION_RATIO", + "VARIANT_MEMORY_AREA_SIZE", + "DTC_RCY_PARAL_BUF_LIST_SIZE", +] +mysql_kernel_params_list = [ + "max_connections", + "table_open_cache", + "table_open_cache_instances", +] numa_params_list = ["CANTIAN_NUMA_CPU_INFO", "MYSQL_NUMA_CPU_INFO"] MEM_SPEC = { "0": { @@ -35,7 +62,7 @@ MEM_SPEC = { "CR_POOL_SIZE": "256M", "max_connections": "128", "table_open_cache": "5120", - "table_open_cache_instances": "4" + "table_open_cache_instances": "4", }, "1": { "SESSIONS": "1024", @@ -49,7 +76,7 @@ MEM_SPEC = { "CR_POOL_SIZE": "512M", "max_connections": "1024", "table_open_cache": "10240", - "table_open_cache_instances": "8" + "table_open_cache_instances": "8", }, "2": { "SESSIONS": "2048", @@ -63,7 +90,7 @@ MEM_SPEC = { "CR_POOL_SIZE": "1G", "max_connections": "2048", "table_open_cache": "20480", - "table_open_cache_instances": "16" + "table_open_cache_instances": "16", }, "3": { "SESSIONS": "4096", @@ -77,8 +104,8 @@ MEM_SPEC = { "CR_POOL_SIZE": "2G", "max_connections": "4096", "table_open_cache": "40960", - "table_open_cache_instances": "32" - } + "table_open_cache_instances": "32", + }, } @@ -116,13 +143,13 @@ with open(ENV_FILE, "r", encoding="utf-8") as f: def get_value(param): if param == "mysql_user": - return info.get('deploy_user').split(':')[0] + return info.get("deploy_user").split(":")[0] if param == "mysql_group": - return info.get('deploy_user').split(':')[1] - if param == 'cantian_in_container': - return info.get('cantian_in_container', '0') - if param == 'SYS_PASSWORD': - return info_cantian.get('SYS_PASSWORD', "") + return info.get("deploy_user").split(":")[1] + if param == "cantian_in_container": + return info.get("cantian_in_container", "0") + if param == "SYS_PASSWORD": + return info_cantian.get("SYS_PASSWORD", "") if param == "deploy_user": for line in env_config: if line.startswith("cantian_user"): @@ -131,12 +158,12 @@ def get_value(param): for line in env_config: if line.startswith("cantian_group"): return line.split("=")[1].strip("\n").strip('"') - if param == 'CANTIAN_START_STATUS': - return info_cantian_start.get('start_status', "") - if param == 'CANTIAN_DB_CREATE_STATUS': - return info_cantian_start.get('db_create_status', "") - if param == 'CANTIAN_EVER_START': - return info_cantian_start.get('ever_started', "") + if param == "CANTIAN_START_STATUS": + return info_cantian_start.get("start_status", "") + if param == "CANTIAN_DB_CREATE_STATUS": + return info_cantian_start.get("db_create_status", "") + if param == "CANTIAN_EVER_START": + return info_cantian_start.get("ever_started", "") if param in numa_params_list: return numa_config.get(param, "") if param in kernel_params_list: @@ -144,9 +171,11 @@ def get_value(param): if param in mysql_kernel_params_list: return info_cantian_config.get(param, "") if param == "deploy_mode": - if info.get('deploy_mode', ""): - return info.get('deploy_mode') - return "dbstor" if info.get('deploy_policy', "") in ["ModeB", "ModeC"] else "file" + if info.get("deploy_mode", ""): + return info.get("deploy_mode") + return ( + "dbstor" if info.get("deploy_policy", "") in ["ModeB", "ModeC"] else "file" + ) return info.get(param, "") @@ -155,4 +184,3 @@ if __name__ == "__main__": _param = sys.argv[1] res = get_value(_param) print(res) - diff --git a/pkg/deploy/action/cantian/log.py b/pkg/deploy/action/cantian/log.py index c6de9e4d..f67da3c5 100644 --- a/pkg/deploy/action/cantian/log.py +++ b/pkg/deploy/action/cantian/log.py @@ -23,9 +23,9 @@ CONSOLE_CONF = { "log_file_backup_count": 5, "log_date_format": "%Y-%m-%d %H:%M:%S", "logging_default_format_string": "%(asctime)s %(levelname)s [pid:%(process)d] [%(threadName)s] " - "[tid:%(thread)d] [%(filename)s:%(lineno)d %(funcName)s] %(message)s", + "[tid:%(thread)d] [%(filename)s:%(lineno)d %(funcName)s] %(message)s", "logging_context_format_string": "%(asctime)s %(levelname)s [pid:%(process)d] [%(threadName)s] " - "[tid:%(thread)d] [%(filename)s:%(lineno)d %(funcName)s] %(message)s" + "[tid:%(thread)d] [%(filename)s:%(lineno)d %(funcName)s] %(message)s", } } @@ -48,14 +48,30 @@ def _get_log_file_path(project): os.makedirs(logger_dir) return os.path.join(logger_dir, "{}.log".format(project)) - return '' + return "" SENSITIVE_STR = [ - 'Password', 'passWord', 'PASSWORD', 'password', 'Pswd', - 'PSWD', 'pwd', 'signature', 'HmacSHA256', 'newPasswd', - 'private', 'certfile', 'secret', 'token', 'Token', 'pswd', - 'passwd', 'mysql -u', 'session', 'cookie' + "Password", + "passWord", + "PASSWORD", + "password", + "Pswd", + "PSWD", + "pwd", + "signature", + "HmacSHA256", + "newPasswd", + "private", + "certfile", + "secret", + "token", + "Token", + "pswd", + "passwd", + "mysql -u", + "session", + "cookie", ] @@ -83,7 +99,7 @@ def setup(project_name): stream_handler.setFormatter( logging.Formatter( fmt=log_config.get("logging_context_format_string"), - datefmt=log_config.get("log_date_format") + datefmt=log_config.get("log_date_format"), ) ) log_root.addHandler(stream_handler) @@ -91,12 +107,14 @@ def setup(project_name): log_path = _get_log_file_path(project_name) if log_path: file_log = handlers.RotatingFileHandler( - log_path, maxBytes=log_config.get("log_file_max_size"), - backupCount=log_config.get("log_file_backup_count")) + log_path, + maxBytes=log_config.get("log_file_max_size"), + backupCount=log_config.get("log_file_backup_count"), + ) file_log.setFormatter( logging.Formatter( fmt=log_config.get("logging_context_format_string"), - datefmt=log_config.get("log_date_format") + datefmt=log_config.get("log_date_format"), ) ) log_root.addHandler(file_log) @@ -113,4 +131,4 @@ def setup(project_name): LOGGER = setup(os.path.basename(LOG_FILE).split(".")[0]) log_directory = log_config.get("log_dir") os.chmod(log_directory, 0o750) -os.chmod(f'{str(Path(log_directory, os.path.basename(LOG_FILE)))}', 0o640) +os.chmod(f"{str(Path(log_directory, os.path.basename(LOG_FILE)))}", 0o640) diff --git a/pkg/deploy/action/cantian/options.py b/pkg/deploy/action/cantian/options.py index 66344a59..82daabad 100644 --- a/pkg/deploy/action/cantian/options.py +++ b/pkg/deploy/action/cantian/options.py @@ -73,4 +73,4 @@ class Options(object): self.cert_encrypt_pwd = "" - self.storage_dbstor_fs = "" \ No newline at end of file + self.storage_dbstor_fs = "" diff --git a/pkg/deploy/action/cantian_common/crypte_adapter.py b/pkg/deploy/action/cantian_common/crypte_adapter.py index f4c5858a..3deabebd 100644 --- a/pkg/deploy/action/cantian_common/crypte_adapter.py +++ b/pkg/deploy/action/cantian_common/crypte_adapter.py @@ -31,12 +31,18 @@ class KmcResolve(object): if mode == "decrypted": ret_pwd = kmc_adapter.decrypt(plain_text) except Exception as error: - raise Exception("Failed to %s password of user [sys]. Error: %s" % (mode, str(error))) from error + raise Exception( + "Failed to %s password of user [sys]. Error: %s" % (mode, str(error)) + ) from error finally: kmc_adapter.finalize() - split_env = os.environ['LD_LIBRARY_PATH'].split(":") - filtered_env = [single_env for single_env in split_env if "/opt/cantian/dbstor/lib" not in single_env] - os.environ['LD_LIBRARY_PATH'] = ":".join(filtered_env) + split_env = os.environ["LD_LIBRARY_PATH"].split(":") + filtered_env = [ + single_env + for single_env in split_env + if "/opt/cantian/dbstor/lib" not in single_env + ] + os.environ["LD_LIBRARY_PATH"] = ":".join(filtered_env) return ret_pwd def encrypted(self, pwd): diff --git a/pkg/deploy/action/cantian_common/exec_sql.py b/pkg/deploy/action/cantian_common/exec_sql.py index f9b306b3..d5b2b221 100644 --- a/pkg/deploy/action/cantian_common/exec_sql.py +++ b/pkg/deploy/action/cantian_common/exec_sql.py @@ -2,17 +2,18 @@ import subprocess import sys import os import signal + CURRENT_PATH = os.path.dirname(os.path.abspath(__file__)) sys.path.append(os.path.join(CURRENT_PATH, "../")) from cantian_common.crypte_adapter import KmcResolve -ZSQL_INI_PATH = '/mnt/dbdata/local/cantian/tmp/data/cfg/ctsql.ini' +ZSQL_INI_PATH = "/mnt/dbdata/local/cantian/tmp/data/cfg/ctsql.ini" TIME_OUT = 5 def file_reader(file_path): - with open(file_path, 'r') as file: + with open(file_path, "r") as file: return file.read() @@ -21,11 +22,11 @@ def close_child_process(proc): os.killpg(proc.pid, signal.SIGKILL) except ProcessLookupError as err: _ = err - return 'success' + return "success" except Exception as err: return str(err) - return 'success' + return "success" def exec_popen(cmd, timeout=TIME_OUT): @@ -35,8 +36,14 @@ def exec_popen(cmd, timeout=TIME_OUT): return: status code, standard output, error output """ bash_cmd = ["bash"] - pobj = subprocess.Popen(bash_cmd, shell=False, stdin=subprocess.PIPE, - stdout=subprocess.PIPE, stderr=subprocess.PIPE, preexec_fn=os.setsid) + pobj = subprocess.Popen( + bash_cmd, + shell=False, + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + preexec_fn=os.setsid, + ) pobj.stdin.write(cmd.encode()) pobj.stdin.write(os.linesep.encode()) try: @@ -62,27 +69,30 @@ class ExecSQL(object): def decrypted(self): ctsql_ini_data = file_reader(ZSQL_INI_PATH) - encrypt_pwd = ctsql_ini_data[ctsql_ini_data.find('=') + 1:].strip() + encrypt_pwd = ctsql_ini_data[ctsql_ini_data.find("=") + 1 :].strip() ctsql_passwd = KmcResolve.kmc_resolve_password("decrypted", encrypt_pwd) return ctsql_passwd def execute(self): ctsql_passwd = self.decrypted() - sql = ("source ~/.bashrc && echo -e \"%s\" | " - "ctsql sys@127.0.0.1:1611 -q -c \"%s\"") % (ctsql_passwd, self.sql) + sql = ( + 'source ~/.bashrc && echo -e "%s" | ' 'ctsql sys@127.0.0.1:1611 -q -c "%s"' + ) % (ctsql_passwd, self.sql) return_code, stdout, stderr = exec_popen(sql) if return_code: output = stdout + stderr - err_msg = "Exec [%s] failed, details: %s" % (self.sql, output.replace(ctsql_passwd, "***")) + err_msg = "Exec [%s] failed, details: %s" % ( + self.sql, + output.replace(ctsql_passwd, "***"), + ) raise Exception(err_msg) return stdout -if __name__ == '__main__': +if __name__ == "__main__": _sql_cmd = input() exec_sql = ExecSQL(_sql_cmd) try: print(exec_sql.execute()) except Exception as e: exit(str(e)) - diff --git a/pkg/deploy/action/cantian_common/get_config_info.py b/pkg/deploy/action/cantian_common/get_config_info.py index 6828348c..ae699347 100644 --- a/pkg/deploy/action/cantian_common/get_config_info.py +++ b/pkg/deploy/action/cantian_common/get_config_info.py @@ -35,4 +35,4 @@ def get_value(param): if __name__ == "__main__": _param = sys.argv[1] res = get_value(_param) - print(res) \ No newline at end of file + print(res) diff --git a/pkg/deploy/action/cantian_common/mysql_shell.py b/pkg/deploy/action/cantian_common/mysql_shell.py index 1a1ab680..81b68d52 100644 --- a/pkg/deploy/action/cantian_common/mysql_shell.py +++ b/pkg/deploy/action/cantian_common/mysql_shell.py @@ -10,8 +10,18 @@ import shlex LOCK_INSTANCE_FOR_BACKUP_TIMEOUT = 100000 + class MysqlShell: - def __init__(self, mysql_cmd, user, password='', database=None, host=None, port=None, socket=None): + def __init__( + self, + mysql_cmd, + user, + password="", + database=None, + host=None, + port=None, + socket=None, + ): self.mysql_cmd = mysql_cmd self.user = user self.password = password @@ -23,22 +33,24 @@ class MysqlShell: self.process = None def start_session(self, timeout=10): - cmd = shlex.split(self.mysql_cmd) + ['-u', self.user] + cmd = shlex.split(self.mysql_cmd) + ["-u", self.user] if self.password: - cmd.append('-p' + self.password) + cmd.append("-p" + self.password) else: - cmd.append('--skip-password') + cmd.append("--skip-password") if self.database: - cmd.extend(['-D', self.database]) + cmd.extend(["-D", self.database]) if self.host: - cmd.extend(['--host', self.host]) + cmd.extend(["--host", self.host]) if self.port: - cmd.extend(['--port', str(self.port)]) + cmd.extend(["--port", str(self.port)]) if self.socket: - cmd.extend(['--socket', self.socket]) + cmd.extend(["--socket", self.socket]) self.master_fd, slave_fd = pty.openpty() - self.process = subprocess.Popen(cmd, stdin=slave_fd, stdout=slave_fd, stderr=slave_fd, close_fds=True) + self.process = subprocess.Popen( + cmd, stdin=slave_fd, stdout=slave_fd, stderr=slave_fd, close_fds=True + ) # 检查连接是否成功 try: @@ -58,14 +70,14 @@ class MysqlShell: if self.master_fd in select.select([self.master_fd], [], [], 1)[0]: data = os.read(self.master_fd, 1024).decode() output.append(data) - if 'mysql>' in data: + if "mysql>" in data: break - elif 'Access denied' in data: + elif "Access denied" in data: raise Exception("Access denied for user.") except OSError: break time.sleep(0.1) - return ''.join(output) + return "".join(output) def execute_command(self, command, timeout=None): if self.process is not None: @@ -77,7 +89,7 @@ class MysqlShell: def close_session(self, timeout=10): try: if self.process is not None: - os.write(self.master_fd, b'exit\n') + os.write(self.master_fd, b"exit\n") start_time = time.time() while self.process.poll() is None: if (time.time() - start_time) > timeout: @@ -93,13 +105,15 @@ class MysqlShell: def lock_instance_for_backup(): - parser = argparse.ArgumentParser(description='Lock MySQL instance for backup.') - parser.add_argument('--mysql_cmd', type=str, required=True, help='Path to mysql executable') - parser.add_argument('--user', type=str, required=True, help='MySQL username') - parser.add_argument('--host', type=str, help='MySQL host') - parser.add_argument('--port', type=int, help='MySQL port') - parser.add_argument('--database', type=str, help='Database name') - parser.add_argument('--socket', type=str, help='MySQL socket') + parser = argparse.ArgumentParser(description="Lock MySQL instance for backup.") + parser.add_argument( + "--mysql_cmd", type=str, required=True, help="Path to mysql executable" + ) + parser.add_argument("--user", type=str, required=True, help="MySQL username") + parser.add_argument("--host", type=str, help="MySQL host") + parser.add_argument("--port", type=int, help="MySQL port") + parser.add_argument("--database", type=str, help="Database name") + parser.add_argument("--socket", type=str, help="MySQL socket") args = parser.parse_args() @@ -114,7 +128,7 @@ def lock_instance_for_backup(): database=args.database, host=args.host, port=args.port, - socket=args.socket + socket=args.socket, ) try: mysql_shell.start_session() @@ -154,5 +168,5 @@ def lock_instance_for_backup(): mysql_shell.close_session(timeout=10) -if __name__ == '__main__': +if __name__ == "__main__": lock_instance_for_backup() diff --git a/pkg/deploy/action/check_pwd.py b/pkg/deploy/action/check_pwd.py index 2c0b663d..ea872490 100644 --- a/pkg/deploy/action/check_pwd.py +++ b/pkg/deploy/action/check_pwd.py @@ -6,13 +6,13 @@ from OpenSSL import crypto from om_log import LOGGER as LOG PWD_LEN = 8 -CERT_FILE_PATH="/opt/cantian/certificate/" +CERT_FILE_PATH = "/opt/cantian/certificate/" class PassWordChecker: def __init__(self, pwd): self.pwd = pwd - self.user = 'ctcliuser' + self.user = "ctcliuser" def verify_new_passwd(self, shortest_len=PWD_LEN): """ @@ -36,14 +36,19 @@ class PassWordChecker: if passwd_set & cases: types += 1 if types < 3: - LOG.error("Error: Password must contains at least three different types of characters.") + LOG.error( + "Error: Password must contains at least three different types of characters." + ) return 1 # Only can contains enumerated cases all_cases = upper_cases | lower_cases | digits | special_cases un_cases = passwd_set - all_cases if un_cases: - LOG.error("Error: There are characters that are not allowed in the password: '%s'", "".join(un_cases)) + LOG.error( + "Error: There are characters that are not allowed in the password: '%s'", + "".join(un_cases), + ) return 1 return 0 @@ -51,21 +56,26 @@ class PassWordChecker: def check_cert_passwd(self): cert_file = os.path.join(CERT_FILE_PATH, "mes.crt") key_file = os.path.join(CERT_FILE_PATH, "mes.key") - # 加载证书 - with open(cert_file, 'rb') as f: + # 加载证书 + with open(cert_file, "rb") as f: cert_data = f.read() cert = crypto.load_certificate(crypto.FILETYPE_PEM, cert_data) # 加载私钥 - with open(key_file, 'rb') as f: + with open(key_file, "rb") as f: key_data = f.read() - private_key = serialization.load_pem_private_key(key_data, self.pwd.encode("utf-8"), default_backend()) + private_key = serialization.load_pem_private_key( + key_data, self.pwd.encode("utf-8"), default_backend() + ) # 获取证书的公钥 cert_public_key = cert.get_pubkey().to_cryptography_key() # 校验证书和私钥是否匹配 # 比较证书的公钥和私钥的公钥是否匹配 - if cert_public_key.public_numbers() == private_key.public_key().public_numbers(): + if ( + cert_public_key.public_numbers() + == private_key.public_key().public_numbers() + ): LOG.info("Certificate and private key are valid.") return 0 else: @@ -73,13 +83,13 @@ class PassWordChecker: return 1 -if __name__ == '__main__': +if __name__ == "__main__": pwd_checker = PassWordChecker(input()) action = "check_pwd" if len(sys.argv) > 1: action = sys.argv[1] operator = { "check_pwd": pwd_checker.verify_new_passwd, - "check_cert_pwd": pwd_checker.check_cert_passwd + "check_cert_pwd": pwd_checker.check_cert_passwd, } operator.get(action)() diff --git a/pkg/deploy/action/clear_history_version.py b/pkg/deploy/action/clear_history_version.py index 97b4065b..cb6eaf9c 100644 --- a/pkg/deploy/action/clear_history_version.py +++ b/pkg/deploy/action/clear_history_version.py @@ -2,12 +2,12 @@ import os import time from pathlib import Path -VERSION_PATH = '/opt/cantian/upgrade_backup' +VERSION_PATH = "/opt/cantian/upgrade_backup" def format_creation_time(file_path): ori_create_time = os.path.getctime(file_path) - return time.strftime('%Y%m%d%H%M%S', time.localtime(ori_create_time)) + return time.strftime("%Y%m%d%H%M%S", time.localtime(ori_create_time)) def delete_version(version_path): @@ -25,9 +25,14 @@ def delete_version(version_path): def execute(): version_names = os.listdir(VERSION_PATH) - version_info = [(str(Path(VERSION_PATH, name)), format_creation_time(str(Path(VERSION_PATH, name)))) - for name in version_names - if name.startswith('cantian_upgrade_bak')] + version_info = [ + ( + str(Path(VERSION_PATH, name)), + format_creation_time(str(Path(VERSION_PATH, name))), + ) + for name in version_names + if name.startswith("cantian_upgrade_bak") + ] version_info.sort(key=lambda x: x[1], reverse=False) for version_path, _ in version_info[:-2]: delete_version(version_path) diff --git a/pkg/deploy/action/clear_upgrade_backup.py b/pkg/deploy/action/clear_upgrade_backup.py index 977f6d3b..afe89cd5 100644 --- a/pkg/deploy/action/clear_upgrade_backup.py +++ b/pkg/deploy/action/clear_upgrade_backup.py @@ -41,16 +41,20 @@ class BackupRemover: LOG.error(f"upgrade backup note: {BACKUP_NOTE_FILE} is not exist") exit(1) - with open(BACKUP_NOTE_FILE, 'r', encoding='utf-8') as file: - backup_lis = file.read().split('\n') + with open(BACKUP_NOTE_FILE, "r", encoding="utf-8") as file: + backup_lis = file.read().split("\n") backup_lis.remove("") for each_backup_info in backup_lis: # each_backup_info为：版本号:生成备份时的时间戳 - backup_info_lis = each_backup_info.split(':') - - if len(backup_info_lis) < 2: # 长度小于2 说明文件格式不对； 为兼容错误情况，返回空列表 - LOG.error(f"upgrade backup note: {BACKUP_NOTE_FILE} with un_excepted format") + backup_info_lis = each_backup_info.split(":") + + if ( + len(backup_info_lis) < 2 + ): # 长度小于2 说明文件格式不对； 为兼容错误情况，返回空列表 + LOG.error( + f"upgrade backup note: {BACKUP_NOTE_FILE} with un_excepted format" + ) exit(1) return_list.append(backup_info_lis[0]) @@ -75,7 +79,7 @@ class BackupRemover: LOG.info(f"remove buackup files: {rm_dir} success") -if __name__ == '__main__': +if __name__ == "__main__": br = BackupRemover() try: br.clear_upgrade_backup() diff --git a/pkg/deploy/action/cms/cms_node0_stop.py b/pkg/deploy/action/cms/cms_node0_stop.py index ec23ccdc..66704779 100644 --- a/pkg/deploy/action/cms/cms_node0_stop.py +++ b/pkg/deploy/action/cms/cms_node0_stop.py @@ -12,8 +12,13 @@ def _exec_popen(cmd, values=None): if not values: values = [] bash_cmd = ["bash"] - pobj = subprocess.Popen(bash_cmd, shell=False, stdin=subprocess.PIPE, - stdout=subprocess.PIPE, stderr=subprocess.PIPE) + pobj = subprocess.Popen( + bash_cmd, + shell=False, + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + ) py_version = platform.python_version() if py_version[0] == "3": @@ -54,15 +59,18 @@ def stop_services(): def ping_kubernetes_service(): try: - subprocess.check_output(["timeout", "1", "ping", "-c", "1", "kubernetes.default.svc"], stderr=subprocess.STDOUT) + subprocess.check_output( + ["timeout", "1", "ping", "-c", "1", "kubernetes.default.svc"], + stderr=subprocess.STDOUT, + ) return True except subprocess.CalledProcessError: return False def main(): - node_id = get_value('node_id') - cantian_in_container = get_value('cantian_in_container') + node_id = get_value("node_id") + cantian_in_container = get_value("cantian_in_container") if node_id == "0" and cantian_in_container in ["1", "2"]: if not ping_kubernetes_service(): @@ -75,4 +83,4 @@ if __name__ == "__main__": try: main() except Exception as err: - LOGGER.error(f"Error stopping CMS process: {err}") \ No newline at end of file + LOGGER.error(f"Error stopping CMS process: {err}") diff --git a/pkg/deploy/action/cms/cmsctl.py b/pkg/deploy/action/cms/cmsctl.py index 5bafe0b2..177407cc 100644 --- a/pkg/deploy/action/cms/cmsctl.py +++ b/pkg/deploy/action/cms/cmsctl.py @@ -18,7 +18,8 @@ import json import glob from log import LOGGER from get_config_info import get_value -sys.path.append('../') + +sys.path.append("../") try: from obtains_lsid import LSIDGenerate except Exception as err: @@ -53,7 +54,7 @@ class CommonValue(object): MAX_DIRECTORY_MODE = 750 MIN_DIRECTORY_MODE = 550 - KEY_DIRECTORY_MODE_STR = '0700' + KEY_DIRECTORY_MODE_STR = "0700" MIN_FILE_PERMISSION = 0o400 MID_FILE_PERMISSION = 0o500 @@ -76,10 +77,20 @@ CANTIAN_GROUP = get_value("deploy_user") USE_DBSTOR = ["dbstor", "combined"] USE_DSS = ["dss"] -VALID_RUNNING_MODE = {CANTIAND, CANTIAND_WITH_MYSQL, CANTIAND_WITH_MYSQL_ST, CANTIAND_IN_CLUSTER, - CANTIAND_WITH_MYSQL_IN_CLUSTER, MYSQLD} +VALID_RUNNING_MODE = { + CANTIAND, + CANTIAND_WITH_MYSQL, + CANTIAND_WITH_MYSQL_ST, + CANTIAND_IN_CLUSTER, + CANTIAND_WITH_MYSQL_IN_CLUSTER, + MYSQLD, +} -VALID_SINGLE_MYSQL_RUNNING_MODE = {CANTIAND_WITH_MYSQL_IN_CLUSTER, CANTIAND_WITH_MYSQL_ST, CANTIAND_WITH_MYSQL} +VALID_SINGLE_MYSQL_RUNNING_MODE = { + CANTIAND_WITH_MYSQL_IN_CLUSTER, + CANTIAND_WITH_MYSQL_ST, + CANTIAND_WITH_MYSQL, +} CLUSTER_SIZE = 2 # default to 2, 4 node cluster mode need add parameter to specify this CMS_TOOL_CONFIG_COUNT = 4 @@ -90,18 +101,18 @@ MYSQL_BIN_DIR = "/opt/cantian/mysql/install/mysql" MYSQL_DATA_DIR = "" MYSQL_LOG_FILE = "" -a_ascii = ord('a') -z_ascii = ord('z') -aa_ascii = ord('A') -zz_ascii = ord('Z') -num0_ascii = ord('0') -num9_ascii = ord('9') -blank_ascii = ord(' ') +a_ascii = ord("a") +z_ascii = ord("z") +aa_ascii = ord("A") +zz_ascii = ord("Z") +num0_ascii = ord("0") +num9_ascii = ord("9") +blank_ascii = ord(" ") sep1_ascii = ord(os.sep) -sep2_ascii = ord('_') -sep3_ascii = ord(':') -sep4_ascii = ord('-') -sep5_ascii = ord('.') +sep2_ascii = ord("_") +sep3_ascii = ord(":") +sep4_ascii = ord("-") +sep5_ascii = ord(".") SEP_SED = r"\/" @@ -132,7 +143,10 @@ def check_runner(): if owner_uid == 0: if runner_uid != 0: runner = pwd.getpwuid(runner_uid).pw_name - err_msg = "the owner of *.sh has root privilege,can't run it by user [%s]." % runner + err_msg = ( + "the owner of *.sh has root privilege,can't run it by user [%s]." + % runner + ) LOGGER.error(err_msg) if FORCE_UNINSTALL != "force": raise Exception(err_msg) @@ -146,9 +160,15 @@ def check_runner(): elif runner_uid != owner_uid: runner = pwd.getpwuid(runner_uid).pw_name owner = pwd.getpwuid(owner_uid).pw_name - LOGGER.error("the owner of *.sh [%s] is different with the executor [%s]." % (owner, runner)) + LOGGER.error( + "the owner of *.sh [%s] is different with the executor [%s]." + % (owner, runner) + ) if FORCE_UNINSTALL != "force": - raise Exception("the owner of *.sh [%s] is different with the executor [%s]." % (owner, runner)) + raise Exception( + "the owner of *.sh [%s] is different with the executor [%s]." + % (owner, runner) + ) def _exec_popen(cmd, values=None): @@ -160,8 +180,13 @@ def _exec_popen(cmd, values=None): if not values: values = [] bash_cmd = ["bash"] - pobj = subprocess.Popen(bash_cmd, shell=False, stdin=subprocess.PIPE, - stdout=subprocess.PIPE, stderr=subprocess.PIPE) + pobj = subprocess.Popen( + bash_cmd, + shell=False, + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + ) if gPyVersion[0] == "3": pobj.stdin.write(cmd.encode()) @@ -202,7 +227,9 @@ def run_cmd(str_cmd, wrong_info): output = stdout + stderr LOGGER.error("%s.\ncommand: %s.\noutput: %s" % (wrong_info, str_cmd, output)) if FORCE_UNINSTALL != "force": - raise Exception("%s.\ncommand: %s.\noutput: %s" % (wrong_info, str_cmd, output)) + raise Exception( + "%s.\ncommand: %s.\noutput: %s" % (wrong_info, str_cmd, output) + ) return stdout @@ -251,26 +278,36 @@ def check_user(user, group): if FORCE_UNINSTALL != "force": raise Exception(err_msg) elif user == "root" or user_.pw_uid == 0: - LOGGER.error("parameter input error: -U, can not install program to" - " root user.") + LOGGER.error( + "parameter input error: -U, can not install program to" " root user." + ) if FORCE_UNINSTALL != "force": - raise Exception("parameter input error: -U, can not install program to" - " root user.") + raise Exception( + "parameter input error: -U, can not install program to" " root user." + ) elif group == "root" or user_.pw_gid == 0: - LOGGER.error("parameter input error: -U, can not install program to" - " user with root privilege.") + LOGGER.error( + "parameter input error: -U, can not install program to" + " user with root privilege." + ) if FORCE_UNINSTALL != "force": - raise Exception("parameter input error: -U, can not install program to" - " user with root privilege.") + raise Exception( + "parameter input error: -U, can not install program to" + " user with root privilege." + ) runner_uid = os.getuid() if runner_uid != 0 and runner_uid != user_.pw_uid: runner = pwd.getpwuid(runner_uid).pw_name - LOGGER.error("Parameter input error: -U, has to be the same as the" - " executor [%s]" % runner) + LOGGER.error( + "Parameter input error: -U, has to be the same as the" + " executor [%s]" % runner + ) if FORCE_UNINSTALL != "force": - raise Exception("Parameter input error: -U, has to be the same as the" - " executor [%s]" % runner) + raise Exception( + "Parameter input error: -U, has to be the same as the" + " executor [%s]" % runner + ) def check_path(path_type_in): @@ -280,19 +317,9 @@ def check_path(path_type_in): :return: weather validity """ path_len = len(path_type_in) - char_check_list1 = [blank_ascii, - sep1_ascii, - sep2_ascii, - sep4_ascii, - sep5_ascii - ] - - char_check_list2 = [blank_ascii, - sep1_ascii, - sep2_ascii, - sep3_ascii, - sep4_ascii - ] + char_check_list1 = [blank_ascii, sep1_ascii, sep2_ascii, sep4_ascii, sep5_ascii] + + char_check_list2 = [blank_ascii, sep1_ascii, sep2_ascii, sep3_ascii, sep4_ascii] if CURRENT_OS == "Linux": return check_path_linux(path_len, path_type_in, char_check_list1) elif CURRENT_OS == "Windows": @@ -305,10 +332,12 @@ def check_path(path_type_in): def check_path_linux(path_len, path_type_in, char_check_list): for i in range(0, path_len): char_check = ord(path_type_in[i]) - if (not (a_ascii <= char_check <= z_ascii - or aa_ascii <= char_check <= zz_ascii - or num0_ascii <= char_check <= num9_ascii - or char_check in char_check_list)): + if not ( + a_ascii <= char_check <= z_ascii + or aa_ascii <= char_check <= zz_ascii + or num0_ascii <= char_check <= num9_ascii + or char_check in char_check_list + ): return False return True @@ -316,10 +345,12 @@ def check_path_linux(path_len, path_type_in, char_check_list): def check_path_windows(path_len, path_type_in, char_check_list): for i in range(0, path_len): char_check = ord(path_type_in[i]) - if (not (a_ascii <= char_check <= z_ascii - or aa_ascii <= char_check <= zz_ascii - or num0_ascii <= char_check <= num9_ascii - or char_check in char_check_list)): + if not ( + a_ascii <= char_check <= z_ascii + or aa_ascii <= char_check <= zz_ascii + or num0_ascii <= char_check <= num9_ascii + or char_check in char_check_list + ): return False return True @@ -343,12 +374,13 @@ def genreg_string(text): ins_list = ins_str.split(os.sep) reg_string = "" for i in ins_list: - if (i == ""): + if i == "": continue else: reg_string += SEP_SED + i return reg_string + deploy_mode = get_value("deploy_mode") cantian_in_container = get_value("cantian_in_container") mes_type = get_value("mes_type") if deploy_mode in USE_DBSTOR else "TCP" @@ -380,41 +412,31 @@ CMS_CONFIG = { "_CMS_MES_MESSAGE_CHANNEL_NUM": "1", "_CMS_GCC_BAK": "", # generate by installer "_CLUSTER_ID": 0, - "_USE_DBSTOR": "", # generate by installer - "_DBSTOR_NAMESPACE": "", # generate by installer - "_CMS_MES_PIPE_TYPE": mes_type + "_USE_DBSTOR": "", # generate by installer + "_DBSTOR_NAMESPACE": "", # generate by installer + "_CMS_MES_PIPE_TYPE": mes_type, } PRIMARY_KEYSTORE = "/opt/cantian/common/config/primary_keystore_bak.ks" STANDBY_KEYSTORE = "/opt/cantian/common/config/standby_keystore_bak.ks" YOUMAI_DEMO = "/opt/cantian/youmai_demo" MES_CONFIG = { - "_CMS_MES_SSL_SWITCH": mes_ssl_switch, - "_CMS_MES_SSL_KEY_PWD": None, - "_CMS_MES_SSL_CRT_KEY_PATH": "/opt/cantian/common/config/certificates", - "KMC_KEY_FILES": f"({PRIMARY_KEYSTORE}, {STANDBY_KEYSTORE})" + "_CMS_MES_SSL_SWITCH": mes_ssl_switch, + "_CMS_MES_SSL_KEY_PWD": None, + "_CMS_MES_SSL_CRT_KEY_PATH": "/opt/cantian/common/config/certificates", + "KMC_KEY_FILES": f"({PRIMARY_KEYSTORE}, {STANDBY_KEYSTORE})", } -GCC_TYPE = { - "dbstor": "DBS", - "combined": "DBS", - "file": "file", - "dss": "SD" -} +GCC_TYPE = {"dbstor": "DBS", "combined": "DBS", "file": "file", "dss": "SD"} -GCC_HOME = { - "dbstor": "", - "combined": "", - "file": "", - "dss": "" -} +GCC_HOME = {"dbstor": "", "combined": "", "file": "", "dss": ""} CMS_CONFIG.update(MES_CONFIG) class NormalException(Exception): """ - Exception for exit(0) + Exception for exit(0) """ @@ -463,7 +485,9 @@ class CmsCtl(object): def __init__(self): install_config = "../../config/deploy_param.json" - self.install_config_file = os.path.join(os.path.dirname(os.path.abspath(__file__)), install_config) + self.install_config_file = os.path.join( + os.path.dirname(os.path.abspath(__file__)), install_config + ) @staticmethod def cms_check_share_logic_ip_isvalid(node_ip): @@ -478,17 +502,24 @@ class CmsCtl(object): def ping_execute(p_cmd): cmd = "%s %s -i 1 -c 3 | grep ttl | wc -l" % (p_cmd, node_ip) ret_code, stdout, stderr = _exec_popen(cmd) - if ret_code or stdout != '3': - LOGGER.info("The invalid IP address is %s. " - "ret_code : %s, stdout : %s, stderr : %s" % (node_ip, ret_code, stdout, stderr)) + if ret_code or stdout != "3": + LOGGER.info( + "The invalid IP address is %s. " + "ret_code : %s, stdout : %s, stderr : %s" + % (node_ip, ret_code, stdout, stderr) + ) return False return True LOGGER.info("check the node IP address or domain name.") if not ping_execute("ping") and not ping_execute("ping6"): - LOGGER.error("checked the node IP address or domain name failed: %s" % node_ip) + LOGGER.error( + "checked the node IP address or domain name failed: %s" % node_ip + ) if FORCE_UNINSTALL != "force": - raise Exception("checked the node IP address or domain name failed: %s" % node_ip) + raise Exception( + "checked the node IP address or domain name failed: %s" % node_ip + ) LOGGER.info("checked the node IP address or domain name success: %s" % node_ip) @@ -562,9 +593,13 @@ class CmsCtl(object): self.group = load_dict["group"] def check_link_type(self, load_dict): - if "link_type" in load_dict and (load_dict["link_type"] == "0" or load_dict["link_type"] == "TCP"): + if "link_type" in load_dict and ( + load_dict["link_type"] == "0" or load_dict["link_type"] == "TCP" + ): self.link_type = "TCP" - if "link_type" in load_dict and (load_dict["link_type"] == "2" or load_dict["link_type"] == "RDMA_1823"): + if "link_type" in load_dict and ( + load_dict["link_type"] == "2" or load_dict["link_type"] == "RDMA_1823" + ): self.link_type = "RDMA_1823" def parse_parameters(self, config_file): @@ -572,7 +607,7 @@ class CmsCtl(object): flags = os.O_RDONLY | os.O_EXCL modes = stat.S_IWUSR | stat.S_IRUSR try: - with os.fdopen(os.open(config_file, flags, modes), 'r') as load_f: + with os.fdopen(os.open(config_file, flags, modes), "r") as load_f: load_dict = json.load(load_f) self.load_user_config(load_dict) self.load_cms_run_config(load_dict) @@ -580,7 +615,9 @@ class CmsCtl(object): node_str = "node" + str(self.node_id) metadata_str = "metadata_" + self.storage_metadata_fs global MYSQL_DATA_DIR - MYSQL_DATA_DIR = os.path.join("/mnt/dbdata/remote", metadata_str, node_str) + MYSQL_DATA_DIR = os.path.join( + "/mnt/dbdata/remote", metadata_str, node_str + ) global MYSQL_LOG_FILE MYSQL_LOG_FILE = os.path.join(MYSQL_DATA_DIR, "mysql.log") except OSError as ex: @@ -630,7 +667,9 @@ class CmsCtl(object): flags = os.O_RDWR | os.O_CREAT | os.O_TRUNC modes = stat.S_IRUSR | stat.S_IWUSR | stat.S_IRGRP try: - with os.fdopen(os.open(self.cms_new_config, flags, modes), "w") as file_object: + with os.fdopen( + os.open(self.cms_new_config, flags, modes), "w" + ) as file_object: json.dump(conf_dict, file_object) except OSError as ex: LOGGER.error("failed to read config : %s" % str(ex)) @@ -665,7 +704,9 @@ class CmsCtl(object): if "GCC_TYPE" in common_parameters: common_parameters["GCC_DIR"] = common_parameters["GCC_HOME"] if deploy_mode not in USE_DSS: - common_parameters["GCC_HOME"] = os.path.join(common_parameters["GCC_HOME"], "gcc_file") + common_parameters["GCC_HOME"] = os.path.join( + common_parameters["GCC_HOME"], "gcc_file" + ) self.clean_old_conf(list(common_parameters.keys()), conf_file) self.set_new_conf(common_parameters, conf_file) @@ -685,17 +726,25 @@ class CmsCtl(object): node_ip.append("127.0.0.1") gcc_home = self.gcc_home gcc_dir = self.gcc_dir - + # gcc_home is lun, gcc_home:/dev/sda if self.gcc_type != "SD": gcc_dir = gcc_home gcc_home = os.path.join(gcc_home, "gcc_file") - if 'LD_LIBRARY_PATH' in os.environ: - ld_library_path = ("%s:%s:%s" % (os.path.join(self.install_path, "lib"), os.path.join( - self.install_path, "add-ons",), os.environ['LD_LIBRARY_PATH'])) + if "LD_LIBRARY_PATH" in os.environ: + ld_library_path = "%s:%s:%s" % ( + os.path.join(self.install_path, "lib"), + os.path.join( + self.install_path, + "add-ons", + ), + os.environ["LD_LIBRARY_PATH"], + ) else: - ld_library_path = ("%s:%s" % (os.path.join(self.install_path, "lib"), os.path.join( - self.install_path, "add-ons"),)) + ld_library_path = "%s:%s" % ( + os.path.join(self.install_path, "lib"), + os.path.join(self.install_path, "add-ons"), + ) common_parameters = { "GCC_HOME": gcc_home, "REPORT_FILE": LOG_FILE, @@ -744,7 +793,10 @@ class CmsCtl(object): """ cmd = "" for parameter in param_list: - cmd += "sed -i '/^%s/d' %s;" % (parameter.replace('[', '\[').replace(']', '\]'), conf_file) + cmd += "sed -i '/^%s/d' %s;" % ( + parameter.replace("[", "\[").replace("]", "\]"), + conf_file, + ) if cmd: cmd = cmd.strip(";") run_cmd(cmd, "failed to write the %s" % conf_file) @@ -767,27 +819,41 @@ class CmsCtl(object): socket_check.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) socket_check.settimeout(time_out) if gPyVersion < PYTHON242: - LOGGER.error("this install script can not support python version" - " : " + gPyVersion) + LOGGER.error( + "this install script can not support python version" " : " + gPyVersion + ) if FORCE_UNINSTALL != "force": - raise Exception("this install script can not support python version" - " : " + gPyVersion) + raise Exception( + "this install script can not support python version" + " : " + gPyVersion + ) try: socket_check.bind((node_ip, inner_port)) except socket.error as err_socket: socket_check.close() - if (int(err_socket.errno) == 98 or int(err_socket.errno) == 95 - or int(err_socket.errno) == 13): - LOGGER.info("Error: port %s has been used,the detail" - " information is as follows:" % value) + if ( + int(err_socket.errno) == 98 + or int(err_socket.errno) == 95 + or int(err_socket.errno) == 13 + ): + LOGGER.info( + "Error: port %s has been used,the detail" + " information is as follows:" % value + ) str_cmd = "netstat -unltp | grep %s" % value ret_code, stdout, stderr = _exec_popen(str_cmd) - LOGGER.error("can not get detail information of the" - " port, command:%s, output:%s, stderr:%s" % (str_cmd, str(stdout), stderr)) + LOGGER.error( + "can not get detail information of the" + " port, command:%s, output:%s, stderr:%s" + % (str_cmd, str(stdout), stderr) + ) if FORCE_UNINSTALL != "force": - raise Exception("can not get detail information of the" - " port, command:%s, output:%s, stderr:%s" % (str_cmd, str(stdout), stderr)) + raise Exception( + "can not get detail information of the" + " port, command:%s, output:%s, stderr:%s" + % (str_cmd, str(stdout), stderr) + ) socket_check.close() @@ -826,7 +892,7 @@ class CmsCtl(object): """ if not nodeip: return False - allowed_chars = set('0:.') + allowed_chars = set("0:.") if set(nodeip).issubset(allowed_chars): return True else: @@ -839,7 +905,7 @@ class CmsCtl(object): output: NA """ LOGGER.info("check the node IP address.") - if get_value("cantian_in_container") == '0': + if get_value("cantian_in_container") == "0": try: socket.inet_aton(nodeip) except socket.error: @@ -847,7 +913,10 @@ class CmsCtl(object): try: socket.inet_pton(socket.AF_INET6, nodeip) except socket.error: - err_msg = "the invalid IP address : %s is not ipv4 or ipv6 format." % nodeip + err_msg = ( + "the invalid IP address : %s is not ipv4 or ipv6 format." + % nodeip + ) LOGGER.error(err_msg) if FORCE_UNINSTALL != "force": raise Exception(err_msg) @@ -859,12 +928,18 @@ class CmsCtl(object): cmd = "%s %s -i 1 -c 3 | grep ttl | wc -l" % (ping_cmd, nodeip) ret_code, stdout, stderr = _exec_popen(cmd) - if ret_code or stdout != '3': - LOGGER.error("The invalid IP address is %s. " - "ret_code : %s, stdout : %s, stderr : %s" % (nodeip, ret_code, stdout, stderr)) + if ret_code or stdout != "3": + LOGGER.error( + "The invalid IP address is %s. " + "ret_code : %s, stdout : %s, stderr : %s" + % (nodeip, ret_code, stdout, stderr) + ) if FORCE_UNINSTALL != "force": - raise Exception("The invalid IP address is %s. " - "ret_code : %s, stdout : %s, stderr : %s" % (nodeip, ret_code, stdout, stderr)) + raise Exception( + "The invalid IP address is %s. " + "ret_code : %s, stdout : %s, stderr : %s" + % (nodeip, ret_code, stdout, stderr) + ) ip_is_found = 1 if nodeip in self.ip_addr: @@ -879,11 +954,17 @@ class CmsCtl(object): ip_is_found = 0 if ret_code or not int(ip_is_found): - LOGGER.error("The invalid IP address is %s. " - "ret_code : %s, ip_is_found : %s, stderr : %s" % (nodeip, ret_code, ip_is_found, stderr)) + LOGGER.error( + "The invalid IP address is %s. " + "ret_code : %s, ip_is_found : %s, stderr : %s" + % (nodeip, ret_code, ip_is_found, stderr) + ) if FORCE_UNINSTALL != "force": - raise Exception("The invalid IP address is %s. " - "ret_code : %s, ip_is_found : %s, stderr : %s" % (nodeip, ret_code, ip_is_found, stderr)) + raise Exception( + "The invalid IP address is %s. " + "ret_code : %s, ip_is_found : %s, stderr : %s" + % (nodeip, ret_code, ip_is_found, stderr) + ) LOGGER.info("checked the node IP address : %s" % nodeip) @@ -891,26 +972,39 @@ class CmsCtl(object): """ function: after decompression install package, change file permission """ - str_cmd = "chmod %s %s -R" % (CommonValue.KEY_DIRECTORY_MODE, - self.install_path) - str_cmd += ("&& find '%s'/add-ons -type f | xargs chmod %s " - % (self.install_path, CommonValue.MID_FILE_MODE)) - str_cmd += ("&& find '%s'/admin -type f | xargs chmod %s " - % (self.install_path, CommonValue.MIN_FILE_MODE)) - str_cmd += ("&& find '%s'/lib -type f | xargs chmod %s" - % (self.install_path, CommonValue.MID_FILE_MODE)) - str_cmd += ("&& find '%s'/bin -type f | xargs chmod %s " - % (self.install_path, CommonValue.MID_FILE_MODE)) - str_cmd += ("&& find '%s'/cfg -type f | xargs chmod %s " - % (self.install_path, CommonValue.KEY_FILE_MODE)) + str_cmd = "chmod %s %s -R" % (CommonValue.KEY_DIRECTORY_MODE, self.install_path) + str_cmd += "&& find '%s'/add-ons -type f | xargs chmod %s " % ( + self.install_path, + CommonValue.MID_FILE_MODE, + ) + str_cmd += "&& find '%s'/admin -type f | xargs chmod %s " % ( + self.install_path, + CommonValue.MIN_FILE_MODE, + ) + str_cmd += "&& find '%s'/lib -type f | xargs chmod %s" % ( + self.install_path, + CommonValue.MID_FILE_MODE, + ) + str_cmd += "&& find '%s'/bin -type f | xargs chmod %s " % ( + self.install_path, + CommonValue.MID_FILE_MODE, + ) + str_cmd += "&& find '%s'/cfg -type f | xargs chmod %s " % ( + self.install_path, + CommonValue.KEY_FILE_MODE, + ) package_xml = os.path.join(self.install_path, "package.xml") if os.path.exists(package_xml): - str_cmd += ("&& chmod %s '%s'/package.xml" - % (CommonValue.MIN_FILE_MODE, self.install_path)) + str_cmd += "&& chmod %s '%s'/package.xml" % ( + CommonValue.MIN_FILE_MODE, + self.install_path, + ) LOGGER.info("change app permission cmd: %s" % str_cmd) run_cmd(str_cmd, "failed to chmod %s" % CommonValue.KEY_DIRECTORY_MODE) - if cantian_in_container == "0" and (deploy_mode == "file" or os.path.exists(YOUMAI_DEMO)): + if cantian_in_container == "0" and ( + deploy_mode == "file" or os.path.exists(YOUMAI_DEMO) + ): self.chown_gcc_dirs() def export_user_env(self): @@ -921,20 +1015,29 @@ class CmsCtl(object): modes = stat.S_IWUSR | stat.S_IRUSR try: with os.fdopen(os.open(self.user_profile, flags, modes), "a") as _file: - _file.write("export CMS_HOME=\"%s\"" % self.cms_home) + _file.write('export CMS_HOME="%s"' % self.cms_home) _file.write(os.linesep) - _file.write("export PATH=\"%s\":$PATH" - % os.path.join(self.install_path, "bin")) + _file.write( + 'export PATH="%s":$PATH' % os.path.join(self.install_path, "bin") + ) _file.write(os.linesep) if "LD_LIBRARY_PATH" in os.environ: - _file.write("export LD_LIBRARY_PATH=\"%s\":\"%s\"" - ":$LD_LIBRARY_PATH" - % (os.path.join(self.install_path, "lib"), - os.path.join(self.install_path, "add-ons"))) + _file.write( + 'export LD_LIBRARY_PATH="%s":"%s"' + ":$LD_LIBRARY_PATH" + % ( + os.path.join(self.install_path, "lib"), + os.path.join(self.install_path, "add-ons"), + ) + ) else: - _file.write("export LD_LIBRARY_PATH=\"%s\":\"%s\"" - % (os.path.join(self.install_path, "lib"), - os.path.join(self.install_path, "add-ons"))) + _file.write( + 'export LD_LIBRARY_PATH="%s":"%s"' + % ( + os.path.join(self.install_path, "lib"), + os.path.join(self.install_path, "add-ons"), + ) + ) _file.write(os.linesep) _file.flush() LOGGER.info("write export for cms_home, path, lib_path") @@ -952,11 +1055,16 @@ class CmsCtl(object): str_cmd = "echo ~" ret_code, stdout, stderr = _exec_popen(str_cmd) if ret_code: - LOGGER.error("failed to get user home." - "ret_code : %s, stdout : %s, stderr : %s" % (ret_code, stdout, stderr)) + LOGGER.error( + "failed to get user home." + "ret_code : %s, stdout : %s, stderr : %s" % (ret_code, stdout, stderr) + ) if FORCE_UNINSTALL != "force": - raise Exception("failed to get user home." - "ret_code : %s, stdout : %s, stderr : %s" % (ret_code, stdout, stderr)) + raise Exception( + "failed to get user home." + "ret_code : %s, stdout : %s, stderr : %s" + % (ret_code, stdout, stderr) + ) output = os.path.realpath(os.path.normpath(stdout)) if not check_path(output): err_msg = "the user home directory is invalid." @@ -984,14 +1092,17 @@ class CmsCtl(object): with os.fdopen(os.open(self.user_profile, flags, modes), "r") as _file: while True: str_line = _file.readline() - if (not str_line): + if not str_line: break str_line = str_line.strip() - if (str_line.startswith("#")): + if str_line.startswith("#"): continue user_info = str_line.split() - if (len(user_info) >= 2 and user_info[0] == "export" - and user_info[1].startswith("CMS_HOME=") > 0): + if ( + len(user_info) >= 2 + and user_info[0] == "export" + and user_info[1].startswith("CMS_HOME=") > 0 + ): is_find = True break else: @@ -1003,7 +1114,10 @@ class CmsCtl(object): raise Exception(err_msg) def skip_execute_in_node_1(self): - if self.running_mode in [CANTIAND_IN_CLUSTER, CANTIAND_WITH_MYSQL_IN_CLUSTER] and self.node_id == 1: + if ( + self.running_mode in [CANTIAND_IN_CLUSTER, CANTIAND_WITH_MYSQL_IN_CLUSTER] + and self.node_id == 1 + ): return True return False @@ -1012,7 +1126,7 @@ class CmsCtl(object): chown data and gcc dirs :return: """ - cmd = "chown %s:%s -hR \"%s\";" % (self.user, self.group, self.gcc_home) + cmd = 'chown %s:%s -hR "%s";' % (self.user, self.group, self.gcc_home) LOGGER.info("change owner cmd: %s" % cmd) self.cms_check_share_logic_ip_isvalid(self.share_logic_ip) LOGGER.info("if blocked here, please check if the network is normal") @@ -1023,23 +1137,37 @@ class CmsCtl(object): LOGGER.info("prepare gcc home dir") self.cms_check_share_logic_ip_isvalid(self.share_logic_ip) LOGGER.info("if blocked here, please check if the network is normal") - if (deploy_mode == "file" or os.path.exists(YOUMAI_DEMO)) and not os.path.exists(self.gcc_home): + if ( + deploy_mode == "file" or os.path.exists(YOUMAI_DEMO) + ) and not os.path.exists(self.gcc_home): os.makedirs(self.gcc_home, CommonValue.KEY_DIRECTORY_PERMISSION) LOGGER.info("makedir for gcc_home %s" % self.gcc_home) def install_xnet_lib(self): if self.is_rdma_startup(): - str_cmd = "cp -arf %s/add-ons/mlnx/lib* %s/add-ons/" % (self.install_path, self.install_path) + str_cmd = "cp -arf %s/add-ons/mlnx/lib* %s/add-ons/" % ( + self.install_path, + self.install_path, + ) elif self.is_rdma_1823_startup(): - str_cmd = "cp -arf %s/add-ons/1823/lib* %s/add-ons/" % (self.install_path, self.install_path) + str_cmd = "cp -arf %s/add-ons/1823/lib* %s/add-ons/" % ( + self.install_path, + self.install_path, + ) else: - str_cmd = "cp -arf %s/add-ons/nomlnx/lib* %s/add-ons/" % (self.install_path, self.install_path) + str_cmd = "cp -arf %s/add-ons/nomlnx/lib* %s/add-ons/" % ( + self.install_path, + self.install_path, + ) LOGGER.info("install xnet lib cmd: " + str_cmd) run_cmd(str_cmd, "failed to install xnet lib") def install_kmc_lib(self): - str_cmd = "cp -arf %s/add-ons/kmc_shared/lib* %s/add-ons/" % (self.install_path, self.install_path) + str_cmd = "cp -arf %s/add-ons/kmc_shared/lib* %s/add-ons/" % ( + self.install_path, + self.install_path, + ) LOGGER.info("install kmc lib cmd: " + str_cmd) run_cmd(str_cmd, "failed to install kmc lib") @@ -1049,30 +1177,45 @@ class CmsCtl(object): """ ret_code, stdout, stderr = _exec_popen("which ofed_info") if ret_code: - LOGGER.info("no ofed_info cmd found" - "ret_code : %s, stdout : %s, stderr : %s" % (ret_code, stdout, stderr)) + LOGGER.info( + "no ofed_info cmd found" + "ret_code : %s, stdout : %s, stderr : %s" % (ret_code, stdout, stderr) + ) return False ret_code, stdout, stderr = _exec_popen("ofed_info -s") if ret_code: - LOGGER.info("exec ofed_info cmd failed" - "ret_code : %s, stdout : %s, stderr : %s" % (ret_code, stdout, stderr)) + LOGGER.info( + "exec ofed_info cmd failed" + "ret_code : %s, stdout : %s, stderr : %s" % (ret_code, stdout, stderr) + ) return False - if 'MLNX_OFED_LINUX-5.5' in stdout: + if "MLNX_OFED_LINUX-5.5" in stdout: LOGGER.info("mlnx version 5.5") return True ret_code, os_arch, stderr = _exec_popen("uname -i") if ret_code: - LOGGER.error("failed to get linux release version." - "ret_code : %s, os_arch : %s, stderr : %s" % (ret_code, os_arch, stderr)) + LOGGER.error( + "failed to get linux release version." + "ret_code : %s, os_arch : %s, stderr : %s" % (ret_code, os_arch, stderr) + ) if FORCE_UNINSTALL != "force": - raise Exception("failed to get linux release version." - "ret_code : %s, os_arch : %s, stderr : %s" % (ret_code, os_arch, stderr)) - aarch_mlnx_version_list = ['OFED-internal-5.8-2.0.3', 'MLNX_OFED_LINUX-5.8', 'MLNX_OFED_LINUX-5.9'] - aarch_version_check_result = any(mlnx_version if mlnx_version in stdout else False - for mlnx_version in aarch_mlnx_version_list) + raise Exception( + "failed to get linux release version." + "ret_code : %s, os_arch : %s, stderr : %s" + % (ret_code, os_arch, stderr) + ) + aarch_mlnx_version_list = [ + "OFED-internal-5.8-2.0.3", + "MLNX_OFED_LINUX-5.8", + "MLNX_OFED_LINUX-5.9", + ] + aarch_version_check_result = any( + mlnx_version if mlnx_version in stdout else False + for mlnx_version in aarch_mlnx_version_list + ) if os_arch == "aarch64" and aarch_version_check_result == True: LOGGER.info("Is mlnx 5.8~5.9") return True @@ -1098,15 +1241,17 @@ class CmsCtl(object): def check_parameter_install(self): if self.ip_cluster != "": - _list = self.ip_cluster.split(';') + _list = self.ip_cluster.split(";") self.ip_addr = _list[self.node_id] for item in re.split(r"[;,]", self.ip_cluster): if len(_list) != 1 and self.all_zero_addr_after_ping(item): - LOGGER.error("ip contains all-zero ip," - " can not specify other ip.") + LOGGER.error( + "ip contains all-zero ip," " can not specify other ip." + ) if FORCE_UNINSTALL != "force": - raise Exception("ip contains all-zero ip," - " can not specify other ip.") + raise Exception( + "ip contains all-zero ip," " can not specify other ip." + ) if cantian_in_container == "0": self.check_ip_isvaild(item) else: @@ -1136,10 +1281,18 @@ class CmsCtl(object): if not os.path.exists(self.install_path): os.makedirs(self.install_path, CommonValue.KEY_DIRECTORY_PERMISSION) cms_pkg_file = "/opt/cantian/image/Cantian-RUN-CENTOS-64bit" - str_cmd = ("cp -arf %s/add-ons %s/admin %s/bin %s/cfg %s/lib %s/package.xml %s" - % (cms_pkg_file, cms_pkg_file, cms_pkg_file, - cms_pkg_file, cms_pkg_file, cms_pkg_file, - self.install_path)) + str_cmd = ( + "cp -arf %s/add-ons %s/admin %s/bin %s/cfg %s/lib %s/package.xml %s" + % ( + cms_pkg_file, + cms_pkg_file, + cms_pkg_file, + cms_pkg_file, + cms_pkg_file, + cms_pkg_file, + self.install_path, + ) + ) LOGGER.info("copy install files cmd: " + str_cmd) run_cmd(str_cmd, "failed to install cms lib files") @@ -1171,14 +1324,24 @@ class CmsCtl(object): self.gcc_home = "/dev/gcc-disk" self.cms_gcc_bak = "/dev/gcc-disk" else: - self.gcc_home = os.path.join("/mnt/dbdata/remote/share_" + self.storage_share_fs, "gcc_home") - self.cms_gcc_bak = os.path.join("/mnt/dbdata/remote", "archive_" + self.storage_archive_fs) + self.gcc_home = os.path.join( + "/mnt/dbdata/remote/share_" + self.storage_share_fs, "gcc_home" + ) + self.cms_gcc_bak = os.path.join( + "/mnt/dbdata/remote", "archive_" + self.storage_archive_fs + ) if os.path.exists(YOUMAI_DEMO): - self.gcc_home = os.path.join("/mnt/dbdata/remote/share_" + self.storage_share_fs, "gcc_home") - self.cms_gcc_bak = os.path.join("/mnt/dbdata/remote", "archive_" + self.storage_archive_fs) + self.gcc_home = os.path.join( + "/mnt/dbdata/remote/share_" + self.storage_share_fs, "gcc_home" + ) + self.cms_gcc_bak = os.path.join( + "/mnt/dbdata/remote", "archive_" + self.storage_archive_fs + ) self.gcc_dir = self.gcc_home - LOGGER.info("======================== begin to pre_install cms configs ========================") + LOGGER.info( + "======================== begin to pre_install cms configs ========================" + ) check_user(self.user, self.group) if deploy_mode == "file" and not check_path(self.gcc_home): @@ -1191,85 +1354,174 @@ class CmsCtl(object): self.check_old_install() self.install_step = 1 self.set_cms_conf() - LOGGER.info("======================== pre_install cms configs successfully ========================") + LOGGER.info( + "======================== pre_install cms configs successfully ========================" + ) def prepare_cms_tool_dbstor_config(self): for i in range(10, CMS_TOOL_CONFIG_COUNT + 10): file_num = i - 9 uuid_generate = LSIDGenerate(2, self.cluster_id, i, self.node_id) inst_id, cms_tool_uuid = uuid_generate.execute() - str_cmd = ("cp -raf /opt/cantian/dbstor/tools/dbstor_config.ini" - " %s/dbstor/conf/dbs/dbstor_config_tool_%s.ini" % (self.cms_home, str(file_num))) - str_cmd += " && echo 'DBSTOR_OWNER_NAME = cms' >> %s/dbstor/conf/dbs/dbstor_config_tool_%s.ini" % ( - self.cms_home, str(file_num)) - str_cmd += " && echo 'CLUSTER_NAME = %s' >> %s/dbstor/conf/dbs/dbstor_config_tool_%s.ini" % ( - self.cluster_name, self.cms_home, str(file_num)) - str_cmd += " && echo 'CLUSTER_UUID = %s' >> %s/dbstor/conf/dbs/dbstor_config_tool_%s.ini" % ( - self.cluster_uuid, self.cms_home, str(file_num)) - str_cmd += (" && echo 'INST_ID = %s' >> %s/dbstor/conf/dbs/dbstor_config_tool_%s.ini" - % (inst_id, self.cms_home, str(file_num))) - str_cmd += " && echo 'CMS_TOOL_UUID = %s' >> %s/dbstor/conf/dbs/dbstor_config_tool_%s.ini" % ( - cms_tool_uuid, self.cms_home, str(file_num)) - str_cmd += (" && sed -i '/^\s*$/d' %s/dbstor/conf/dbs/dbstor_config_tool_%s.ini" - % (self.cms_home, str(file_num))) - str_cmd += " && chown -R %s:%s %s/dbstor" % (self.user, self.group, self.cms_home) - str_cmd += " && chmod 640 %s/dbstor/conf/dbs/dbstor_config_tool_%s.ini" % (self.cms_home, str(file_num)) + str_cmd = ( + "cp -raf /opt/cantian/dbstor/tools/dbstor_config.ini" + " %s/dbstor/conf/dbs/dbstor_config_tool_%s.ini" + % (self.cms_home, str(file_num)) + ) + str_cmd += ( + " && echo 'DBSTOR_OWNER_NAME = cms' >> %s/dbstor/conf/dbs/dbstor_config_tool_%s.ini" + % (self.cms_home, str(file_num)) + ) + str_cmd += ( + " && echo 'CLUSTER_NAME = %s' >> %s/dbstor/conf/dbs/dbstor_config_tool_%s.ini" + % (self.cluster_name, self.cms_home, str(file_num)) + ) + str_cmd += ( + " && echo 'CLUSTER_UUID = %s' >> %s/dbstor/conf/dbs/dbstor_config_tool_%s.ini" + % (self.cluster_uuid, self.cms_home, str(file_num)) + ) + str_cmd += ( + " && echo 'INST_ID = %s' >> %s/dbstor/conf/dbs/dbstor_config_tool_%s.ini" + % (inst_id, self.cms_home, str(file_num)) + ) + str_cmd += ( + " && echo 'CMS_TOOL_UUID = %s' >> %s/dbstor/conf/dbs/dbstor_config_tool_%s.ini" + % (cms_tool_uuid, self.cms_home, str(file_num)) + ) + str_cmd += ( + " && sed -i '/^\s*$/d' %s/dbstor/conf/dbs/dbstor_config_tool_%s.ini" + % (self.cms_home, str(file_num)) + ) + str_cmd += " && chown -R %s:%s %s/dbstor" % ( + self.user, + self.group, + self.cms_home, + ) + str_cmd += " && chmod 640 %s/dbstor/conf/dbs/dbstor_config_tool_%s.ini" % ( + self.cms_home, + str(file_num), + ) ret_code, stdout, stderr = _exec_popen(str_cmd) if ret_code: - LOGGER.error("prepare cms tool dbstor config file failed, return: " + - str(ret_code) + os.linesep + stdout + os.linesep + stderr) + LOGGER.error( + "prepare cms tool dbstor config file failed, return: " + + str(ret_code) + + os.linesep + + stdout + + os.linesep + + stderr + ) if FORCE_UNINSTALL != "force": - raise Exception("prepare cms tool dbstor config file failed, return: " + - str(ret_code) + os.linesep + stdout + os.linesep + stderr) + raise Exception( + "prepare cms tool dbstor config file failed, return: " + + str(ret_code) + + os.linesep + + stdout + + os.linesep + + stderr + ) def copy_dbstor_config(self): if os.path.exists(os.path.join(self.cms_home, "dbstor")): shutil.rmtree(os.path.join(self.cms_home, "dbstor")) - os.makedirs("%s/dbstor/conf/dbs" % self.cms_home, CommonValue.KEY_DIRECTORY_PERMISSION) - os.makedirs("%s/dbstor/conf/infra/config" % self.cms_home, CommonValue.KEY_DIRECTORY_PERMISSION) - os.makedirs("%s/dbstor/data/logs" % self.cms_home, CommonValue.KEY_DIRECTORY_PERMISSION) - os.makedirs("%s/dbstor/data/ftds" % self.cms_home, CommonValue.KEY_DIRECTORY_PERMISSION) + os.makedirs( + "%s/dbstor/conf/dbs" % self.cms_home, CommonValue.KEY_DIRECTORY_PERMISSION + ) + os.makedirs( + "%s/dbstor/conf/infra/config" % self.cms_home, + CommonValue.KEY_DIRECTORY_PERMISSION, + ) + os.makedirs( + "%s/dbstor/data/logs" % self.cms_home, CommonValue.KEY_DIRECTORY_PERMISSION + ) + os.makedirs( + "%s/dbstor/data/ftds" % self.cms_home, CommonValue.KEY_DIRECTORY_PERMISSION + ) if self.is_rdma_startup() or self.is_rdma_1823_startup(): - str_cmd = "cp -rfa %s/cfg/node_config_rdma_cms.xml %s/dbstor/conf/infra/config/node_config.xml" % ( - self.install_path, self.cms_home) + str_cmd = ( + "cp -rfa %s/cfg/node_config_rdma_cms.xml %s/dbstor/conf/infra/config/node_config.xml" + % (self.install_path, self.cms_home) + ) else: - str_cmd = "cp -raf %s/cfg/node_config_tcp_cms.xml %s/dbstor/conf/infra/config/node_config.xml" % ( - self.install_path, self.cms_home) + str_cmd = ( + "cp -raf %s/cfg/node_config_tcp_cms.xml %s/dbstor/conf/infra/config/node_config.xml" + % (self.install_path, self.cms_home) + ) generate_cluster_uuid = LSIDGenerate(0, self.cluster_id, 0, 0) generate_inst_id = LSIDGenerate(2, self.cluster_id, 0, self.node_id) _, self.cluster_uuid = generate_cluster_uuid.execute() inst_id, _ = generate_inst_id.execute() - str_cmd += " && cp -raf %s/cfg/osd.cfg %s/dbstor/conf/infra/config/osd.cfg" % (self.install_path, self.cms_home) - str_cmd += " && cp -raf /opt/cantian/dbstor/tools/dbstor_config.ini %s/dbstor/conf/dbs/" % (self.cms_home) - str_cmd += " && echo 'DBSTOR_OWNER_NAME = cms' >> %s/dbstor/conf/dbs/dbstor_config.ini" % (self.cms_home) - str_cmd += (" && echo 'CLUSTER_NAME = %s' >> %s/dbstor/conf/dbs/dbstor_config.ini" - % (self.cluster_name, self.cms_home)) - str_cmd += (" && echo 'CLUSTER_UUID = %s' >> %s/dbstor/conf/dbs/dbstor_config.ini" - % (self.cluster_uuid, self.cms_home)) - str_cmd += " && echo 'INST_ID = %s' >> %s/dbstor/conf/dbs/dbstor_config.ini" % (inst_id, self.cms_home) - str_cmd += " && sed -i '/^\s*$/d' %s/dbstor/conf/dbs/dbstor_config.ini" % (self.cms_home) - str_cmd += " && chown -R %s:%s %s/dbstor" % (self.user, self.group, self.cms_home) - str_cmd += " && chmod 640 %s/dbstor/conf/dbs/dbstor_config.ini" % (self.cms_home) + str_cmd += " && cp -raf %s/cfg/osd.cfg %s/dbstor/conf/infra/config/osd.cfg" % ( + self.install_path, + self.cms_home, + ) + str_cmd += ( + " && cp -raf /opt/cantian/dbstor/tools/dbstor_config.ini %s/dbstor/conf/dbs/" + % (self.cms_home) + ) + str_cmd += ( + " && echo 'DBSTOR_OWNER_NAME = cms' >> %s/dbstor/conf/dbs/dbstor_config.ini" + % (self.cms_home) + ) + str_cmd += ( + " && echo 'CLUSTER_NAME = %s' >> %s/dbstor/conf/dbs/dbstor_config.ini" + % (self.cluster_name, self.cms_home) + ) + str_cmd += ( + " && echo 'CLUSTER_UUID = %s' >> %s/dbstor/conf/dbs/dbstor_config.ini" + % (self.cluster_uuid, self.cms_home) + ) + str_cmd += " && echo 'INST_ID = %s' >> %s/dbstor/conf/dbs/dbstor_config.ini" % ( + inst_id, + self.cms_home, + ) + str_cmd += " && sed -i '/^\s*$/d' %s/dbstor/conf/dbs/dbstor_config.ini" % ( + self.cms_home + ) + str_cmd += " && chown -R %s:%s %s/dbstor" % ( + self.user, + self.group, + self.cms_home, + ) + str_cmd += " && chmod 640 %s/dbstor/conf/dbs/dbstor_config.ini" % ( + self.cms_home + ) LOGGER.info("copy config files cmd: " + str_cmd) ret_code, stdout, stderr = _exec_popen(str_cmd) if ret_code: - LOGGER.error("copy dbstor config file failed, return: " + - str(ret_code) + os.linesep + stdout + os.linesep + stderr) + LOGGER.error( + "copy dbstor config file failed, return: " + + str(ret_code) + + os.linesep + + stdout + + os.linesep + + stderr + ) if FORCE_UNINSTALL != "force": - raise Exception("copy dbstor config file failed, return: " + - str(ret_code) + os.linesep + stdout + os.linesep + stderr) + raise Exception( + "copy dbstor config file failed, return: " + + str(ret_code) + + os.linesep + + stdout + + os.linesep + + stderr + ) def install(self): """ install cms process, copy bin """ - LOGGER.info("======================== begin to install cms ========================") + LOGGER.info( + "======================== begin to install cms ========================" + ) if self.install_step == 2: LOGGER.info("cms install already") - LOGGER.info("======================== install cms module successfully ========================") + LOGGER.info( + "======================== install cms module successfully ========================" + ) return if self.install_step == 0: err_msg = "please run pre_install previously" @@ -1290,20 +1542,27 @@ class CmsCtl(object): self.copy_dbstor_config() self.prepare_cms_tool_dbstor_config() - cmd = "sh %s -P install_cms >> %s 2>&1" % (os.path.join(self.cms_scripts, "start_cms.sh"), LOG_FILE) + cmd = "sh %s -P install_cms >> %s 2>&1" % ( + os.path.join(self.cms_scripts, "start_cms.sh"), + LOG_FILE, + ) run_cmd(cmd, "failed to set cms node information") self.install_step = 2 self.set_cms_conf() - LOGGER.info("======================== install cms module successfully ========================") + LOGGER.info( + "======================== install cms module successfully ========================" + ) def check_start_status(self): """ cms启动后检查server和voting状态，确保在拉起cantiand前cms状态正常 """ - server_status_cmd = "source ~/.bashrc && cms stat -server %s | grep -v \"NODE_ID\" | awk '{print $2}'" \ - % self.node_id + server_status_cmd = ( + "source ~/.bashrc && cms stat -server %s | grep -v \"NODE_ID\" | awk '{print $2}'" + % self.node_id + ) voting_status_cmd = "source ~/.bashrc && cms node -connected | awk '{print $1, $NF}' | grep -v \"VOTING\"" timeout = 300 while timeout: @@ -1314,7 +1573,9 @@ class CmsCtl(object): LOGGER.error(err_msg) raise Exception(err_msg) try: - server_status = run_cmd(server_status_cmd, "failed query cms server status") + server_status = run_cmd( + server_status_cmd, "failed query cms server status" + ) except ValueError as _err: time.sleep(1) continue @@ -1337,7 +1598,9 @@ class CmsCtl(object): LOGGER.error(err_msg) raise Exception(err_msg) try: - voting_status = run_cmd(voting_status_cmd, "failed to query voting status") + voting_status = run_cmd( + voting_status_cmd, "failed to query voting status" + ) except ValueError as _err2: time.sleep(1) continue @@ -1361,28 +1624,39 @@ class CmsCtl(object): """ start cms process: check>>start>>change status """ - LOGGER.info("========================= begin to start cms process ========================") + LOGGER.info( + "========================= begin to start cms process ========================" + ) if self.install_step <= 1: LOGGER.error("please run install previously") if FORCE_UNINSTALL != "force": raise Exception("please run install previously") if self.install_step == 3: LOGGER.info("warning: cms started already") - LOGGER.info("========================= start cms process successfully ========================") + LOGGER.info( + "========================= start cms process successfully ========================" + ) return - cmd = "sh %s -P start_cms >> %s 2>&1" % (os.path.join(self.cms_scripts, "start_cms.sh"), LOG_FILE) + cmd = "sh %s -P start_cms >> %s 2>&1" % ( + os.path.join(self.cms_scripts, "start_cms.sh"), + LOG_FILE, + ) run_cmd(cmd, "failed to start cms process") self.install_step = 3 self.set_cms_conf() self.check_start_status() - LOGGER.info("======================== start cms process successfully ========================") + LOGGER.info( + "======================== start cms process successfully ========================" + ) def check_status(self): """ check cms process status """ - LOGGER.info("======================== begin to check cms process status ========================") + LOGGER.info( + "======================== begin to check cms process status ========================" + ) if self.install_step <= 1: err_msg = "please install cms previously" LOGGER.error(err_msg) @@ -1394,7 +1668,9 @@ class CmsCtl(object): for node_info in node_list: _node_id = node_info.split(" ")[0].strip(" ") if self.node_id == int(_node_id) and "TRUE" in node_info: - LOGGER.info("======================== check cms process status successfully ========================") + LOGGER.info( + "======================== check cms process status successfully ========================" + ) return err_msg = "check cms process status failed" LOGGER.error(err_msg) @@ -1405,15 +1681,24 @@ class CmsCtl(object): """ stop cms process: kill>>change status """ - LOGGER.info("======================== begin to stop the cms process ========================") - cms_processes = ["cms server", "cmsctl.py start", "cms/start.sh", "start_cms.sh"] + LOGGER.info( + "======================== begin to stop the cms process ========================" + ) + cms_processes = [ + "cms server", + "cmsctl.py start", + "cms/start.sh", + "start_cms.sh", + ] for cms_process in cms_processes: self.kill_process(cms_process) self.check_process_status(cms_process) self.install_step = 2 self.set_cms_conf() - LOGGER.info("======================== stop the cms successfully ========================") + LOGGER.info( + "======================== stop the cms successfully ========================" + ) def check_gcc_home_process(self): """ @@ -1436,7 +1721,10 @@ class CmsCtl(object): LOGGER.info("gcc_home occupied by a process:%s" % _stdout) def delete_only_start_file(self): - cmd = "dbstor --delete-file --fs-name=%s --file-name=onlyStart.file" % self.storage_share_fs + cmd = ( + "dbstor --delete-file --fs-name=%s --file-name=onlyStart.file" + % self.storage_share_fs + ) ret_code, stdout, stderr = _exec_popen(cmd) if ret_code: LOGGER.error("Failed to delete onlyStart.file") @@ -1445,29 +1733,46 @@ class CmsCtl(object): """ uninstall cms: environment values and app files """ - LOGGER.info("======================== begin to uninstall cms module ========================") + LOGGER.info( + "======================== begin to uninstall cms module ========================" + ) if self.gcc_home == "": if deploy_mode not in USE_DBSTOR: - self.gcc_home = os.path.join("/mnt/dbdata/remote/share_" + self.storage_share_fs, "gcc_home") + self.gcc_home = os.path.join( + "/mnt/dbdata/remote/share_" + self.storage_share_fs, "gcc_home" + ) elif deploy_mode == "dss": self.gcc_home = "/dev/gcc-disk" - + if self.node_id == 0: stdout, stderr = "", "" if cantian_in_container == "0": self.cms_check_share_logic_ip_isvalid(self.share_logic_ip) LOGGER.info("if blocked here, please check if the network is normal") if deploy_mode == "file" or os.path.exists(YOUMAI_DEMO): - versions_yml = os.path.join("/mnt/dbdata/remote/share_" + self.storage_share_fs, "versions.yml") - gcc_backup = os.path.join("/mnt/dbdata/remote/archive_" + self.storage_archive_fs, "gcc_backup") - str_cmd = "rm -rf %s && rm -rf %s && rm -rf %s" % (self.gcc_home, versions_yml, gcc_backup) - ret_code, stdout, stderr = _exec_popen("timeout 10 ls %s" % self.gcc_home) + versions_yml = os.path.join( + "/mnt/dbdata/remote/share_" + self.storage_share_fs, "versions.yml" + ) + gcc_backup = os.path.join( + "/mnt/dbdata/remote/archive_" + self.storage_archive_fs, + "gcc_backup", + ) + str_cmd = "rm -rf %s && rm -rf %s && rm -rf %s" % ( + self.gcc_home, + versions_yml, + gcc_backup, + ) + ret_code, stdout, stderr = _exec_popen( + "timeout 10 ls %s" % self.gcc_home + ) if deploy_mode in USE_DBSTOR: self.delete_only_start_file() - str_cmd = "cms gcc -del && dbstor --delete-file --fs-name=%s --file-name=versions.yml && " \ - "dbstor --delete-file --fs-name=%s --file-name=gcc_backup" \ - % (self.storage_share_fs, self.storage_archive_fs) + str_cmd = ( + "cms gcc -del && dbstor --delete-file --fs-name=%s --file-name=versions.yml && " + "dbstor --delete-file --fs-name=%s --file-name=gcc_backup" + % (self.storage_share_fs, self.storage_archive_fs) + ) ret_code = 0 if deploy_mode in USE_DSS: str_cmd = "dd if=/dev/zero of=/dev/gcc-disk bs=1M count=1 conv=notrunc" @@ -1480,38 +1785,55 @@ class CmsCtl(object): elif ret_code: output = stdout + stderr self.check_gcc_home_process() - err_msg = "failed to remove gcc home.\ncommand: %s.\noutput: %s \ + err_msg = ( + "failed to remove gcc home.\ncommand: %s.\noutput: %s \ \npossible reasons: \ \n1. user was using cms tool when uninstall. \ \n2. cms has not stopped. \ \n3. dbstor link was error. \ - \n4. others, please contact the engineer to solve." % (str_cmd, output) + \n4. others, please contact the engineer to solve." + % (str_cmd, output) + ) LOGGER.error(err_msg) if FORCE_UNINSTALL != "force": raise Exception(err_msg) elif FORCE_UNINSTALL != "force" and ret_code != 2: - LOGGER.error("can not connect to remote %s" - "ret_code : %s, stdout : %s, stderr : %s" % (self.gcc_home, ret_code, stdout, stderr)) + LOGGER.error( + "can not connect to remote %s" + "ret_code : %s, stdout : %s, stderr : %s" + % (self.gcc_home, ret_code, stdout, stderr) + ) if FORCE_UNINSTALL != "force": - raise Exception("can not connect to remote %s" - "ret_code : %s, stdout : %s, stderr : %s" % (self.gcc_home, ret_code, stdout, stderr)) + raise Exception( + "can not connect to remote %s" + "ret_code : %s, stdout : %s, stderr : %s" + % (self.gcc_home, ret_code, stdout, stderr) + ) self.clean_environment() self.clean_install_path() str_cmd = "rm -rf {0}/cms_server.lck {0}/local {0}/gcc_backup {0}/cantian.ctd.cms*".format( - self.cms_home) + self.cms_home + ) run_cmd(str_cmd, "failed to remove running files in cms home") - LOGGER.info("======================== uninstall cms module successfully ========================") + LOGGER.info( + "======================== uninstall cms module successfully ========================" + ) def backup(self): """ save cms config """ - LOGGER.info("======================== begin to backup config files ========================") + LOGGER.info( + "======================== begin to backup config files ========================" + ) config_json = os.path.join(self.cms_home, "cfg/cms.json") if os.path.exists(config_json): - str_cmd = ("cp -arf %s/cfg/* %s" % (self.cms_home, os.path.dirname(self.cms_old_config))) + str_cmd = "cp -arf %s/cfg/* %s" % ( + self.cms_home, + os.path.dirname(self.cms_old_config), + ) LOGGER.info("save cms json files cmd: " + str_cmd) run_cmd(str_cmd, "failed to save cms config files") else: @@ -1519,21 +1841,31 @@ class CmsCtl(object): LOGGER.error(err_msg) if FORCE_UNINSTALL != "force": raise Exception(err_msg) - LOGGER.info("======================== backup config files successfully ========================") + LOGGER.info( + "======================== backup config files successfully ========================" + ) def upgrade(self): - LOGGER.info("======================== begin to upgrade cms dbstor config ========================") - if deploy_mode in USE_DBSTOR and not glob.glob("%s/dbstor/conf/dbs/dbstor_config_tool*" % self.cms_home): + LOGGER.info( + "======================== begin to upgrade cms dbstor config ========================" + ) + if deploy_mode in USE_DBSTOR and not glob.glob( + "%s/dbstor/conf/dbs/dbstor_config_tool*" % self.cms_home + ): self.copy_dbstor_config() self.prepare_cms_tool_dbstor_config() - LOGGER.info("======================== upgrade cms dbstor config successfully ========================") + LOGGER.info( + "======================== upgrade cms dbstor config successfully ========================" + ) def init_container(self): """ cms init in container """ - LOGGER.info("======================== begin to init cms process =======================") + LOGGER.info( + "======================== begin to init cms process =======================" + ) if self.install_step == 3: LOGGER.info("Warning: cms start already") return @@ -1541,15 +1873,22 @@ class CmsCtl(object): self.copy_dbstor_config() if deploy_mode == "dbstor" or deploy_mode == "combined": self.prepare_cms_tool_dbstor_config() - - LOGGER.info("======================= init cms process ============================") - if self.node_id == 0: - cmd = "sh %s -P init_container >> %s 2>&1" %(os.path.join(os.path.dirname(__file__), "start_cms.sh"), LOG_FILE) + + LOGGER.info( + "======================= init cms process ============================" + ) + if self.node_id == 0: + cmd = "sh %s -P init_container >> %s 2>&1" % ( + os.path.join(os.path.dirname(__file__), "start_cms.sh"), + LOG_FILE, + ) run_cmd(cmd, "failed to init cms") - + self.install_step = 2 self.set_cms_conf() - LOGGER.info("======================= init cms process successfully ======================") + LOGGER.info( + "======================= init cms process successfully ======================" + ) def kill_process(self, process_name): """ @@ -1557,10 +1896,14 @@ class CmsCtl(object): input: process name output: NA """ - kill_cmd = (r"proc_pid_list=`ps ux | grep '%s' | grep -v grep" - r"|awk '{print $2}'` && " % process_name) - kill_cmd += (r"(if [[ X\"$proc_pid_list\" != X\"\" ]];then echo " - r"$proc_pid_list | xargs kill -9; fi)") + kill_cmd = ( + r"proc_pid_list=`ps ux | grep '%s' | grep -v grep" + r"|awk '{print $2}'` && " % process_name + ) + kill_cmd += ( + r"(if [[ X\"$proc_pid_list\" != X\"\" ]];then echo " + r"$proc_pid_list | xargs kill -9; fi)" + ) LOGGER.info("kill process cmd: %s" % kill_cmd) run_cmd(kill_cmd, "failed to kill process %s" % process_name) @@ -1573,9 +1916,15 @@ class CmsCtl(object): get_cmd = "ps ux | grep '%s' | grep -v grep | awk '{print $2}'" % process_name ret_code, stdout, stderr = _exec_popen(get_cmd) if ret_code: - LOGGER.error("Failed to get %s pid. cmd: %s. Error: %s" % (process_name, get_cmd, stderr)) + LOGGER.error( + "Failed to get %s pid. cmd: %s. Error: %s" + % (process_name, get_cmd, stderr) + ) if FORCE_UNINSTALL != "force": - raise Exception("Failed to get %s pid. cmd: %s. Error: %s" % (process_name, get_cmd, stderr)) + raise Exception( + "Failed to get %s pid. cmd: %s. Error: %s" + % (process_name, get_cmd, stderr) + ) return stdout def check_process_status(self, process_name): @@ -1587,7 +1936,9 @@ class CmsCtl(object): for i in range(CHECK_MAX_TIMES): pid = self.get_pid(process_name) if pid: - LOGGER.info("checked %s times, %s pid is %s" % (i + 1, process_name, pid)) + LOGGER.info( + "checked %s times, %s pid is %s" % (i + 1, process_name, pid) + ) if i != CHECK_MAX_TIMES - 1: time.sleep(5) else: @@ -1621,11 +1972,13 @@ class CmsCtl(object): output: NA """ LOGGER.info("cleaning user environment variables...") - path_cmd = (r"/^\s*export\s*PATH=\"%s\/bin\":\$PATH$/d" - % genreg_string(self.install_path)) - lib_cmd = (r"/^\s*export\s*LD_LIBRARY_PATH=\"%s\/lib\":\"%s\/add-ons\".*$/d" - % (genreg_string(self.install_path), - genreg_string(self.install_path))) + path_cmd = r"/^\s*export\s*PATH=\"%s\/bin\":\$PATH$/d" % genreg_string( + self.install_path + ) + lib_cmd = r"/^\s*export\s*LD_LIBRARY_PATH=\"%s\/lib\":\"%s\/add-ons\".*$/d" % ( + genreg_string(self.install_path), + genreg_string(self.install_path), + ) cms_cmd = r"/^\s*export\s*CMS_HOME=\".*\"$/d" cmds = [path_cmd, lib_cmd, cms_cmd] @@ -1653,7 +2006,15 @@ def main(): cms.parse_parameters(cms.cms_new_config) cms.install() - if arg in {"start", "check_status", "stop", "uninstall", "backup", "upgrade", "init_container"}: + if arg in { + "start", + "check_status", + "stop", + "uninstall", + "backup", + "upgrade", + "init_container", + }: if os.path.exists("/opt/cantian/cms/cfg/cms.json"): install_cms_cfg = "/opt/cantian/cms/cfg/cms.json" diff --git a/pkg/deploy/action/cms/get_config_info.py b/pkg/deploy/action/cms/get_config_info.py index 3bc1a922..10ce9e2f 100644 --- a/pkg/deploy/action/cms/get_config_info.py +++ b/pkg/deploy/action/cms/get_config_info.py @@ -34,9 +34,11 @@ def get_value(param): return cms_conf.get("install_step") if param == "deploy_mode": - if info.get('deploy_mode', ""): - return info.get('deploy_mode') - return "dbstor" if info.get('deploy_policy', "") in ["ModeB", "ModeC"] else "file" + if info.get("deploy_mode", ""): + return info.get("deploy_mode") + return ( + "dbstor" if info.get("deploy_policy", "") in ["ModeB", "ModeC"] else "file" + ) return info.get(param, "") @@ -45,4 +47,3 @@ if __name__ == "__main__": _param = sys.argv[1] res = get_value(_param) print(res) - diff --git a/pkg/deploy/action/cms/log.py b/pkg/deploy/action/cms/log.py index 7991b87e..c516ff38 100644 --- a/pkg/deploy/action/cms/log.py +++ b/pkg/deploy/action/cms/log.py @@ -16,9 +16,9 @@ CONSOLE_CONF = { "log_file_backup_count": 5, "log_date_format": "%Y-%m-%d %H:%M:%S", "logging_default_format_string": "%(asctime)s %(levelname)s [pid:%(process)d] [%(threadName)s] " - "[tid:%(thread)d] [%(filename)s:%(lineno)d %(funcName)s] %(message)s", + "[tid:%(thread)d] [%(filename)s:%(lineno)d %(funcName)s] %(message)s", "logging_context_format_string": "%(asctime)s %(levelname)s [pid:%(process)d] [%(threadName)s] " - "[tid:%(thread)d] [%(filename)s:%(lineno)d %(funcName)s] %(message)s" + "[tid:%(thread)d] [%(filename)s:%(lineno)d %(funcName)s] %(message)s", } } @@ -41,14 +41,30 @@ def _get_log_file_path(project): os.makedirs(logger_dir) return os.path.join(logger_dir, "{}.log".format(project)) - return '' + return "" SENSITIVE_STR = [ - 'Password', 'passWord', 'PASSWORD', 'password', 'Pswd', - 'PSWD', 'pwd', 'signature', 'HmacSHA256', 'newPasswd', - 'private', 'certfile', 'secret', 'token', 'Token', 'pswd', - 'passwd', 'mysql -u', 'session', 'cookie' + "Password", + "passWord", + "PASSWORD", + "password", + "Pswd", + "PSWD", + "pwd", + "signature", + "HmacSHA256", + "newPasswd", + "private", + "certfile", + "secret", + "token", + "Token", + "pswd", + "passwd", + "mysql -u", + "session", + "cookie", ] @@ -74,8 +90,10 @@ def setup(project_name): log_path = _get_log_file_path(project_name) if log_path: file_log = handlers.RotatingFileHandler( - log_path, maxBytes=log_config.get("log_file_max_size"), - backupCount=log_config.get("log_file_backup_count")) + log_path, + maxBytes=log_config.get("log_file_max_size"), + backupCount=log_config.get("log_file_backup_count"), + ) log_root.addHandler(file_log) log_root.addFilter(DefaultLogFilter()) @@ -83,7 +101,9 @@ def setup(project_name): handler.setFormatter( logging.Formatter( fmt=log_config.get("logging_context_format_string"), - datefmt=log_config.get("log_date_format"))) + datefmt=log_config.get("log_date_format"), + ) + ) if log_config.get("debug"): log_root.setLevel(logging.DEBUG) @@ -95,4 +115,4 @@ def setup(project_name): LOGGER = setup(os.path.basename(LOG_FILE).split(".")[0]) log_directory = log_config.get("log_dir") os.chmod(log_directory, 0o750) -os.chmod(f'{str(Path(log_directory, os.path.basename(LOG_FILE)))}', 0o640) +os.chmod(f"{str(Path(log_directory, os.path.basename(LOG_FILE)))}", 0o640) diff --git a/pkg/deploy/action/dbstor/dbstor_backup.py b/pkg/deploy/action/dbstor/dbstor_backup.py index ad5144f7..a106006a 100644 --- a/pkg/deploy/action/dbstor/dbstor_backup.py +++ b/pkg/deploy/action/dbstor/dbstor_backup.py @@ -31,6 +31,7 @@ MAX_DIRECTORY_MODE = 755 class ReadConfigParserNoCast(ConfigParser): "Inherit from built-in class: ConfigParser" + def optionxform(self, optionstr): "Rewrite without lower()" return optionstr @@ -40,6 +41,7 @@ class Options(object): """ class for command line options """ + def __init__(self): self.install_user_privilege = "withoutroot" self.conf_file_path = "/opt/cantian/backup/files" @@ -48,9 +50,11 @@ class Options(object): self.log_file = "/opt/cantian/log/dbstor/backup.log" self.ini_file = "/opt/cantian/dbstor/tools/dbstor_config.ini" self.docker_ini_file = "/home/regress/cantian_data" - self.js_conf_file = os.path.join(os.path.dirname(os.path.abspath(__file__)), "../../config/deploy_param.json") + self.js_conf_file = os.path.join( + os.path.dirname(os.path.abspath(__file__)), "../../config/deploy_param.json" + ) self.dbstor_config = {} - self.section = 'CLIENT' + self.section = "CLIENT" self.note_id = "" self.cluster_name = "" @@ -60,11 +64,13 @@ gPyVersion = platform.python_version() logger = logging.getLogger() logger.setLevel(logging.DEBUG) -logger_handle = logging.FileHandler(g_opts.log_file, 'a', "utf-8") +logger_handle = logging.FileHandler(g_opts.log_file, "a", "utf-8") logger_handle.setLevel(logging.DEBUG) -logger_formatter = logging.Formatter('[%(asctime)s]-[%(filename)s]-[line:%(lineno)d]-[%(levelname)s]-' - '%(message)s-[%(process)s]') +logger_formatter = logging.Formatter( + "[%(asctime)s]-[%(filename)s]-[line:%(lineno)d]-[%(levelname)s]-" + "%(message)s-[%(process)s]" +) logger_handle.setFormatter(logger_formatter) logger.addHandler(logger_handle) logger.info("init logging success") @@ -88,8 +94,10 @@ def log_exit(msg): :return: NA """ console_and_log("Error: " + msg) - print("Please refer to install log \"%s\" for more detailed information." - % g_opts.log_file) + print( + 'Please refer to install log "%s" for more detailed information.' + % g_opts.log_file + ) raise ValueError(str(msg)) @@ -102,8 +110,13 @@ def _exec_popen(cmd, values=None): if not values: values = [] bash_cmd = ["bash"] - p_obj = subprocess.Popen(bash_cmd, shell=False, stdin=subprocess.PIPE, - stdout=subprocess.PIPE, stderr=subprocess.PIPE) + p_obj = subprocess.Popen( + bash_cmd, + shell=False, + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + ) if gPyVersion[0] == "3": p_obj.stdin.write(cmd.encode()) @@ -170,7 +183,10 @@ def clean_backup_ini(): try: os.remove(g_opts.backup_ini_file) except IOError as io_err: - log_exit("Error: Can not remove backup dbstor config file: " + g_opts.backup_ini_file) + log_exit( + "Error: Can not remove backup dbstor config file: " + + g_opts.backup_ini_file + ) def read_ini_parameter(): @@ -211,13 +227,19 @@ def check_backup_ini_path(): def read_file_path(): - with os.fdopen(os.open(g_opts.js_conf_file, os.O_RDONLY | os.O_EXCL, stat.S_IWUSR | stat.S_IRUSR), "r",)\ - as file_handle: + with os.fdopen( + os.open( + g_opts.js_conf_file, os.O_RDONLY | os.O_EXCL, stat.S_IWUSR | stat.S_IRUSR + ), + "r", + ) as file_handle: json_data = json.load(file_handle) - g_opts.note_id = json_data.get('node_id', "").strip() - g_opts.cluster_name = json_data.get('cluster_name', "").strip() + g_opts.note_id = json_data.get("node_id", "").strip() + g_opts.cluster_name = json_data.get("cluster_name", "").strip() g_opts.ini_file = "/opt/cantian/dbstor/tools/dbstor_config.ini" - g_opts.backup_ini_file = os.path.join(g_opts.conf_file_path, "dbstor_config.ini") + g_opts.backup_ini_file = os.path.join( + g_opts.conf_file_path, "dbstor_config.ini" + ) def main(): diff --git a/pkg/deploy/action/dbstor/dbstor_install.py b/pkg/deploy/action/dbstor/dbstor_install.py index 920ac0a8..1db4a3cc 100644 --- a/pkg/deploy/action/dbstor/dbstor_install.py +++ b/pkg/deploy/action/dbstor/dbstor_install.py @@ -30,12 +30,16 @@ try: LOG_PATH = "/opt/cantian/log/dbstor" LOG_FILE = "/opt/cantian/log/dbstor/install.log" - JS_CONF_FILE = os.path.join(os.path.dirname(os.path.abspath(__file__)), "../../config/deploy_param.json") + JS_CONF_FILE = os.path.join( + os.path.dirname(os.path.abspath(__file__)), "../../config/deploy_param.json" + ) DBSTOR_CONF_FILE = "/mnt/dbdata/remote/share_" - CONTAINER_DBSTOR_CONF_DIR = os.path.join(os.path.dirname(os.path.abspath(__file__)), "../../config/container") + CONTAINER_DBSTOR_CONF_DIR = os.path.join( + os.path.dirname(os.path.abspath(__file__)), "../../config/container" + ) DOCKER_DBSTOR_CONF_FILE = "/home/regress/cantian_data" BACKUP_CONF_FILE = "/opt/cantian/backup/files" - SECTION = 'CLIENT' + SECTION = "CLIENT" MAX_DIRECTORY_MODE = 0o755 PYTHON242 = "2.4.2" PYTHON25 = "2.5" @@ -43,11 +47,13 @@ try: logger = logging.getLogger() logger.setLevel(logging.DEBUG) - logger_handle = logging.FileHandler(LOG_FILE, 'a', "utf-8") + logger_handle = logging.FileHandler(LOG_FILE, "a", "utf-8") logger_handle.setLevel(logging.DEBUG) - logger_formatter = logging.Formatter('[%(asctime)s]-[%(filename)s]-[line:%(lineno)d]-[%(levelname)s]-' - '%(message)s-[%(process)s]') + logger_formatter = logging.Formatter( + "[%(asctime)s]-[%(filename)s]-[line:%(lineno)d]-[%(levelname)s]-" + "%(message)s-[%(process)s]" + ) logger_handle.setFormatter(logger_formatter) logger.addHandler(logger_handle) logger.info("init logging success") @@ -57,7 +63,9 @@ try: elif gPyVersion >= PYTHON25: pass else: - import_msg = "This install script can not support python version: %s" % gPyVersion + import_msg = ( + "This install script can not support python version: %s" % gPyVersion + ) logger.info(import_msg) raise ValueError(import_msg) sys.path.append(os.path.split(os.path.realpath(__file__))[0]) @@ -71,6 +79,7 @@ GLOBAL_KMC_EXT = None class ReadConfigParserNoCast(ConfigParser): "Inherit from built-in class: ConfigParser" + def optionxform(self, optionstr): "Rewrite without lower()" return optionstr @@ -83,22 +92,35 @@ def check_path(path_type_in): :return: weather validity """ path_len = len(path_type_in) - ascii_map = {'a_ascii': ord('a'), 'z_ascii': ord('z'), 'a_upper_ascii': ord('A'), 'z_upper_ascii': ord('Z'), - 'num0_ascii': ord('0'), 'num9_ascii': ord('9'), 'blank_ascii': ord(' '), 'sep1_ascii': ord(os.sep), - 'sep2_ascii': ord('_'), 'sep3_ascii': ord(':'), 'sep4_ascii': ord('-'), 'sep5_ascii': ord('.')} - char_check_list1 = [ascii_map.get('blank_ascii', 0), - ascii_map.get('sep1_ascii', 0), - ascii_map.get('sep2_ascii', 0), - ascii_map.get('sep4_ascii', 0), - ascii_map.get('sep5_ascii', 0) - ] - - char_check_list2 = [ascii_map.get('blank_ascii', 0), - ascii_map.get('sep1_ascii', 0), - ascii_map.get('sep2_ascii', 0), - ascii_map.get('sep3_ascii', 0), - ascii_map.get('sep4_ascii', 0) - ] + ascii_map = { + "a_ascii": ord("a"), + "z_ascii": ord("z"), + "a_upper_ascii": ord("A"), + "z_upper_ascii": ord("Z"), + "num0_ascii": ord("0"), + "num9_ascii": ord("9"), + "blank_ascii": ord(" "), + "sep1_ascii": ord(os.sep), + "sep2_ascii": ord("_"), + "sep3_ascii": ord(":"), + "sep4_ascii": ord("-"), + "sep5_ascii": ord("."), + } + char_check_list1 = [ + ascii_map.get("blank_ascii", 0), + ascii_map.get("sep1_ascii", 0), + ascii_map.get("sep2_ascii", 0), + ascii_map.get("sep4_ascii", 0), + ascii_map.get("sep5_ascii", 0), + ] + + char_check_list2 = [ + ascii_map.get("blank_ascii", 0), + ascii_map.get("sep1_ascii", 0), + ascii_map.get("sep2_ascii", 0), + ascii_map.get("sep3_ascii", 0), + ascii_map.get("sep4_ascii", 0), + ] if CURRENT_OS == "Linux": return check_path_linux(path_len, path_type_in, ascii_map, char_check_list1) elif CURRENT_OS == "Windows": @@ -112,10 +134,16 @@ def check_path(path_type_in): def check_path_linux(path_len, path_type_in, ascii_map, char_check_list): for i in range(0, path_len): char_check = ord(path_type_in[i]) - if (not (ascii_map.get('a_ascii', 0) <= char_check <= ascii_map.get('z_ascii', 0) - or ascii_map.get('a_upper_ascii', 0) <= char_check <= ascii_map.get('z_upper_ascii', 0) - or ascii_map.get('num0_ascii', 0) <= char_check <= ascii_map.get('num9_ascii', 0) - or char_check in char_check_list)): + if not ( + ascii_map.get("a_ascii", 0) <= char_check <= ascii_map.get("z_ascii", 0) + or ascii_map.get("a_upper_ascii", 0) + <= char_check + <= ascii_map.get("z_upper_ascii", 0) + or ascii_map.get("num0_ascii", 0) + <= char_check + <= ascii_map.get("num9_ascii", 0) + or char_check in char_check_list + ): return False return True @@ -123,10 +151,16 @@ def check_path_linux(path_len, path_type_in, ascii_map, char_check_list): def check_path_windows(path_len, path_type_in, ascii_map, char_check_list): for i in range(0, path_len): char_check = ord(path_type_in[i]) - if (not (ascii_map.get('a_ascii', 0) <= char_check <= ascii_map.get('z_ascii', 0) - or ascii_map.get('a_upper_ascii', 0) <= char_check <= ascii_map.get('z_upper_ascii', 0) - or ascii_map.get('num0_ascii', 0) <= char_check <= ascii_map.get('num9_ascii', 0) - or char_check in char_check_list)): + if not ( + ascii_map.get("a_ascii", 0) <= char_check <= ascii_map.get("z_ascii", 0) + or ascii_map.get("a_upper_ascii", 0) + <= char_check + <= ascii_map.get("z_upper_ascii", 0) + or ascii_map.get("num0_ascii", 0) + <= char_check + <= ascii_map.get("num9_ascii", 0) + or char_check in char_check_list + ): return False return True @@ -149,8 +183,7 @@ def log_exit(msg): :return: NA """ console_and_log("Error: " + msg) - print("Please refer to install log \"%s\" for more detailed information." - % LOG_FILE) + print('Please refer to install log "%s" for more detailed information.' % LOG_FILE) raise ValueError(str(msg)) @@ -162,18 +195,24 @@ def check_runner(): if owner_uid == 0: if runner_uid != 0: runner = pwd.getpwuid(runner_uid).pw_name - log_exit("Error: The owner of install.py has root privilege," - " can't run it by user [%s]." % runner) + log_exit( + "Error: The owner of install.py has root privilege," + " can't run it by user [%s]." % runner + ) else: if runner_uid == 0: owner = pwd.getpwuid(owner_uid).pw_name - log_exit("Error: The owner of install.py is [%s]," - " can't run it by root." % owner) + log_exit( + "Error: The owner of install.py is [%s]," + " can't run it by root." % owner + ) elif runner_uid != owner_uid: runner = pwd.getpwuid(runner_uid).pw_name owner = pwd.getpwuid(owner_uid).pw_name - log_exit("Error: The owner of install.py [%s] is different" - " with the executor [%s]." % (owner, runner)) + log_exit( + "Error: The owner of install.py [%s] is different" + " with the executor [%s]." % (owner, runner) + ) def _exec_popen(cmd, values=None): @@ -185,8 +224,13 @@ def _exec_popen(cmd, values=None): if not values: values = [] bash_cmd = ["bash"] - p_obj = subprocess.Popen(bash_cmd, shell=False, stdin=subprocess.PIPE, - stdout=subprocess.PIPE, stderr=subprocess.PIPE) + p_obj = subprocess.Popen( + bash_cmd, + shell=False, + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + ) if gPyVersion[0] == "3": p_obj.stdin.write(cmd.encode()) @@ -217,14 +261,18 @@ def check_user(user, group): try: user_ = pwd.getpwnam(user) except KeyError as key_error_msg: - message = "Parameter input error: -U, the user does not exists.%s" % str(key_error_msg) + message = "Parameter input error: -U, the user does not exists.%s" % str( + key_error_msg + ) logger.error(message) raise ValueError(message) from key_error_msg try: group_ = grp.getgrnam(group) except KeyError as key_error_msg: - message = "Parameter input error: -U, the group does not exists.%s" % str(key_error_msg) + message = "Parameter input error: -U, the group does not exists.%s" % str( + key_error_msg + ) logger.error(message) raise ValueError(message) from key_error_msg @@ -244,16 +292,19 @@ def check_user(user, group): runner_uid = os.getuid() if runner_uid != 0 and runner_uid != user_.pw_uid: runner = pwd.getpwuid(runner_uid).pw_name - message = "Parameter input error: -U, has to be the same as the executor [%s]" % runner + message = ( + "Parameter input error: -U, has to be the same as the executor [%s]" + % runner + ) logger.error(message) raise ValueError(message) class DBStor: - """ This is DBStor installer. """ + """This is DBStor installer.""" def __init__(self): - """ Constructor for the Installer class. """ + """Constructor for the Installer class.""" logger.info("Begin init...") logger.info("dbstor install runs on python version : %s", gPyVersion) @@ -287,8 +338,8 @@ class DBStor: self.dbstor_fs_vstore_id = "0" self.dbstor_page_fs_vstore_id = "0" self.dbstor_archive_fs_vstore_id = "0" - self.dbstor_home="/opt/cantian/dbstor" - self.dbstor_log="/opt/cantian/log/dbstor" + self.dbstor_home = "/opt/cantian/dbstor" + self.dbstor_log = "/opt/cantian/log/dbstor" def check_ini(self): """ @@ -296,12 +347,16 @@ class DBStor: """ # check the log path if not check_path(self.dbstor_conf_file): - log_exit("Error: There is invalid character in specified dbstor config file.") + log_exit( + "Error: There is invalid character in specified dbstor config file." + ) if os.path.exists(self.dbstor_conf_file): try: os.remove(self.dbstor_conf_file) except OSError as ex: - log_exit("Error: Can not remove dbstor config file: " + self.dbstor_conf_file) + log_exit( + "Error: Can not remove dbstor config file: " + self.dbstor_conf_file + ) def check_log(self): """ @@ -323,58 +378,80 @@ class DBStor: except OSError as ex: log_exit("Error: Can not chmod log file: %s", LOG_FILE) - def read_dbstor_para(self): - with os.fdopen(os.open(JS_CONF_FILE, os.O_RDONLY | os.O_EXCL, stat.S_IWUSR | stat.S_IRUSR), "r") as file_obj: + with os.fdopen( + os.open(JS_CONF_FILE, os.O_RDONLY | os.O_EXCL, stat.S_IWUSR | stat.S_IRUSR), + "r", + ) as file_obj: json_data = json.load(file_obj) - self.dbstor_config['NAMESPACE_FSNAME'] = json_data.get('storage_dbstor_fs', "").strip() - self.dbstor_config['NAMESPACE_PAGE_FSNAME'] = json_data.get('storage_dbstor_page_fs', "").strip() - self.dbstor_config['LOCAL_IP'] = json_data.get('cantian_vlan_ip', "").strip() - self.dbstor_config['REMOTE_IP'] = json_data.get('storage_vlan_ip', "").strip() - self.dbstor_config['NODE_ID'] = json_data.get('node_id', "").strip() - self.dbstor_config['LINK_TYPE'] = json_data.get('link_type', "").strip() - self.dbstor_config['LOG_VSTOR'] = json_data.get('dbstor_fs_vstore_id', "0").strip() - self.dbstor_config['PAGE_VSTOR'] = json_data.get('dbstor_page_fs_vstore_id', "0").strip() - self.dbstor_config['ARCHIVE_VSTOR'] = json_data.get('dbstor_archive_fs_vstore_id', "0").strip() - if json_data.get('link_type', "").strip() != '0': - self.dbstor_config['LINK_TYPE'] = '1' - self.dbstor_config['CLUSTER_ID'] = json_data.get('cluster_id', "").strip() + self.dbstor_config["NAMESPACE_FSNAME"] = json_data.get( + "storage_dbstor_fs", "" + ).strip() + self.dbstor_config["NAMESPACE_PAGE_FSNAME"] = json_data.get( + "storage_dbstor_page_fs", "" + ).strip() + self.dbstor_config["LOCAL_IP"] = json_data.get( + "cantian_vlan_ip", "" + ).strip() + self.dbstor_config["REMOTE_IP"] = json_data.get( + "storage_vlan_ip", "" + ).strip() + self.dbstor_config["NODE_ID"] = json_data.get("node_id", "").strip() + self.dbstor_config["LINK_TYPE"] = json_data.get("link_type", "").strip() + self.dbstor_config["LOG_VSTOR"] = json_data.get( + "dbstor_fs_vstore_id", "0" + ).strip() + self.dbstor_config["PAGE_VSTOR"] = json_data.get( + "dbstor_page_fs_vstore_id", "0" + ).strip() + self.dbstor_config["ARCHIVE_VSTOR"] = json_data.get( + "dbstor_archive_fs_vstore_id", "0" + ).strip() + if json_data.get("link_type", "").strip() != "0": + self.dbstor_config["LINK_TYPE"] = "1" + self.dbstor_config["CLUSTER_ID"] = json_data.get("cluster_id", "").strip() if self.cantian_in_container == "0": - self.dbstor_config['IS_CONTAINER'] = "0" + self.dbstor_config["IS_CONTAINER"] = "0" else: - self.dbstor_config['IS_CONTAINER'] = "1" - self.cluster_name = json_data.get('cluster_name', '') - self.dbstor_config['NAMESPACE_SHARE_FSNAME'] = json_data.get('storage_share_fs', "").strip() - self.dbstor_config['NAMESPACE_ARCHIVE_FSNAME'] = json_data.get('storage_archive_fs', "").strip() + self.dbstor_config["IS_CONTAINER"] = "1" + self.cluster_name = json_data.get("cluster_name", "") + self.dbstor_config["NAMESPACE_SHARE_FSNAME"] = json_data.get( + "storage_share_fs", "" + ).strip() + self.dbstor_config["NAMESPACE_ARCHIVE_FSNAME"] = json_data.get( + "storage_archive_fs", "" + ).strip() def check_dbstor_para(self): logger.info("Checking parameters.") - if len(self.dbstor_config.get('NAMESPACE_FSNAME', "").strip()) == 0: + if len(self.dbstor_config.get("NAMESPACE_FSNAME", "").strip()) == 0: message = "The storage_dbstor_fs parameter is not entered" console_and_log(message) raise ValueError(message) - if len(self.dbstor_config.get('NAMESPACE_PAGE_FSNAME', "").strip()) == 0: + if len(self.dbstor_config.get("NAMESPACE_PAGE_FSNAME", "").strip()) == 0: message = "The storage_dbstor_page_fs parameter is not entered" console_and_log(message) raise ValueError(message) - if len(self.dbstor_config.get('LOCAL_IP', "").strip()) == 0: + if len(self.dbstor_config.get("LOCAL_IP", "").strip()) == 0: message = "The cantian_vlan_ip parameter is not entered" console_and_log(message) raise ValueError(message) - if len(self.dbstor_config.get('REMOTE_IP', "").strip()) == 0: + if len(self.dbstor_config.get("REMOTE_IP", "").strip()) == 0: message = "The storage_vlan_ip parameter is not entered" console_and_log(message) raise ValueError(message) - if len(self.dbstor_config.get('NODE_ID', "").strip()) == 0: + if len(self.dbstor_config.get("NODE_ID", "").strip()) == 0: message = "The node_id parameter is not entered" console_and_log(message) raise ValueError(message) - if len(self.dbstor_config.get('CLUSTER_ID', "").strip()) == 0: + if len(self.dbstor_config.get("CLUSTER_ID", "").strip()) == 0: message = "The cluster_id parameter is not entered" console_and_log(message) raise ValueError(message) elif self.cantian_in_container == "0": - remote_ip_list = re.split(r"[;|]", self.dbstor_config.get('REMOTE_IP', "").strip()) + remote_ip_list = re.split( + r"[;|]", self.dbstor_config.get("REMOTE_IP", "").strip() + ) link_cnt = 0 global DBSTOR_WARN_TYPE for remote_ip in remote_ip_list: @@ -382,30 +459,32 @@ class DBStor: logger.info(cmd) ret_code, stdout, stderr = _exec_popen(cmd) if ret_code: - console_and_log("Failed to ping remote ip. Error: %s" % remote_ip.strip()) + console_and_log( + "Failed to ping remote ip. Error: %s" % remote_ip.strip() + ) DBSTOR_WARN_TYPE += 1 else: link_cnt += 1 if link_cnt == 0: log_exit("Error: failed to ping all remote ip") - if len(self.dbstor_config.get('DPU_UUID', "").strip()) == 0: + if len(self.dbstor_config.get("DPU_UUID", "").strip()) == 0: cmd = "uuidgen" ret_code, stdout, stderr = _exec_popen(cmd) if ret_code: log_exit("Failed to get dpu uuid. Error: %s", stderr) - self.dbstor_config['DPU_UUID'] = stdout.strip() + self.dbstor_config["DPU_UUID"] = stdout.strip() if len(self.cluster_name) == 0: message = "The cluster_name parameter is not entered" console_and_log(message) raise ValueError(message) - if len(self.dbstor_config.get('NAMESPACE_SHARE_FSNAME', "").strip()) == 0: + if len(self.dbstor_config.get("NAMESPACE_SHARE_FSNAME", "").strip()) == 0: message = "The storage_share_fs parameter is not entered" console_and_log(message) raise ValueError(message) - if len(self.dbstor_config.get('NAMESPACE_ARCHIVE_FSNAME', "").strip()) == 0: + if len(self.dbstor_config.get("NAMESPACE_ARCHIVE_FSNAME", "").strip()) == 0: message = "The storage_archive_fs parameter is not entered" console_and_log(message) raise ValueError(message) @@ -417,10 +496,12 @@ class DBStor: """ if len(passwd) < shortest_len or len(passwd) > longest_len: - console_and_log("The length of input must be %s to %s." - % (shortest_len, longest_len)) - raise ValueError("The length of input must be %s to %s." - % (shortest_len, longest_len)) + console_and_log( + "The length of input must be %s to %s." % (shortest_len, longest_len) + ) + raise ValueError( + "The length of input must be %s to %s." % (shortest_len, longest_len) + ) # Can't save with user name upper_cases = set("ABCDEFGHIJKLMNOPQRSTUVWXYZ") lower_cases = set("abcdefghijklmnopqrstuvwxyz") @@ -437,12 +518,14 @@ class DBStor: all_cases = upper_cases | lower_cases | digits | special_cases un_cases = passwd_set - all_cases if un_cases: - console_and_log("Error: There are characters that are not" - " allowed in the password: '%s'" - % "".join(un_cases)) - raise ValueError("Error: There are characters that are not" - " allowed in the password: '%s'" - % "".join(un_cases)) + console_and_log( + "Error: There are characters that are not" + " allowed in the password: '%s'" % "".join(un_cases) + ) + raise ValueError( + "Error: There are characters that are not" + " allowed in the password: '%s'" % "".join(un_cases) + ) logger.info("Successfully written user name.") def verify_dbstor_passwd(self, in_type, passwd, shortest_len, longest_len): @@ -452,23 +535,33 @@ class DBStor: """ # eg 'length in [8-16]' if len(passwd) < shortest_len or len(passwd) > longest_len: - console_and_log("The length of input must be %s to %s." - % (shortest_len, longest_len)) - raise ValueError("The length of input must be %s to %s." - % (shortest_len, longest_len)) + console_and_log( + "The length of input must be %s to %s." % (shortest_len, longest_len) + ) + raise ValueError( + "The length of input must be %s to %s." % (shortest_len, longest_len) + ) # Can't save with user name - user_name = self.dbstor_config.get('USER_NAME', "") + user_name = self.dbstor_config.get("USER_NAME", "") if user_name and passwd == user_name: console_and_log("Error: Password can't be the same as username.") raise ValueError("Error: Password can't be the same as username.") elif user_name and passwd == user_name[::-1]: - console_and_log("Error: Password cannot be the same as username in reverse order") - raise ValueError("Error: Password cannot be the same as username in reverse order") + console_and_log( + "Error: Password cannot be the same as username in reverse order" + ) + raise ValueError( + "Error: Password cannot be the same as username in reverse order" + ) # The same character cannot appear three times consecutively for i in range(0, len(passwd) - 2): if passwd[i] == passwd[i + 1] and passwd[i + 1] == passwd[i + 2]: - console_and_log("Error: The same character cannot appear three times consecutively ") - raise ValueError("Error: The same character cannot appear three times consecutively") + console_and_log( + "Error: The same character cannot appear three times consecutively " + ) + raise ValueError( + "Error: The same character cannot appear three times consecutively" + ) upper_cases = set("ABCDEFGHIJKLMNOPQRSTUVWXYZ") lower_cases = set("abcdefghijklmnopqrstuvwxyz") @@ -487,19 +580,25 @@ class DBStor: if passwd_set & cases: types += 1 if types < 3: - console_and_log("Error: Password must contains at least three different types of characters.") - raise ValueError("Error: Password must contains at least three" - " different types of characters.") + console_and_log( + "Error: Password must contains at least three different types of characters." + ) + raise ValueError( + "Error: Password must contains at least three" + " different types of characters." + ) # Only can contains enumerated cases all_cases = upper_cases | lower_cases | digits | special_cases un_cases = passwd_set - all_cases if un_cases: - console_and_log("Error: There are characters that are not" - " allowed in the password: '%s'" - % "".join(un_cases)) - raise ValueError("Error: There are characters that are not" - " allowed in the password: '%s'" - % "".join(un_cases)) + console_and_log( + "Error: There are characters that are not" + " allowed in the password: '%s'" % "".join(un_cases) + ) + raise ValueError( + "Error: There are characters that are not" + " allowed in the password: '%s'" % "".join(un_cases) + ) def getch(self): file_handle = sys.stdin.fileno() @@ -529,7 +628,9 @@ class DBStor: sys.stdout.flush() password += char - def get_dbstor_usernm_passwd(self, input_prompt, file_prompt, shortest_len, longest_len): + def get_dbstor_usernm_passwd( + self, input_prompt, file_prompt, shortest_len, longest_len + ): """Get new passwd""" flag = 0 new_param = "" @@ -538,10 +639,14 @@ class DBStor: try: if input_prompt == "UserName" and self.cantian_in_container == "0": new_param = input("UserName: ") - self.verify_dbstor_usernm(input_prompt, new_param, shortest_len, longest_len) + self.verify_dbstor_usernm( + input_prompt, new_param, shortest_len, longest_len + ) elif self.cantian_in_container == "0": new_param = input("PassWord: ") - self.verify_dbstor_passwd(input_prompt, new_param, shortest_len, longest_len) + self.verify_dbstor_passwd( + input_prompt, new_param, shortest_len, longest_len + ) break except ValueError as error: logger.error(str(error)) @@ -566,7 +671,9 @@ class DBStor: conf.add_section(SECTION) # 对密码进行加密 if encrypt_passwd and self.cantian_in_container == "0": - self.dbstor_config['PASSWORD'] = GLOBAL_KMC_EXT.encrypt(self.dbstor_config.get('PASSWORD', "")) + self.dbstor_config["PASSWORD"] = GLOBAL_KMC_EXT.encrypt( + self.dbstor_config.get("PASSWORD", "") + ) for key in self.dbstor_config: conf.set(SECTION, key, self.dbstor_config[key]) flags = os.O_CREAT | os.O_RDWR | os.O_TRUNC @@ -579,24 +686,29 @@ class DBStor: """Generate DBstor Config parameter.""" try: logger.info("Get username and password of dbstor config.") - self.dbstor_config['USER_NAME'] = self.get_dbstor_usernm_passwd(input_prompt="UserName", - file_prompt="dbstor config", - shortest_len=6, longest_len=32) - if self.dbstor_config.get('USER_NAME', 0) == 0: + self.dbstor_config["USER_NAME"] = self.get_dbstor_usernm_passwd( + input_prompt="UserName", + file_prompt="dbstor config", + shortest_len=6, + longest_len=32, + ) + if self.dbstor_config.get("USER_NAME", 0) == 0: raise ValueError("create config file failed") - self.dbstor_config['PASSWORD'] = self.get_dbstor_usernm_passwd(input_prompt="PassWord", - file_prompt="dbstor config", - shortest_len=8, longest_len=16) - if self.dbstor_config.get('PASSWORD', 0) == 0: + self.dbstor_config["PASSWORD"] = self.get_dbstor_usernm_passwd( + input_prompt="PassWord", + file_prompt="dbstor config", + shortest_len=8, + longest_len=16, + ) + if self.dbstor_config.get("PASSWORD", 0) == 0: raise ValueError("create config file failed") logger.info("Successfully to get user name and password") logger.info("Generate DBstor Config File.") - self.dbstor_config['DBS_LOG_PATH'] = self.dbstor_log + self.dbstor_config["DBS_LOG_PATH"] = self.dbstor_log self.set_dbstor_conf(self.dbstor_config, self.dbstor_conf_file, True) except ValueError as error: log_exit(str(error)) - def read_old_dbstor_config(self): """read old DBstor Config file.""" logger.info("read old DBstor Config file.") @@ -624,9 +736,15 @@ class DBStor: check backup ini file exists """ if not check_path(self.backup_conf_file): - log_exit("Error: There is invalid character in specified backup dbstor config file.") + log_exit( + "Error: There is invalid character in specified backup dbstor config file." + ) if not os.path.exists(self.backup_conf_file): - log_exit("Error: Backup dbstor config file {} not existed " .format(self.backup_conf_file)) + log_exit( + "Error: Backup dbstor config file {} not existed ".format( + self.backup_conf_file + ) + ) def check_ini_path(self): """ @@ -641,21 +759,24 @@ class DBStor: except ValueError as error: log_exit("Failed to create dbstor config file path. Error: %s", error) - def install(self): self.dbstor_config = self.dbstor_config_tmp self.check_log() - with os.fdopen(os.open(JS_CONF_FILE, os.O_RDONLY | os.O_EXCL, stat.S_IWUSR | stat.S_IRUSR), "r")\ - as file_handle: + with os.fdopen( + os.open(JS_CONF_FILE, os.O_RDONLY | os.O_EXCL, stat.S_IWUSR | stat.S_IRUSR), + "r", + ) as file_handle: json_data = json.load(file_handle) - self.backup = json_data.get('install_type', "override").strip() - self.node_id = json_data.get('node_id', "").strip() - self.cluster_id = json_data.get('cluster_id', "").strip() - self.cantian_in_container = json_data.get('cantian_in_container', "0").strip() - self.dbstor_fs_vstore_id = json_data.get('dbstor_fs_vstore_id', "0").strip() + self.backup = json_data.get("install_type", "override").strip() + self.node_id = json_data.get("node_id", "").strip() + self.cluster_id = json_data.get("cluster_id", "").strip() + self.cantian_in_container = json_data.get( + "cantian_in_container", "0" + ).strip() + self.dbstor_fs_vstore_id = json_data.get("dbstor_fs_vstore_id", "0").strip() self.conf_file_path = "/opt/cantian/dbstor/tools" self.backup_conf_file = os.path.join(BACKUP_CONF_FILE, "dbstor_config.ini") - self.cluster_name = json_data.get("cluster_name", '') + self.cluster_name = json_data.get("cluster_name", "") self.check_ini_path() self.dbstor_conf_file = os.path.join(self.conf_file_path, "dbstor_config.ini") @@ -666,16 +787,22 @@ class DBStor: self.read_dbstor_para() self.check_dbstor_para() self.generate_db_config() - with os.fdopen(os.open(JS_CONF_FILE, os.O_RDONLY | os.O_EXCL, stat.S_IWUSR | stat.S_IRUSR), "r") as file_obj: + with os.fdopen( + os.open(JS_CONF_FILE, os.O_RDONLY | os.O_EXCL, stat.S_IWUSR | stat.S_IRUSR), + "r", + ) as file_obj: json_data = json.load(file_obj) deploy_mode = json_data.get("deploy_mode") if deploy_mode == "dbstor" or deploy_mode == "combined": configTool = ConfigTool() configTool.create_unify_dbstor_config() - + def cp_ini_to_client_test(self): self.dbstor_conf_file - cmd = "cp -rf %s/dbstor_config.ini /opt/cantian/dbstor/tools/dbstor_config.ini" % self.conf_file_path.strip() + cmd = ( + "cp -rf %s/dbstor_config.ini /opt/cantian/dbstor/tools/dbstor_config.ini" + % self.conf_file_path.strip() + ) logger.info(cmd) ret_code, stdout, stderr = _exec_popen(cmd) if ret_code: diff --git a/pkg/deploy/action/dbstor/dbstor_uninstall.py b/pkg/deploy/action/dbstor/dbstor_uninstall.py index 290d4a85..6da0f1b0 100644 --- a/pkg/deploy/action/dbstor/dbstor_uninstall.py +++ b/pkg/deploy/action/dbstor/dbstor_uninstall.py @@ -31,6 +31,7 @@ TIMEOUT_COUNT = 1800 class ReadConfigParserNoCast(ConfigParser): "Inherit from built-in class: ConfigParser" + def optionxform(self, optionstr): "Rewrite without lower()" return optionstr @@ -40,6 +41,7 @@ class Options(object): """ class for command line options """ + def __init__(self): self.install_user_privilege = "withoutroot" self.dbstor_log_path = "/opt/cantian/dbstor" @@ -47,7 +49,9 @@ class Options(object): self.log_file = "/opt/cantian/log/dbstor/uninstall.log" self.ini_file = "/opt/cantian/dbstor/tools/dbstor_config.ini" self.docker_ini_file = "/home/regress/cantian_data" - self.js_conf_file = os.path.join(os.path.dirname(os.path.abspath(__file__)), "../../config/deploy_param.json") + self.js_conf_file = os.path.join( + os.path.dirname(os.path.abspath(__file__)), "../../config/deploy_param.json" + ) self.note_id = "" self.ini_exist = False self.share_logic_ip = "" @@ -59,11 +63,13 @@ g_opts = Options() gPyVersion = platform.python_version() logger = logging.getLogger() logger.setLevel(logging.DEBUG) -logger_handle = logging.FileHandler(g_opts.log_file, 'a', "utf-8") +logger_handle = logging.FileHandler(g_opts.log_file, "a", "utf-8") logger_handle.setLevel(logging.DEBUG) -logger_formatter = logging.Formatter('[%(asctime)s]-[%(filename)s]-[line:%(lineno)d]-[%(levelname)s]-' - '%(message)s-[%(process)s]') +logger_formatter = logging.Formatter( + "[%(asctime)s]-[%(filename)s]-[line:%(lineno)d]-[%(levelname)s]-" + "%(message)s-[%(process)s]" +) logger_handle.setFormatter(logger_formatter) logger.addHandler(logger_handle) logger.info("init logging success") @@ -78,8 +84,13 @@ def _exec_popen(cmd, values=None): if not values: values = [] bash_cmd = ["bash"] - p_obj = subprocess.Popen(bash_cmd, shell=False, stdin=subprocess.PIPE, - stdout=subprocess.PIPE, stderr=subprocess.PIPE) + p_obj = subprocess.Popen( + bash_cmd, + shell=False, + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + ) if gPyVersion[0] == "3": p_obj.stdin.write(cmd.encode()) @@ -122,8 +133,10 @@ def log_exit(msg): :return: NA """ console_and_log("Error: " + msg) - print("Please refer to install log \"%s\" for more detailed information." - % g_opts.log_file) + print( + 'Please refer to install log "%s" for more detailed information.' + % g_opts.log_file + ) if g_opts.force_uninstall != "force": raise ValueError(str(msg)) @@ -171,17 +184,23 @@ def clean_ini(): try: os.remove(g_opts.ini_file) except OSError as error_msg: - log_exit("Clean dbstor config: can not delete dbstor_config.ini " - "%s\nPlease manually delete it." % str(error_msg)) + log_exit( + "Clean dbstor config: can not delete dbstor_config.ini " + "%s\nPlease manually delete it." % str(error_msg) + ) def read_file_path(): - with os.fdopen(os.open(g_opts.js_conf_file, os.O_RDONLY | os.O_EXCL, stat.S_IWUSR | stat.S_IRUSR), "r")\ - as file_handle: + with os.fdopen( + os.open( + g_opts.js_conf_file, os.O_RDONLY | os.O_EXCL, stat.S_IWUSR | stat.S_IRUSR + ), + "r", + ) as file_handle: json_data = json.load(file_handle) - g_opts.note_id = json_data.get('node_id', "").strip() - g_opts.share_logic_ip = json_data.get('share_logic_ip', "").strip() - g_opts.cluster_name = json_data.get('cluster_name', "").strip() + g_opts.note_id = json_data.get("node_id", "").strip() + g_opts.share_logic_ip = json_data.get("share_logic_ip", "").strip() + g_opts.cluster_name = json_data.get("cluster_name", "").strip() g_opts.ini_file = "/opt/cantian/dbstor/tools/dbstor_config.ini" @@ -200,8 +219,10 @@ def main(): check_ini() if g_opts.ini_exist: clean_ini() - console_and_log("dbstor config was successfully removed from your computer, " - "for more message please see %s." % g_opts.log_file) + console_and_log( + "dbstor config was successfully removed from your computer, " + "for more message please see %s." % g_opts.log_file + ) if __name__ == "__main__": diff --git a/pkg/deploy/action/dbstor/init_unify_config.py b/pkg/deploy/action/dbstor/init_unify_config.py index ee2a56a4..4391ada5 100644 --- a/pkg/deploy/action/dbstor/init_unify_config.py +++ b/pkg/deploy/action/dbstor/init_unify_config.py @@ -10,11 +10,15 @@ from configparser import ConfigParser LOG_PATH = "/opt/cantian/log/dbstor" LOG_FILE = "/opt/cantian/log/dbstor/install.log" -JS_CONF_FILE = os.path.join(os.path.dirname(os.path.abspath(__file__)), "../../config/deploy_param.json") +JS_CONF_FILE = os.path.join( + os.path.dirname(os.path.abspath(__file__)), "../../config/deploy_param.json" +) DBSTOR_CONF_FILE = "/mnt/dbdata/remote/share_" -CONTAINER_DBSTOR_CONF_DIR = os.path.join(os.path.dirname(os.path.abspath(__file__)), "../../config/container") +CONTAINER_DBSTOR_CONF_DIR = os.path.join( + os.path.dirname(os.path.abspath(__file__)), "../../config/container" +) DOCKER_DBSTOR_CONF_FILE = "/home/regress/cantian_data" -SECTION = 'CLIENT' +SECTION = "CLIENT" MAX_DIRECTORY_MODE = 0o755 PYTHON242 = "2.4.2" PYTHON25 = "2.5" @@ -24,14 +28,17 @@ gPyVersion = platform.python_version() logger = logging.getLogger() logger.setLevel(logging.DEBUG) -logger_handle = logging.FileHandler(LOG_FILE, 'a', "utf-8") +logger_handle = logging.FileHandler(LOG_FILE, "a", "utf-8") logger_handle.setLevel(logging.DEBUG) -logger_formatter = logging.Formatter('[%(asctime)s]-[%(filename)s]-[line:%(lineno)d]-[%(levelname)s]-' - '%(message)s-[%(process)s]') +logger_formatter = logging.Formatter( + "[%(asctime)s]-[%(filename)s]-[line:%(lineno)d]-[%(levelname)s]-" + "%(message)s-[%(process)s]" +) logger_handle.setFormatter(logger_formatter) logger.addHandler(logger_handle) + def _exec_popen(cmd, values=None): """ subprocess.Popen in python2 and 3. @@ -41,8 +48,13 @@ def _exec_popen(cmd, values=None): if not values: values = [] bash_cmd = ["bash"] - p_obj = subprocess.Popen(bash_cmd, shell=False, stdin=subprocess.PIPE, - stdout=subprocess.PIPE, stderr=subprocess.PIPE) + p_obj = subprocess.Popen( + bash_cmd, + shell=False, + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + ) if gPyVersion[0] == "3": p_obj.stdin.write(cmd.encode()) @@ -67,6 +79,7 @@ def _exec_popen(cmd, values=None): stderr = stderr[:-1] return p_obj.returncode, stdout, stderr + class ConfigTool: def __init__(self) -> None: self.conf_file_path = "" @@ -79,8 +92,8 @@ class ConfigTool: self.dbstor_fs_vstore_id = "0" self.dbstor_page_fs_vstore_id = "0" self.dbstor_archive_fs_vstore_id = "0" - self.dbstor_home="/opt/cantian/dbstor" - self.dbstor_log_path="/opt/cantian/log/dbstor" + self.dbstor_home = "/opt/cantian/dbstor" + self.dbstor_log_path = "/opt/cantian/log/dbstor" self.dbstor_config_tmp = { # dbstor_config.ini default parameters "NAMESPACE_FSNAME": "", "NAMESPACE_PAGE_FSNAME": "", @@ -96,64 +109,85 @@ class ConfigTool: "NAMESPACE_ARCHIVE_FSNAME": "", } self.dbstor_config = self.dbstor_config_tmp - with os.fdopen(os.open(JS_CONF_FILE, os.O_RDONLY | os.O_EXCL, stat.S_IWUSR | stat.S_IRUSR), "r")\ - as file_handle: + with os.fdopen( + os.open(JS_CONF_FILE, os.O_RDONLY | os.O_EXCL, stat.S_IWUSR | stat.S_IRUSR), + "r", + ) as file_handle: json_data = json.load(file_handle) - self.node_id = json_data.get('node_id', "").strip() - self.cluster_id = json_data.get('cluster_id', "").strip() - self.cantian_in_container = json_data.get('cantian_in_container', "0").strip() - self.dbstor_fs_vstore_id = json_data.get('dbstor_fs_vstore_id', "0").strip() - self.cluster_name = json_data.get("cluster_name", '') - + self.node_id = json_data.get("node_id", "").strip() + self.cluster_id = json_data.get("cluster_id", "").strip() + self.cantian_in_container = json_data.get( + "cantian_in_container", "0" + ).strip() + self.dbstor_fs_vstore_id = json_data.get("dbstor_fs_vstore_id", "0").strip() + self.cluster_name = json_data.get("cluster_name", "") + def create_unify_dbstor_config(self): - logger.info('Deploy_mode = dbstor, begin to set config.') + logger.info("Deploy_mode = dbstor, begin to set config.") config = ConfigParser() config.optionxform = str conf_file_path = "/opt/cantian/dbstor/tools" dbstor_conf_file = os.path.join(conf_file_path, "dbstor_config.ini") config.read(dbstor_conf_file) - split_env = os.environ['LD_LIBRARY_PATH'].split(":") - filtered_env = [single_env for single_env in split_env if "/opt/cantian/dbstor/lib" not in single_env] - os.environ['LD_LIBRARY_PATH'] = ":".join(filtered_env) - for i in range(7,11): + split_env = os.environ["LD_LIBRARY_PATH"].split(":") + filtered_env = [ + single_env + for single_env in split_env + if "/opt/cantian/dbstor/lib" not in single_env + ] + os.environ["LD_LIBRARY_PATH"] = ":".join(filtered_env) + for i in range(7, 11): file_num = i - 6 cmd = "python3 %s/../obtains_lsid.py %s %s %s %s" - ret_code, stdout, stderr = _exec_popen(cmd % ("/opt/cantian/action/dbstor", 2, self.cluster_id, i, self.node_id)) + ret_code, stdout, stderr = _exec_popen( + cmd + % ("/opt/cantian/action/dbstor", 2, self.cluster_id, i, self.node_id) + ) if ret_code: - raise OSError("Failed to execute LSIDGenerate." - " Error: %s" % (stderr + os.linesep + stderr)) + raise OSError( + "Failed to execute LSIDGenerate." + " Error: %s" % (stderr + os.linesep + stderr) + ) data = stdout.split("\n") if len(data) == 2: inst_id, dbs_tool_uuid = data[0], data[1] else: raise ValueError("Data parse error: length of parsed data is not 2.") - logger.info('Generate inst_id, dbs_tool_uuid success.') - ret_code, stdout, stderr = _exec_popen(cmd % ("/opt/cantian/action/dbstor", 0, self.cluster_id, 0, 0)) + logger.info("Generate inst_id, dbs_tool_uuid success.") + ret_code, stdout, stderr = _exec_popen( + cmd % ("/opt/cantian/action/dbstor", 0, self.cluster_id, 0, 0) + ) if ret_code: - raise OSError("Failed to execute LSIDGenerate." - " Error: %s" % (stderr + os.linesep + stderr)) + raise OSError( + "Failed to execute LSIDGenerate." + " Error: %s" % (stderr + os.linesep + stderr) + ) data = stdout.split("\n") if len(data) == 2: self.cluster_uuid = data[1] else: raise ValueError("Data parse error: length of parsed data is not 2.") - logger.info('Generate cluster_uuid success.') + logger.info("Generate cluster_uuid success.") folder_path = "%s/conf/dbs/" % (self.dbstor_home) if not os.path.exists(folder_path): os.makedirs(folder_path) - config.set('CLIENT', 'DBSTOR_OWNER_NAME', 'dbstor') - config.set('CLIENT', 'CLUSTER_NAME', str(self.cluster_name)) - config.set('CLIENT', 'CLUSTER_UUID', str(self.cluster_uuid)) - config.set('CLIENT', 'INST_ID', inst_id) - config.set('CLIENT', 'DBS_TOOL_UUID', dbs_tool_uuid) - config.set('CLIENT', 'DBS_LOG_PATH', self.dbstor_log_path) + config.set("CLIENT", "DBSTOR_OWNER_NAME", "dbstor") + config.set("CLIENT", "CLUSTER_NAME", str(self.cluster_name)) + config.set("CLIENT", "CLUSTER_UUID", str(self.cluster_uuid)) + config.set("CLIENT", "INST_ID", inst_id) + config.set("CLIENT", "DBS_TOOL_UUID", dbs_tool_uuid) + config.set("CLIENT", "DBS_LOG_PATH", self.dbstor_log_path) flags = os.O_CREAT | os.O_RDWR | os.O_TRUNC modes = stat.S_IWUSR | stat.S_IRUSR - file_path = "%s/conf/dbs/dbstor_config_tool_%s.ini" % (self.dbstor_home, str(file_num)) + file_path = "%s/conf/dbs/dbstor_config_tool_%s.ini" % ( + self.dbstor_home, + str(file_num), + ) with os.fdopen(os.open(file_path, flags, modes), "w") as file_obj: config.write(file_obj) - logger.info('Set config success.') + logger.info("Set config success.") + -if __name__ == '__main__': +if __name__ == "__main__": configTool = ConfigTool() - configTool.create_unify_dbstor_config() \ No newline at end of file + configTool.create_unify_dbstor_config() diff --git a/pkg/deploy/action/dbstor/kmc_adapter.py b/pkg/deploy/action/dbstor/kmc_adapter.py index a005f971..2129771f 100644 --- a/pkg/deploy/action/dbstor/kmc_adapter.py +++ b/pkg/deploy/action/dbstor/kmc_adapter.py @@ -7,7 +7,16 @@ import logging import ctypes import json import base64 -from ctypes import CFUNCTYPE, c_char, c_int, c_void_p, pointer, c_char_p, Structure, POINTER +from ctypes import ( + CFUNCTYPE, + c_char, + c_int, + c_void_p, + pointer, + c_char_p, + Structure, + POINTER, +) from enum import Enum """ @@ -21,20 +30,17 @@ SEC_PATH_MAX = 4096 # PATH_MAX: 待确认linux环境下大小 MAX_MK_COUNT = 4096 USE_DBSTOR = ["combined", "dbstor"] -JS_CONF_FILE = os.path.join(os.path.dirname(os.path.abspath(__file__)), "../../config/deploy_param.json") +JS_CONF_FILE = os.path.join( + os.path.dirname(os.path.abspath(__file__)), "../../config/deploy_param.json" +) class KeCallBackParam(Structure): - _fields_ = [ - ("notifyCbCtx", c_void_p), - ("loggerCtx", c_void_p), - ("hwCtx", c_void_p)] + _fields_ = [("notifyCbCtx", c_void_p), ("loggerCtx", c_void_p), ("hwCtx", c_void_p)] class KmcHardWareParm(Structure): - _fields_ = [ - ("len", c_int), - ("hardParam", c_char_p)] + _fields_ = [("len", c_int), ("hardParam", c_char_p)] class KmcConfig(Structure): @@ -52,7 +58,8 @@ class KmcConfig(Structure): ("innerHmacAlgId", c_int), ("innerKdfAlgId", c_int), ("workKeyIter", c_int), - ("rootKeyIter", c_int)] + ("rootKeyIter", c_int), + ] def __init__(self, *args, **kw): super(KmcConfig, self).__init__(*args, **kw) @@ -79,7 +86,8 @@ class KmcConfigEx(Structure): ("kmcHardWareParm", KmcHardWareParm), ("keCbParam", POINTER(KeCallBackParam)), ("kmcConfig", KmcConfig), - ("useDefaultHwCB", c_int)] + ("useDefaultHwCB", c_int), + ] class HmacAlgorithm(Enum): @@ -122,7 +130,7 @@ class KmcRole(Enum): @CFUNCTYPE(None, c_void_p, c_int, c_char_p) def kmc_log(ctx, level, _msg): - msg = str(_msg, encoding='utf-8') + msg = str(_msg, encoding="utf-8") if level == KmcLogLevel.LOG_ERROR.value: logging.error(msg) elif level == KmcLogLevel.LOG_WARN.value: @@ -143,15 +151,21 @@ class KmcLogLevel(Enum): class KmcInitConf(object): - def __init__(self, _primary_ks, _standby_ks, _hw_param=None, - _enable_hw=False, - _use_default_hw_cb=0, - _domain_count=2, - _role=KmcRole.ROLE_MASTER, - _sdp_alg=SdpAlgorithm.AES256_GCM, - _hmac_alg=HmacAlgorithm.HMAC_SHA256, - _kdf_alg=KdfAlgorithm.PBKDF2_HMAC_SHA256, - _hash_alg=HashAlgorithm.HASH_SHA256, **kwargs): + def __init__( + self, + _primary_ks, + _standby_ks, + _hw_param=None, + _enable_hw=False, + _use_default_hw_cb=0, + _domain_count=2, + _role=KmcRole.ROLE_MASTER, + _sdp_alg=SdpAlgorithm.AES256_GCM, + _hmac_alg=HmacAlgorithm.HMAC_SHA256, + _kdf_alg=KdfAlgorithm.PBKDF2_HMAC_SHA256, + _hash_alg=HashAlgorithm.HASH_SHA256, + **kwargs + ): self.primary_ks = _primary_ks self.standby_ks = _standby_ks self.hw_param = _hw_param @@ -184,19 +198,23 @@ class CApiConstant: class CApiWrapper(object): - """ python调用kmc-ext接口封装器 + """python调用kmc-ext接口封装器 - 设计说明： 该类仅处理python到C语言类型转换的功能，不包括任何业务逻辑 + 设计说明： 该类仅处理python到C语言类型转换的功能，不包括任何业务逻辑 - 约束： - 1. 不支持硬件加密初始化 - 2. 不支持日志接口注册 - 3. 不支持密钥更新接口注册 + 约束： + 1. 不支持硬件加密初始化 + 2. 不支持日志接口注册 + 3. 不支持密钥更新接口注册 """ - KMC_EXT_LIB_PATH = '/opt/cantian/dbstor/lib/libkmcext.so' - def __init__(self, primary_keystore="/opt/cantian/common/config/primary_keystore.ks", - standby_keystore="/opt/cantian/common/config/standby_keystore.ks"): + KMC_EXT_LIB_PATH = "/opt/cantian/dbstor/lib/libkmcext.so" + + def __init__( + self, + primary_keystore="/opt/cantian/common/config/primary_keystore.ks", + standby_keystore="/opt/cantian/common/config/standby_keystore.ks", + ): self.deploy_mode = None self.initialized = False self.kmc_ctx = None @@ -207,9 +225,12 @@ class CApiWrapper(object): self.kmc_ext = ctypes.cdll.LoadLibrary(self.KMC_EXT_LIB_PATH) def get_dbstor_para(self): - with os.fdopen(os.open(JS_CONF_FILE, os.O_RDONLY | os.O_EXCL, stat.S_IWUSR | stat.S_IRUSR), "r") as file_obj: + with os.fdopen( + os.open(JS_CONF_FILE, os.O_RDONLY | os.O_EXCL, stat.S_IWUSR | stat.S_IRUSR), + "r", + ) as file_obj: json_data = json.load(file_obj) - self.deploy_mode = json_data.get('deploy_mode', "").strip() + self.deploy_mode = json_data.get("deploy_mode", "").strip() def initialize(self): if self.initialized or self.deploy_mode not in USE_DBSTOR: @@ -244,8 +265,13 @@ class CApiWrapper(object): kmc_conf.innerKdfAlgId = c_int(conf.kdf_alg.value) kmc_conf.workKeyIter = c_int(CApiConstant.DEFAULT_WORK_KEY_ITER) kmc_conf.rootKeyIter = c_int(CApiConstant.DEFAULT_ROOT_KEY_ITER) - cfg = KmcConfigEx(c_int(conf.enable_hw), kmc_hardware_param, - pointer(ke_cb_param), kmc_conf, c_int(use_default_hw_cb)) + cfg = KmcConfigEx( + c_int(conf.enable_hw), + kmc_hardware_param, + pointer(ke_cb_param), + kmc_conf, + c_int(use_default_hw_cb), + ) ctx = c_void_p() # 初始化日志 @@ -271,16 +297,21 @@ class CApiWrapper(object): # 函数原型: int KeEncryptByDomainEx(const void *ctx, unsigned int domainID, # const char *plainText, int plainTextLen, # char **cipherText, int *cipherTextLen) - status = self.kmc_ext.KeEncryptByDomainEx(ctx, CApiConstant.DEFAULT_DOMAINID, - plain_text, plain_len, - pointer(cipher), pointer(cipher_len)) + status = self.kmc_ext.KeEncryptByDomainEx( + ctx, + CApiConstant.DEFAULT_DOMAINID, + plain_text, + plain_len, + pointer(cipher), + pointer(cipher_len), + ) if status == 0: - result = cipher.value.decode('utf-8') + result = cipher.value.decode("utf-8") return result def encrypt_by_base64(self, plain): - encoded = base64.b64encode(plain.encode('utf-8')) - return encoded.decode('utf-8') + encoded = base64.b64encode(plain.encode("utf-8")) + return encoded.decode("utf-8") def encrypt(self, plain): if self.deploy_mode in USE_DBSTOR: @@ -300,16 +331,21 @@ class CApiWrapper(object): # 函数原型: int KeDecryptByDomainEx(const void *ctx, unsigned int domainID, # const char *cipherText, int cipherTextLen, # char **plainText, int *plainTextLen) - status = self.kmc_ext.KeDecryptByDomainEx(ctx, CApiConstant.DEFAULT_DOMAINID, - cipher_text, cipher_len, - pointer(plain), pointer(plain_len)) + status = self.kmc_ext.KeDecryptByDomainEx( + ctx, + CApiConstant.DEFAULT_DOMAINID, + cipher_text, + cipher_len, + pointer(plain), + pointer(plain_len), + ) if status == 0: - result = plain.value.decode('utf-8') + result = plain.value.decode("utf-8") return result def decrypt_by_base64(self, cipher): - decoded = base64.b64decode(cipher.encode('utf-8')) - return decoded.decode('utf-8') + decoded = base64.b64decode(cipher.encode("utf-8")) + return decoded.decode("utf-8") def decrypt(self, cipher): if self.deploy_mode in USE_DBSTOR: @@ -324,7 +360,9 @@ class CApiWrapper(object): return self.kmc_ext.KeFinalizeEx(pointer(self.kmc_ctx)) def update_mk(self): - return self.kmc_ext.KeActiveNewKeyEx(self.kmc_ctx, CApiConstant.DEFAULT_DOMAINID) + return self.kmc_ext.KeActiveNewKeyEx( + self.kmc_ctx, CApiConstant.DEFAULT_DOMAINID + ) def get_mk_count(self): return self.kmc_ext.KeGetMkCountEx(self.kmc_ctx) @@ -335,6 +373,7 @@ class CApiWrapper(object): if __name__ == "__main__": import sys + e_pwd = "" d_pwd = "" mode = "" @@ -362,7 +401,10 @@ if __name__ == "__main__": mk_count = res.get_mk_count() res.finalize() if ret == 0: - print("update master key success\nNow there are %s master keys in ksf" % mk_count) + print( + "update master key success\nNow there are %s master keys in ksf" + % mk_count + ) exit(0) else: exit(1) @@ -374,10 +416,10 @@ if __name__ == "__main__": exit(0) else: exit(1) - + if mode == "encrypted": e_pwd = sys.argv[4] encrypted_passwd = res.encrypt(e_pwd) print(encrypted_passwd) - - res.finalize() \ No newline at end of file + + res.finalize() diff --git a/pkg/deploy/action/dbstor/update_dbstor_config.py b/pkg/deploy/action/dbstor/update_dbstor_config.py index 524c4ba1..0f1c9c62 100644 --- a/pkg/deploy/action/dbstor/update_dbstor_config.py +++ b/pkg/deploy/action/dbstor/update_dbstor_config.py @@ -1,5 +1,6 @@ import re import sys + sys.dont_write_bytecode = True try: @@ -17,6 +18,7 @@ try: import copy from configparser import ConfigParser from kmc_adapter import CApiWrapper + PYTHON242 = "2.4.2" PYTHON25 = "2.5" gPyVersion = platform.python_version() @@ -26,8 +28,9 @@ try: elif gPyVersion >= PYTHON25: pass else: - raise ImportError("This install script can not support python version: %s" - % gPyVersion) + raise ImportError( + "This install script can not support python version: %s" % gPyVersion + ) except ImportError as import_err: raise ValueError("Unable to import module: %s." % str(import_err)) from import_err @@ -40,6 +43,7 @@ MAX_DIRECTORY_MODE = 750 class ReadConfigParserNoCast(ConfigParser): "Inherit from built-in class: ConfigParser" + def optionxform(self, optionstr): "Rewrite without lower()" return optionstr @@ -49,30 +53,38 @@ class Options(object): """ command line options """ + def __init__(self): - self.section = 'CLIENT' + self.section = "CLIENT" self.dbstor_config = {} self.inipath = "/mnt/dbdata/remote/share_" self.log_path = "/opt/cantian/log/dbstor" self.log_file = "/opt/cantian/log/dbstor/update.log" - self.js_conf_file = os.path.join(os.path.dirname(os.path.abspath(__file__)), "../../config/deploy_param.json") + self.js_conf_file = os.path.join( + os.path.dirname(os.path.abspath(__file__)), "../../config/deploy_param.json" + ) self.note_id = "" - self.local_ini_path = "/mnt/dbdata/local/cantian/tmp/data/dbstor/conf/dbs/dbstor_config.ini" + self.local_ini_path = ( + "/mnt/dbdata/local/cantian/tmp/data/dbstor/conf/dbs/dbstor_config.ini" + ) self.cstool_ini_path = "/opt/cantian/dbstor/conf/dbs/dbstor_config.ini" self.tools_ini_path = "/opt/cantian/dbstor/tools/dbstor_config.ini" self.cms_ini_path = "/opt/cantian/cms/dbstor/conf/dbs/dbstor_config.ini" self.cluster_name = "" + db_opts = Options() if not os.path.exists(db_opts.log_path): os.makedirs(db_opts.log_path, MAX_DIRECTORY_MODE) logger = logging.getLogger() logger.setLevel(logging.DEBUG) -logger_handle = logging.FileHandler(db_opts.log_file, 'a', "utf-8") +logger_handle = logging.FileHandler(db_opts.log_file, "a", "utf-8") logger_handle.setLevel(logging.DEBUG) -logger_formatter = logging.Formatter('[%(asctime)s]-[%(filename)s]-[line:%(lineno)d]-[%(levelname)s]-' - '%(message)s-[%(process)s]') +logger_formatter = logging.Formatter( + "[%(asctime)s]-[%(filename)s]-[line:%(lineno)d]-[%(levelname)s]-" + "%(message)s-[%(process)s]" +) logger_handle.setFormatter(logger_formatter) logger.addHandler(logger_handle) logger.info("init logging success") @@ -95,8 +107,10 @@ def log_exit(msg): :return: NA """ console_and_log("Error: " + msg) - print("Please refer to install log \"%s\" for more detailed information." - % db_opts.log_file) + print( + 'Please refer to install log "%s" for more detailed information.' + % db_opts.log_file + ) raise ValueError(str(msg)) @@ -110,7 +124,9 @@ def check_log(): modes = stat.S_IWUSR | stat.S_IRUSR | stat.S_IRGRP if not os.path.exists(db_opts.log_file): try: - with os.fdopen(os.open(db_opts.log_file, flags, modes), "w", encoding="utf-8"): + with os.fdopen( + os.open(db_opts.log_file, flags, modes), "w", encoding="utf-8" + ): pass except IOError as ex: log_exit("Error: Can not create or open log file: %s", db_opts.log_file) @@ -130,8 +146,13 @@ def _exec_popen(cmd, values=None): if not values: values = [] bash_cmd = ["bash"] - pobj = subprocess.Popen(bash_cmd, shell=False, stdin=subprocess.PIPE, - stdout=subprocess.PIPE, stderr=subprocess.PIPE) + pobj = subprocess.Popen( + bash_cmd, + shell=False, + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + ) if gPyVersion[0] == "3": pobj.stdin.write(cmd.encode()) @@ -180,7 +201,7 @@ def usage(): def read_default_parameter(): - #Read default parameters + # Read default parameters conf = ReadConfigParserNoCast() conf.read(db_opts.inipath, encoding="utf-8") for option in conf.options(db_opts.section): @@ -203,8 +224,21 @@ def parse_parameter(): try: # Parameters are passed into argv. After parsing, they are stored # in opts as binary tuples. Unresolved parameters are stored in args. - opts, args = getopt.getopt(sys.argv[1:], "", - ["help", "NAMESPACE_FSNAME=", "NAMESPACE_PAGE_FSNAME=", "DPU_UUID=", "LINK_TYPE=", "LOCAL_IP=", "REMOTE_IP=", "NAMESPACE_SHARE_FSNAME=", "NAMESPACE_ARCHIVE_FSNAME="]) + opts, args = getopt.getopt( + sys.argv[1:], + "", + [ + "help", + "NAMESPACE_FSNAME=", + "NAMESPACE_PAGE_FSNAME=", + "DPU_UUID=", + "LINK_TYPE=", + "LOCAL_IP=", + "REMOTE_IP=", + "NAMESPACE_SHARE_FSNAME=", + "NAMESPACE_ARCHIVE_FSNAME=", + ], + ) if args: log_exit("Parameter input error: " + str(args[0])) @@ -219,21 +253,29 @@ def parse_parameter(): if _value.strip() == "": continue if _key == "--NAMESPACE_FSNAME": - db_opts.dbstor_config["NAMESPACE_FSNAME"] = _value.strip().replace('/', ';') + db_opts.dbstor_config["NAMESPACE_FSNAME"] = _value.strip().replace( + "/", ";" + ) if _key == "--NAMESPACE_PAGE_FSNAME": - db_opts.dbstor_config["NAMESPACE_PAGE_FSNAME"] = _value.strip().replace('/', ';') + db_opts.dbstor_config["NAMESPACE_PAGE_FSNAME"] = _value.strip().replace( + "/", ";" + ) elif _key == "--DPU_UUID": db_opts.dbstor_config["DPU_UUID"] = _value.strip() elif _key == "--LINK_TYPE": db_opts.dbstor_config["LINK_TYPE"] = _value.strip() elif _key == "--LOCAL_IP": - db_opts.dbstor_config["LOCAL_IP"] = _value.strip().replace('/', ';') + db_opts.dbstor_config["LOCAL_IP"] = _value.strip().replace("/", ";") elif _key == "--REMOTE_IP": - db_opts.dbstor_config["REMOTE_IP"] = _value.strip().replace('/', ';') + db_opts.dbstor_config["REMOTE_IP"] = _value.strip().replace("/", ";") elif _key == "--NAMESPACE_SHARE_FSNAME": - db_opts.dbstor_config["NAMESPACE_SHARE_FSNAME"] = _value.strip().replace('/', ';') + db_opts.dbstor_config["NAMESPACE_SHARE_FSNAME"] = ( + _value.strip().replace("/", ";") + ) elif _key == "--NAMESPACE_ARCHIVE_FSNAME": - db_opts.dbstor_config["NAMESPACE_ARCHIVE_FSNAME"] = _value.strip().replace('/', ';') + db_opts.dbstor_config["NAMESPACE_ARCHIVE_FSNAME"] = ( + _value.strip().replace("/", ";") + ) except getopt.GetoptError as error: log_exit("Parameter input error: " + error.msg) @@ -255,7 +297,9 @@ def check_parameter(): if len(db_opts.dbstor_config.get("NAMESPACE_ARCHIVE_FSNAME", "").strip()) == 0: log_exit("The storage_archive_fs parameter is not entered") else: - remote_ip_list = re.split(r"[;|,]", db_opts.dbstor_config.get("REMOTE_IP", "").strip()) + remote_ip_list = re.split( + r"[;|,]", db_opts.dbstor_config.get("REMOTE_IP", "").strip() + ) for remote_ip in remote_ip_list: cmd = "ping -c 1 %s" % remote_ip.strip() logger.info("exec cmd: %s", cmd) @@ -274,13 +318,13 @@ def clean_ini(file_path): def update_file_parameter(file_path, ini_file, encrypt_passwd=False): - #Update parameters + # Update parameters clean_ini(file_path) conf = ReadConfigParserNoCast() conf.add_section(db_opts.section) if encrypt_passwd: - ini_file['PASSWORD'] = GLOBAL_KMC_EXT.encrypt(ini_file.get('PASSWORD', "")) + ini_file["PASSWORD"] = GLOBAL_KMC_EXT.encrypt(ini_file.get("PASSWORD", "")) # rewrite parameters for key in ini_file: conf.set(db_opts.section, key, ini_file[key]) @@ -294,15 +338,15 @@ def update_file_parameter(file_path, ini_file, encrypt_passwd=False): def update_parameter(): - #Update parameters + # Update parameters console_and_log("Start to update parameters") cantian_dbstor_config = copy.deepcopy(db_opts.dbstor_config) cantian_dbstor_config["DBSTOR_OWNER_NAME"] = "cantian" cms_dbstor_config = copy.deepcopy(db_opts.dbstor_config) cms_dbstor_config["DBSTOR_OWNER_NAME"] = "cms" - #update share ini file + # update share ini file update_file_parameter(db_opts.inipath, db_opts.dbstor_config) - #update local ini file + # update local ini file update_file_parameter(db_opts.local_ini_path, cantian_dbstor_config) # update cstool ini file update_file_parameter(db_opts.cstool_ini_path, db_opts.dbstor_config) @@ -319,6 +363,7 @@ def update_parameter(): if ret_code: log_exit(f"Failed to execute command: {cmd}.") + def verify_dbstor_usernm(in_type, passwd, shortest_len, longest_len): """ Verify new password. @@ -326,8 +371,9 @@ def verify_dbstor_usernm(in_type, passwd, shortest_len, longest_len): """ if len(passwd) < shortest_len or len(passwd) > longest_len: - raise ValueError("The length of input must be %s to %s." - % (shortest_len, longest_len)) + raise ValueError( + "The length of input must be %s to %s." % (shortest_len, longest_len) + ) # Can't save with user name upper_cases = set("ABCDEFGHIJKLMNOPQRSTUVWXYZ") lower_cases = set("abcdefghijklmnopqrstuvwxyz") @@ -344,12 +390,14 @@ def verify_dbstor_usernm(in_type, passwd, shortest_len, longest_len): all_cases = upper_cases | lower_cases | digits | special_cases un_cases = passwd_set - all_cases if un_cases: - console_and_log("Error: There are characters that are not" - " allowed in the password: '%s'" - % "".join(un_cases)) - raise ValueError("Error: There are characters that are not" - " allowed in the password: '%s'" - % "".join(un_cases)) + console_and_log( + "Error: There are characters that are not" + " allowed in the password: '%s'" % "".join(un_cases) + ) + raise ValueError( + "Error: There are characters that are not" + " allowed in the password: '%s'" % "".join(un_cases) + ) def verify_dbstor_passwd(in_type, passwd, shortest_len, longest_len): @@ -359,23 +407,33 @@ def verify_dbstor_passwd(in_type, passwd, shortest_len, longest_len): """ # eg 'length in [8-16]' if len(passwd) < shortest_len or len(passwd) > longest_len: - console_and_log("The length of input must be %s to %s." - % (shortest_len, longest_len)) - raise ValueError("The length of input must be %s to %s." - % (shortest_len, longest_len)) + console_and_log( + "The length of input must be %s to %s." % (shortest_len, longest_len) + ) + raise ValueError( + "The length of input must be %s to %s." % (shortest_len, longest_len) + ) # Can't save with user name - user_name = db_opts.dbstor_config.get('USER_NAME') + user_name = db_opts.dbstor_config.get("USER_NAME") if user_name and passwd == user_name: console_and_log("Error: Password can't be the same as username.") raise ValueError("Error: Password can't be the same as username.") elif user_name and passwd == user_name[::-1]: - console_and_log("Error: Password cannot be the same as username in reverse order") - raise ValueError("Error: Password cannot be the same as username in reverse order") + console_and_log( + "Error: Password cannot be the same as username in reverse order" + ) + raise ValueError( + "Error: Password cannot be the same as username in reverse order" + ) # The same character cannot appear three times consecutively for i in range(0, len(passwd) - 2): if passwd[i] == passwd[i + 1] and passwd[i + 1] == passwd[i + 2]: - console_and_log("Error: The same character cannot appear three times consecutively ") - raise ValueError("Error: The same character cannot appear three times consecutively") + console_and_log( + "Error: The same character cannot appear three times consecutively " + ) + raise ValueError( + "Error: The same character cannot appear three times consecutively" + ) upper_cases = set("ABCDEFGHIJKLMNOPQRSTUVWXYZ") lower_cases = set("abcdefghijklmnopqrstuvwxyz") @@ -394,36 +452,50 @@ def verify_dbstor_passwd(in_type, passwd, shortest_len, longest_len): if passwd_set & cases: types += 1 if types < 3: - console_and_log("Error: Password must contains at least three different types of characters.") - raise ValueError("Error: Password must contains at least three" - " different types of characters.") + console_and_log( + "Error: Password must contains at least three different types of characters." + ) + raise ValueError( + "Error: Password must contains at least three" + " different types of characters." + ) # Only can contains enumerated cases all_cases = upper_cases | lower_cases | digits | special_cases un_cases = passwd_set - all_cases if un_cases: - console_and_log("Error: There are characters that are not" - " allowed in the password: '%s'" - % "".join(un_cases)) - raise ValueError("Error: There are characters that are not" - " allowed in the password: '%s'" - % "".join(un_cases)) + console_and_log( + "Error: There are characters that are not" + " allowed in the password: '%s'" % "".join(un_cases) + ) + raise ValueError( + "Error: There are characters that are not" + " allowed in the password: '%s'" % "".join(un_cases) + ) def input_username_password(): """Generate DBstor Config parameter.""" try: console_and_log("Input Username and Password") - db_opts.dbstor_config['USER_NAME'] = get_dbstor_usernm_passwd(input_prompt="UserName", - file_prompt="dbstor config", - shortest_len=6, longest_len=32) - if db_opts.dbstor_config.get('USER_NAME', 0) == 0: + db_opts.dbstor_config["USER_NAME"] = get_dbstor_usernm_passwd( + input_prompt="UserName", + file_prompt="dbstor config", + shortest_len=6, + longest_len=32, + ) + if db_opts.dbstor_config.get("USER_NAME", 0) == 0: raise ValueError("input param is invalid") - db_opts.dbstor_config['PASSWORD'] = get_dbstor_usernm_passwd(input_prompt="PassWord", - file_prompt="dbstor config", - shortest_len=8, longest_len=16) - if db_opts.dbstor_config.get('PASSWORD', 0) == 0: + db_opts.dbstor_config["PASSWORD"] = get_dbstor_usernm_passwd( + input_prompt="PassWord", + file_prompt="dbstor config", + shortest_len=8, + longest_len=16, + ) + if db_opts.dbstor_config.get("PASSWORD", 0) == 0: raise ValueError("input param is invalid") - db_opts.dbstor_config['PASSWORD'] = GLOBAL_KMC_EXT.encrypt(db_opts.dbstor_config.get('PASSWORD', "")) + db_opts.dbstor_config["PASSWORD"] = GLOBAL_KMC_EXT.encrypt( + db_opts.dbstor_config.get("PASSWORD", "") + ) console_and_log("\nSuccessfully to input user name and password") except ValueError as error: log_exit(str(error)) @@ -493,12 +565,16 @@ def get_dbstor_usernm_passwd(input_prompt, file_prompt, shortest_len, longest_le def read_file_path(): - with os.fdopen(os.open(db_opts.js_conf_file, os.O_RDONLY | os.O_EXCL, stat.S_IWUSR | stat.S_IRUSR), "r")\ - as file_obj: + with os.fdopen( + os.open( + db_opts.js_conf_file, os.O_RDONLY | os.O_EXCL, stat.S_IWUSR | stat.S_IRUSR + ), + "r", + ) as file_obj: json_data = json.load(file_obj) - db_opts.note_id = json_data.get('node_id', "").strip() + db_opts.note_id = json_data.get("node_id", "").strip() db_opts.inipath = "/opt/cantian/dbstor/tools/dbstor_config.ini" - db_opts.cluster_name = json_data.get('cluster_name', "").strip() + db_opts.cluster_name = json_data.get("cluster_name", "").strip() def check_ini(): diff --git a/pkg/deploy/action/do_snapshot.py b/pkg/deploy/action/do_snapshot.py index 993d0d7a..00fce0c3 100644 --- a/pkg/deploy/action/do_snapshot.py +++ b/pkg/deploy/action/do_snapshot.py @@ -9,13 +9,13 @@ from rest_client import RestClient from om_log import SNAPSHOT_LOGS as LOG CUR_PATH, _ = os.path.split(os.path.abspath(__file__)) -DEPLOY_PARAM_PATH = '/opt/cantian/config/deploy_param.json' +DEPLOY_PARAM_PATH = "/opt/cantian/config/deploy_param.json" NORMAL_STATE, ABNORMAL_STATE = 0, 1 def read_helper(file_path): - with open(file_path, 'r', encoding='utf-8') as f_handler: + with open(file_path, "r", encoding="utf-8") as f_handler: deploy_data = f_handler.read() return deploy_data @@ -24,16 +24,16 @@ def write_helper(file_path, data): modes = stat.S_IWRITE | stat.S_IRUSR flags = os.O_WRONLY | os.O_TRUNC | os.O_CREAT if data: - with os.fdopen(os.open(file_path, flags, modes), 'w', encoding='utf-8') as file: + with os.fdopen(os.open(file_path, flags, modes), "w", encoding="utf-8") as file: file.write(json.dumps(data)) else: - with os.fdopen(os.open(file_path, flags, modes), 'w', encoding='utf-8') as file: + with os.fdopen(os.open(file_path, flags, modes), "w", encoding="utf-8") as file: file.truncate() def get_fs_processed_info(info_path, fs_names): - json_file_path = os.path.join(info_path, 'processed_snapshots.json') - init_fs_info = {name: '' for name in fs_names} + json_file_path = os.path.join(info_path, "processed_snapshots.json") + init_fs_info = {name: "" for name in fs_names} if not os.path.exists(info_path): os.makedirs(info_path) return init_fs_info @@ -58,11 +58,11 @@ def main(mode, ip_address, main_path): passwd = input() config_params = json.loads(read_helper(DEPLOY_PARAM_PATH)) - fs_names = [fs_val - for fs_name, fs_val in config_params.items() - if fs_name.endswith('_fs')] + fs_names = [ + fs_val for fs_name, fs_val in config_params.items() if fs_name.endswith("_fs") + ] - process_fs_path = '{}/cantian_upgrade_snapshots'.format(main_path) + process_fs_path = "{}/cantian_upgrade_snapshots".format(main_path) fs_processed_data = get_fs_processed_info(process_fs_path, fs_names) login_data = (ip_address, user_name, passwd) @@ -72,19 +72,23 @@ def main(mode, ip_address, main_path): try: _ = rest_client_obj.execute(fs_name, mode) except Exception as error: - LOG.error('error happened when try to {} snapshot of {}, err_info: {}, ' - 'err_traceback: {}'.format(mode, fs_name, str(error), traceback.format_exc(limit=-1))) + LOG.error( + "error happened when try to {} snapshot of {}, err_info: {}, " + "err_traceback: {}".format( + mode, fs_name, str(error), traceback.format_exc(limit=-1) + ) + ) return ABNORMAL_STATE - if mode == 'rollback': + if mode == "rollback": query_rollback_process(fs_names, rest_client_obj) - recoder_path = os.path.join(process_fs_path, 'processed_snapshots.json') - if mode == 'create': + recoder_path = os.path.join(process_fs_path, "processed_snapshots.json") + if mode == "create": new_processed_info = rest_client_obj.processed_fs write_helper(recoder_path, new_processed_info) - elif mode == 'rollback': - write_helper(recoder_path, '') + elif mode == "rollback": + write_helper(recoder_path, "") return NORMAL_STATE @@ -100,7 +104,9 @@ def query_rollback_process(fs_names, rest_client_obj): success_list = [] while query_list: for fs_name in query_list: - rollback_rate, rollbacks_status = rest_client_obj.query_rollback_snapshots_process(fs_name) + rollback_rate, rollbacks_status = ( + rest_client_obj.query_rollback_snapshots_process(fs_name) + ) if int(rollbacks_status) == 0: success_list.append(fs_name) query_list = list(set(query_list) - set(success_list)) @@ -113,8 +119,12 @@ if __name__ == "__main__": try: RET_VAL = main(snapshot_mode, ip, main_backup_file_path) except Exception as err: - LOG.error('{} snapshots failed, err_details: {}, ' - 'err_traceback: {}'.format(snapshot_mode, str(err), traceback.format_exc(limit=-1))) + LOG.error( + "{} snapshots failed, err_details: {}, " + "err_traceback: {}".format( + snapshot_mode, str(err), traceback.format_exc(limit=-1) + ) + ) exit(ABNORMAL_STATE) exit(RET_VAL) diff --git a/pkg/deploy/action/docker/cantian_numa.py b/pkg/deploy/action/docker/cantian_numa.py index c7f38531..3fcd2fe4 100644 --- a/pkg/deploy/action/docker/cantian_numa.py +++ b/pkg/deploy/action/docker/cantian_numa.py @@ -5,7 +5,7 @@ import re import sys import time -sys.path.append('/ctdb/cantian_install/cantian_connector/action') +sys.path.append("/ctdb/cantian_install/cantian_connector/action") from logic.common_func import exec_popen from delete_unready_pod import KubernetesService, get_pod_name_from_info @@ -15,7 +15,7 @@ from docker_common.file_utils import open_and_lock_json, write_and_unlock_json, NUMA_INFO_PATH = "/root/.kube/NUMA-INFO/numa-pod.json" TIME_OUT = 100 MAX_CHECK_TIME = 120 # 最大检查时间 -CHECK_INTERVAL = 3 # 每次检查的间隔 +CHECK_INTERVAL = 3 # 每次检查的间隔 class CPUAllocator: @@ -45,9 +45,9 @@ class CPUAllocator: 例如输入 '0-25'，输出 [0, 1, 2, ..., 25] """ cpu_list = [] - for part in cpu_list_str.split(','): - if '-' in part: - start, end = map(int, part.split('-')) + for part in cpu_list_str.split(","): + if "-" in part: + start, end = map(int, part.split("-")) cpu_list.extend(range(start, end + 1)) else: cpu_list.append(int(part)) @@ -61,7 +61,7 @@ class CPUAllocator: err_msg = f"Execute cmd[{cmd}] failed, details:{stderr}" raise Exception(err_msg) - numa_nodes = int(re.search(r'\d+', stdout).group()) + numa_nodes = int(re.search(r"\d+", stdout).group()) cmd = "lscpu | grep -i 'NUMA node[0-9] CPU(s)'" return_code, stdout, stderr = exec_popen(cmd, timeout=TIME_OUT) @@ -72,7 +72,7 @@ class CPUAllocator: cpu_info = {} total_cpus = 0 for line in stdout.splitlines(): - match = re.search(r'NUMA node(\d+) CPU\(s\):\s+([\d,\-]+)', line) + match = re.search(r"NUMA node(\d+) CPU\(s\):\s+([\d,\-]+)", line) if match: node = str(match.group(1)) cpu_list_str = match.group(2) @@ -83,7 +83,7 @@ class CPUAllocator: "available_cpus": cpu_list, "available_cpu_count": node_cpus, "max_cpu": max(cpu_list), - "min_cpu": min(cpu_list) + "min_cpu": min(cpu_list), } return total_cpus, numa_nodes, cpu_info @@ -114,7 +114,7 @@ class CPUAllocator: stdout = self.execute_cmd(cmd) # 解析 taskset 输出的 CPU 列表 - match = re.search(r'list:\s+([\d,-]+)', stdout) + match = re.search(r"list:\s+([\d,-]+)", stdout) if match: cpu_list_str = match.group(1) actual_cpus = self._parse_cpu_list(cpu_list_str) @@ -132,8 +132,16 @@ class CPUAllocator: total_cpus, numa_nodes, cpu_info = self.get_numa_info() numa_info.update(cpu_info) - max_single_numa = max(info['available_cpu_count'] for info in numa_info.values() if isinstance(info, dict)) - total_available_cpus = sum(info['available_cpu_count'] for info in numa_info.values() if isinstance(info, dict)) + max_single_numa = max( + info["available_cpu_count"] + for info in numa_info.values() + if isinstance(info, dict) + ) + total_available_cpus = sum( + info["available_cpu_count"] + for info in numa_info.values() + if isinstance(info, dict) + ) if cpu_num > total_available_cpus: return 0, [] @@ -141,27 +149,33 @@ class CPUAllocator: if cpu_num > max_single_numa: needed_cpus = cpu_num binding_cpus = [] - for node, info in sorted(numa_info.items(), key=lambda x: x[1]['available_cpu_count'], reverse=True): + for node, info in sorted( + numa_info.items(), + key=lambda x: x[1]["available_cpu_count"], + reverse=True, + ): if isinstance(info, dict): - if info['available_cpu_count'] >= needed_cpus: - binding_cpus.extend(info['available_cpus'][:needed_cpus]) + if info["available_cpu_count"] >= needed_cpus: + binding_cpus.extend(info["available_cpus"][:needed_cpus]) return 2, binding_cpus else: - binding_cpus.extend(info['available_cpus']) - needed_cpus -= info['available_cpu_count'] + binding_cpus.extend(info["available_cpus"]) + needed_cpus -= info["available_cpu_count"] return 2, binding_cpus for node, info in numa_info.items(): - if isinstance(info, dict) and info['available_cpu_count'] >= cpu_num: - return 1, info['available_cpus'][:cpu_num] + if isinstance(info, dict) and info["available_cpu_count"] >= cpu_num: + return 1, info["available_cpus"][:cpu_num] needed_cpus = cpu_num binding_cpus = [] - for node, info in sorted(numa_info.items(), key=lambda x: x[1]['available_cpu_count'], reverse=True): + for node, info in sorted( + numa_info.items(), key=lambda x: x[1]["available_cpu_count"], reverse=True + ): if isinstance(info, dict): if needed_cpus > 0: - take_cpus = min(needed_cpus, info['available_cpu_count']) - binding_cpus.extend(info['available_cpus'][:take_cpus]) + take_cpus = min(needed_cpus, info["available_cpu_count"]) + binding_cpus.extend(info["available_cpus"][:take_cpus]) needed_cpus -= take_cpus if needed_cpus == 0: return 2, binding_cpus @@ -175,21 +189,34 @@ class CPUAllocator: for node, info in cpu_info.items(): node_str = str(node) - existing_available_cpus = set(numa_data[self.numa_info_key][node_str].get("available_cpus", [])) + existing_available_cpus = set( + numa_data[self.numa_info_key][node_str].get("available_cpus", []) + ) available_cpus = existing_available_cpus - set(bound_cpus) - numa_data[self.numa_info_key][node_str]["available_cpus"] = sorted(list(available_cpus)) + numa_data[self.numa_info_key][node_str]["available_cpus"] = sorted( + list(available_cpus) + ) numa_data[self.numa_info_key][node_str]["available_cpu_count"] = len( - numa_data[self.numa_info_key][node_str]["available_cpus"]) + numa_data[self.numa_info_key][node_str]["available_cpus"] + ) def clean_up_json(self, numa_data, pod_info, hostname_pattern): """ 清理不存在的 Pod 绑定信息，并将绑定的 CPU 恢复到 numa_info 中。 """ - matching_pods = [pod['pod_name'] for pod in pod_info if re.match(hostname_pattern, pod['pod_name'])] - - keys_to_delete = [key for key in numa_data.keys() if key not in matching_pods and key != self.numa_info_key] + matching_pods = [ + pod["pod_name"] + for pod in pod_info + if re.match(hostname_pattern, pod["pod_name"]) + ] + + keys_to_delete = [ + key + for key in numa_data.keys() + if key not in matching_pods and key != self.numa_info_key + ] for key in keys_to_delete: pod_file_path = os.path.join(os.path.dirname(NUMA_INFO_PATH), key) @@ -200,10 +227,14 @@ class CPUAllocator: self.restore_cpus_to_numa_info(numa_data, bind_cpus) if not os.path.exists(pod_file_path): - LOG.info(f"File {pod_file_path} does not exist. Removing entry from JSON: {key}") + LOG.info( + f"File {pod_file_path} does not exist. Removing entry from JSON: {key}" + ) del numa_data[key] elif LockFile.is_locked(pod_file_path): - LOG.info(f"Skipping removal of {key} because file {pod_file_path} is currently locked.") + LOG.info( + f"Skipping removal of {key} because file {pod_file_path} is currently locked." + ) else: LOG.info(f"Removing outdated entry from JSON and deleting file: {key}") del numa_data[key] @@ -237,7 +268,9 @@ class CPUAllocator: if hostname in numa_data: if numa_data[hostname].get("bind_flag", False): - LOG.info(f"Host {hostname} is already bound successfully. Skipping binding.") + LOG.info( + f"Host {hostname} is already bound successfully. Skipping binding." + ) return # 检查 numa_info 是否为空，如果为空则初始化 @@ -245,7 +278,9 @@ class CPUAllocator: total_cpus, numa_nodes, cpu_info = self.get_numa_info() numa_data[self.numa_info_key].update(cpu_info) - binding_status, binding_cpus = self.determine_binding_strategy(cpu_num, numa_data[self.numa_info_key]) + binding_status, binding_cpus = self.determine_binding_strategy( + cpu_num, numa_data[self.numa_info_key] + ) # 0-绑定失败，没有足够的 CPU if binding_status == 0: @@ -253,11 +288,13 @@ class CPUAllocator: numa_data[hostname] = { "bind_cpus": "", "bind_flag": False, - "taskset_output": "Binding failed due to insufficient CPUs" + "taskset_output": "Binding failed due to insufficient CPUs", } elif binding_status in (1, 2): if binding_status == 2: - LOG.warning(f"Cross NUMA binding detected for host {hostname}. This may affect performance.") + LOG.warning( + f"Cross NUMA binding detected for host {hostname}. This may affect performance." + ) # 执行绑核,1-单个numa，2-跨numa绑核 self.bind_cpu(binding_cpus) @@ -267,7 +304,7 @@ class CPUAllocator: numa_data[hostname] = { "bind_cpus": ",".join(map(str, binding_cpus)), "bind_flag": True, - "taskset_output": taskset_output + "taskset_output": taskset_output, } self.update_available_cpus(numa_data, cpu_info, binding_cpus) else: @@ -275,7 +312,7 @@ class CPUAllocator: numa_data[hostname] = { "bind_cpus": "", "bind_flag": False, - "taskset_output": "Binding failed during verification" + "taskset_output": "Binding failed during verification", } def delete_binding_info(self, short_hostname): @@ -361,7 +398,7 @@ def show_numa_binding_info(numa_info_path): if bind_cpus: try: - cpu_list = list(map(int, bind_cpus.split(','))) if bind_cpus else [] + cpu_list = list(map(int, bind_cpus.split(","))) if bind_cpus else [] formatted_cpus = format_cpu_ranges(cpu_list) except ValueError: formatted_cpus = bind_cpus @@ -422,11 +459,17 @@ def main(): pod_name_full = get_pod_name_from_info(all_pod_info, short_hostname) if pod_name_full: # 清理 JSON 中不再存在的 POD 信息 - hostname_pattern = r'cantian.*-node.*' - cpu_allocator.clean_up_json(numa_data, all_pod_info, hostname_pattern) - - cpu_allocator.execute_binding(cpu_num, pod_name_full, numa_data, cpu_info) - pod_file_path = os.path.join(os.path.dirname(NUMA_INFO_PATH), pod_name_full) + hostname_pattern = r"cantian.*-node.*" + cpu_allocator.clean_up_json( + numa_data, all_pod_info, hostname_pattern + ) + + cpu_allocator.execute_binding( + cpu_num, pod_name_full, numa_data, cpu_info + ) + pod_file_path = os.path.join( + os.path.dirname(NUMA_INFO_PATH), pod_name_full + ) break except Exception as e: err_msg = f"Error during CPU binding: {e}" @@ -454,4 +497,4 @@ if __name__ == "__main__": show_numa_binding_info(NUMA_INFO_PATH) else: pod_file_path = main() - print(pod_file_path) \ No newline at end of file + print(pod_file_path) diff --git a/pkg/deploy/action/docker/delete_unready_pod.py b/pkg/deploy/action/docker/delete_unready_pod.py index d43f185a..f80ebae6 100644 --- a/pkg/deploy/action/docker/delete_unready_pod.py +++ b/pkg/deploy/action/docker/delete_unready_pod.py @@ -12,6 +12,7 @@ from get_config_info import get_value UNREADY_THRESHOLD_SECONDS = 600 CUR_PATH = os.path.dirname(os.path.realpath(__file__)) + def _exec_popen(cmd, values=None): """ subprocess.Popen in python2 and 3. @@ -21,8 +22,13 @@ def _exec_popen(cmd, values=None): if not values: values = [] bash_cmd = ["bash"] - pobj = subprocess.Popen(bash_cmd, shell=False, stdin=subprocess.PIPE, - stdout=subprocess.PIPE, stderr=subprocess.PIPE) + pobj = subprocess.Popen( + bash_cmd, + shell=False, + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + ) pobj.stdin.write(cmd.encode()) pobj.stdin.write(os.linesep.encode()) for value in values: @@ -55,22 +61,23 @@ def get_pod_name_from_info(pod_info, pod_name): return None + def backup_log(): """备份日志函数，unready的pod会被重复检测，只有首次打印备份日志，后续直接返回""" - healthy_file = '/opt/cantian/healthy' + healthy_file = "/opt/cantian/healthy" if os.path.exists(healthy_file): # 读文件内参数 - with open(healthy_file, 'r') as fread: + with open(healthy_file, "r") as fread: data = fread.read() try: healthy_dict = json.loads(data) except json.decoder.JSONDecodeError as e: - healthy_dict = {'delete_unready_pod': 0} - if not healthy_dict['delete_unready_pod']: - healthy_dict['delete_unready_pod'] = 1 + healthy_dict = {"delete_unready_pod": 0} + if not healthy_dict["delete_unready_pod"]: + healthy_dict["delete_unready_pod"] = 1 flags = os.O_CREAT | os.O_RDWR modes = stat.S_IWUSR | stat.S_IRUSR - with os.fdopen(os.open(healthy_file, flags, modes), 'w') as fwrite: + with os.fdopen(os.open(healthy_file, flags, modes), "w") as fwrite: json.dump(healthy_dict, fwrite) else: return @@ -78,16 +85,24 @@ def backup_log(): # healthy文件不存在说明pod状态异常，会有其他处理，直接返回 return - cluster_name = get_value('cluster_name') - cluster_id = get_value('cluster_id') - node_id = get_value('node_id') - deploy_user = get_value('deploy_user') - storage_metadata_fs = get_value('storage_metadata_fs') - cmd = "sh %s/log_backup.sh %s %s %s %s %s" % (CUR_PATH, cluster_name, cluster_id, node_id, deploy_user, storage_metadata_fs) + cluster_name = get_value("cluster_name") + cluster_id = get_value("cluster_id") + node_id = get_value("node_id") + deploy_user = get_value("deploy_user") + storage_metadata_fs = get_value("storage_metadata_fs") + cmd = "sh %s/log_backup.sh %s %s %s %s %s" % ( + CUR_PATH, + cluster_name, + cluster_id, + node_id, + deploy_user, + storage_metadata_fs, + ) ret_code, _, stderr = _exec_popen(cmd) if ret_code: raise Exception("failed to backup log. output:%s" % str(stderr)) + def monitor_pods(k8s_service, pod_name): pod = k8s_service.get_pod_by_name(pod_name) if not pod: @@ -101,15 +116,24 @@ def monitor_pods(k8s_service, pod_name): if condition["type"] == "Ready" and condition["status"] != "True": last_transition_time = condition.get("lastTransitionTime") if last_transition_time: - last_transition_time = datetime.strptime(last_transition_time, "%Y-%m-%dT%H:%M:%SZ") + last_transition_time = datetime.strptime( + last_transition_time, "%Y-%m-%dT%H:%M:%SZ" + ) unready_duration = current_time - last_transition_time - print(f"Pod {pod_name} has been unready for more than {unready_duration.total_seconds()} seconds.") + print( + f"Pod {pod_name} has been unready for more than {unready_duration.total_seconds()} seconds." + ) if unready_duration.total_seconds() > UNREADY_THRESHOLD_SECONDS: - print(f"Pod {pod_name} has been unready for more than {UNREADY_THRESHOLD_SECONDS} seconds. Deleting...") + print( + f"Pod {pod_name} has been unready for more than {UNREADY_THRESHOLD_SECONDS} seconds. Deleting..." + ) backup_log() - k8s_service.delete_pod(name=pod_name, namespace=pod["metadata"]["namespace"]) + k8s_service.delete_pod( + name=pod_name, namespace=pod["metadata"]["namespace"] + ) return + if __name__ == "__main__": pod_name = os.getenv("HOSTNAME") if not pod_name: @@ -125,4 +149,4 @@ if __name__ == "__main__": monitor_pods(k8s_service, pod_name_full) else: print(f"Service not found for pod: {pod_name}") - exit(1) \ No newline at end of file + exit(1) diff --git a/pkg/deploy/action/docker/docker_common/file_utils.py b/pkg/deploy/action/docker/docker_common/file_utils.py index 13a2fb7b..b9bba0e8 100644 --- a/pkg/deploy/action/docker/docker_common/file_utils.py +++ b/pkg/deploy/action/docker/docker_common/file_utils.py @@ -10,6 +10,7 @@ import sys import pwd import grp import stat + CUR_PATH = os.path.dirname(os.path.realpath(__file__)) sys.path.append(os.path.join(CUR_PATH, "../../")) from cantian.get_config_info import get_value @@ -43,7 +44,7 @@ class LockFile: @staticmethod def is_locked(file_path): try: - with open(file_path, 'a') as f: + with open(file_path, "a") as f: try: fcntl.flock(f, fcntl.LOCK_EX | fcntl.LOCK_NB) fcntl.flock(f, fcntl.LOCK_UN) @@ -51,7 +52,9 @@ class LockFile: except IOError: return True except Exception as e: - raise Exception(f"Error checking lock status of {file_path}: {str(e)}") from e + raise Exception( + f"Error checking lock status of {file_path}: {str(e)}" + ) from e def open_and_lock_json(filepath, timeout=20): @@ -65,7 +68,7 @@ def open_and_lock_json(filepath, timeout=20): try: fd = os.open(filepath, os.O_RDWR | os.O_CREAT, 0o644) - file = os.fdopen(fd, 'r+') + file = os.fdopen(fd, "r+") LockFile.lock_with_timeout(file, timeout=timeout) @@ -105,7 +108,7 @@ def open_and_lock_csv(filepath, timeout=20): try: fd = os.open(filepath, os.O_RDWR | os.O_CREAT, 0o644) - file = os.fdopen(fd, 'r+') + file = os.fdopen(fd, "r+") LockFile.lock_with_timeout(file, timeout=timeout) @@ -143,7 +146,7 @@ def read_file(filepath): try: if os.path.exists(filepath): flags = os.O_RDONLY - with os.fdopen(os.open(filepath, flags), 'r') as f: + with os.fdopen(os.open(filepath, flags), "r") as f: fcntl.flock(f, fcntl.LOCK_SH) content = f.readlines() fcntl.flock(f, fcntl.LOCK_UN) @@ -175,7 +178,7 @@ def write_file(filepath, content): flags = os.O_WRONLY | os.O_CREAT | os.O_EXCL modes = original_mode - with os.fdopen(os.open(temp_file, flags, modes), 'w') as f: + with os.fdopen(os.open(temp_file, flags, modes), "w") as f: fcntl.flock(f, fcntl.LOCK_EX) f.writelines(content) fcntl.flock(f, fcntl.LOCK_UN) @@ -216,4 +219,4 @@ def mkdir(path, permissions=0o750): os.chown(path, uid, gid) except Exception as e: - raise RuntimeError(f"Failed to create directory '{path}': {e}") \ No newline at end of file + raise RuntimeError(f"Failed to create directory '{path}': {e}") diff --git a/pkg/deploy/action/docker/docker_common/kubernetes_service.py b/pkg/deploy/action/docker/docker_common/kubernetes_service.py index 3b5f5921..afe1a144 100644 --- a/pkg/deploy/action/docker/docker_common/kubernetes_service.py +++ b/pkg/deploy/action/docker/docker_common/kubernetes_service.py @@ -24,19 +24,31 @@ class KubernetesService: with open(self.kube_config_path, "r") as kube_config_file: kube_config_content = kube_config_file.read() - client_cert_data = re.search(r'client-certificate-data: (.+)', kube_config_content).group(1) - client_key_data = re.search(r'client-key-data: (.+)', kube_config_content).group(1) + client_cert_data = re.search( + r"client-certificate-data: (.+)", kube_config_content + ).group(1) + client_key_data = re.search( + r"client-key-data: (.+)", kube_config_content + ).group(1) client_cert_data = base64.b64decode(client_cert_data) client_key_data = base64.b64decode(client_key_data) cert_file_path = "/tmp/client-cert.pem" key_file_path = "/tmp/client-key.pem" - cert_fd = os.open(cert_file_path, os.O_WRONLY | os.O_CREAT | os.O_TRUNC, stat.S_IRUSR | stat.S_IWUSR) + cert_fd = os.open( + cert_file_path, + os.O_WRONLY | os.O_CREAT | os.O_TRUNC, + stat.S_IRUSR | stat.S_IWUSR, + ) with os.fdopen(cert_fd, "wb") as cert_file: cert_file.write(client_cert_data) - key_fd = os.open(key_file_path, os.O_WRONLY | os.O_CREAT | os.O_TRUNC, stat.S_IRUSR | stat.S_IWUSR) + key_fd = os.open( + key_file_path, + os.O_WRONLY | os.O_CREAT | os.O_TRUNC, + stat.S_IRUSR | stat.S_IWUSR, + ) with os.fdopen(key_fd, "wb") as key_file: key_file.write(client_key_data) @@ -47,7 +59,9 @@ class KubernetesService: def _get(self, path, timeout=5): url = f"{self.api_server}{path}" - response = requests.get(url, headers=self.headers, cert=self.cert, verify=False, timeout=timeout) + response = requests.get( + url, headers=self.headers, cert=self.cert, verify=False, timeout=timeout + ) response.raise_for_status() return response.json() @@ -64,7 +78,9 @@ class KubernetesService: matching_pods = [] for pod in pods_data.get("items", []): pod_labels = pod["metadata"].get("labels", {}) - if all(item in pod_labels.items() for item in service_selector.items()): + if all( + item in pod_labels.items() for item in service_selector.items() + ): matching_pods.append(pod) for pod in matching_pods: @@ -104,11 +120,13 @@ class KubernetesService: for port in ports: container_port = port.get("containerPort") if pod_name and pod_ip and container_port: - pod_info.append({ - "pod_name": pod_name, - "pod_ip": pod_ip, - "container_port": container_port - }) + pod_info.append( + { + "pod_name": pod_name, + "pod_ip": pod_ip, + "container_port": container_port, + } + ) return pod_info @@ -138,12 +156,14 @@ class KubernetesService: for port in ports: container_port = port.get("containerPort") if pod_name_all and pod_ip and container_port: - all_pod_info.append({ - "service_name": service["metadata"]["name"], - "pod_name": pod_name_all, - "pod_ip": pod_ip, - "container_port": container_port - }) + all_pod_info.append( + { + "service_name": service["metadata"]["name"], + "pod_name": pod_name_all, + "pod_ip": pod_ip, + "container_port": container_port, + } + ) return all_pod_info @@ -152,7 +172,9 @@ class KubernetesService: def delete_pod(self, name, namespace, timeout=5): url = f"{self.api_server}/api/v1/namespaces/{namespace}/pods/{name}" - response = requests.delete(url, headers=self.headers, cert=self.cert, verify=False, timeout=timeout) + response = requests.delete( + url, headers=self.headers, cert=self.cert, verify=False, timeout=timeout + ) response.raise_for_status() return response.json() diff --git a/pkg/deploy/action/docker/dr_deploy.py b/pkg/deploy/action/docker/dr_deploy.py index 2f697a48..cc190128 100644 --- a/pkg/deploy/action/docker/dr_deploy.py +++ b/pkg/deploy/action/docker/dr_deploy.py @@ -22,7 +22,9 @@ from om_log import LOGGER as LOG OPT_CONFIG_PATH = "/opt/cantian/config" SCRIPT_PATH = os.path.join(CUR_PATH, "..") CONFIG_PATH = os.path.join(SCRIPT_PATH, "../config") -DORADO_CONF_PATH = "/ctdb/cantian_install/cantian_connector/config/container_conf/dorado_conf" +DORADO_CONF_PATH = ( + "/ctdb/cantian_install/cantian_connector/config/container_conf/dorado_conf" +) DM_USER = "DMUser" DM_PWD = "DMPwd" @@ -30,13 +32,17 @@ DM_PWD = "DMPwd" def init_get_info_fun(): try: get_info_path = os.path.join(CUR_PATH, "../get_config_info.py") - spec = importlib.util.spec_from_file_location("get_info_from_config", get_info_path) + spec = importlib.util.spec_from_file_location( + "get_info_from_config", get_info_path + ) get_info_from_config = importlib.util.module_from_spec(spec) spec.loader.exec_module(get_info_from_config) get_info = get_info_from_config.get_value return get_info except Exception as e: - LOG.error(f"init get_info fun failed {e}, traceback: {traceback.format_exc(limit=-1)}") + LOG.error( + f"init get_info fun failed {e}, traceback: {traceback.format_exc(limit=-1)}" + ) return None @@ -76,11 +82,11 @@ def close_child_process(proc): os.killpg(proc.pid, signal.SIGKILL) except ProcessLookupError as err: _ = err - return 'success' + return "success" except Exception as err: return str(err) - return 'success' + return "success" def exec_popen(cmd, timeout=None): @@ -90,8 +96,14 @@ def exec_popen(cmd, timeout=None): return: status code, standard output, error output """ bash_cmd = ["bash"] - pobj = subprocess.Popen(bash_cmd, shell=False, stdin=subprocess.PIPE, - stdout=subprocess.PIPE, stderr=subprocess.PIPE, preexec_fn=os.setsid) + pobj = subprocess.Popen( + bash_cmd, + shell=False, + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + preexec_fn=os.setsid, + ) pobj.stdin.write(cmd.encode()) pobj.stdin.write(os.linesep.encode()) if not timeout: @@ -142,8 +154,10 @@ def get_dr_status(dm_password=None): dm_password = get_dm_password() if dm_password == "": LOG.error("DM pass word is empty.") - cmd = (f"echo -e '{dm_password}' | sh {SCRIPT_PATH}/appctl.sh dr_operate progress_query " - f"--action=check --display=table 2>&1 | grep -E '^\-|^\|'") + cmd = ( + f"echo -e '{dm_password}' | sh {SCRIPT_PATH}/appctl.sh dr_operate progress_query " + f"--action=check --display=table 2>&1 | grep -E '^\-|^\|'" + ) execute_command(cmd, raise_flag=True, timeout=30) data_json = get_file_json(os.path.join(CONFIG_PATH, "dr_status.json")) return data_json.get("dr_status") @@ -196,7 +210,7 @@ def check_dr_deploy_process_completed(role): return False -def dr_deploy(role=None, dm_password=None, mysql_pwd='', delete_flag=False): +def dr_deploy(role=None, dm_password=None, mysql_pwd="", delete_flag=False): if not dm_password: dm_password = get_dm_password() if dm_password == "": @@ -204,8 +218,10 @@ def dr_deploy(role=None, dm_password=None, mysql_pwd='', delete_flag=False): raise Exception("get dm_pass word failed.") if not role: role = get_value("dr_deploy.role") - cmd = (f"echo -e '{dm_password}' | sh {SCRIPT_PATH}/appctl.sh dr_operate pre_check {role} " - f"--conf=/opt/cantian/config/deploy_param.json") + cmd = ( + f"echo -e '{dm_password}' | sh {SCRIPT_PATH}/appctl.sh dr_operate pre_check {role} " + f"--conf=/opt/cantian/config/deploy_param.json" + ) execute_command(cmd, raise_flag=True, timeout=300) LOG.info("dr_operate pre_check success.") @@ -213,8 +229,10 @@ def dr_deploy(role=None, dm_password=None, mysql_pwd='', delete_flag=False): if os.path.exists(dr_process_file): os.remove(dr_process_file) - cmd = (f"echo -e '{dm_password}\n{mysql_pwd}' | sh {SCRIPT_PATH}/appctl.sh dr_operate deploy {role} " - f"--mysql_cmd='mysql' --mysql_user=root") + cmd = ( + f"echo -e '{dm_password}\n{mysql_pwd}' | sh {SCRIPT_PATH}/appctl.sh dr_operate deploy {role} " + f"--mysql_cmd='mysql' --mysql_user=root" + ) execute_command(cmd, raise_flag=True) if check_dr_deploy_process_completed(role): @@ -222,8 +240,10 @@ def dr_deploy(role=None, dm_password=None, mysql_pwd='', delete_flag=False): if delete_flag: deploy_user = get_value("deploy_user") storage_share_fs = get_value("storage_share_fs") - cmd = (f"su -s /bin/bash - {deploy_user} -c 'dbstor --delete-file " - f"--fs-name={storage_share_fs} --file-name=onlyStart.file'") + cmd = ( + f"su -s /bin/bash - {deploy_user} -c 'dbstor --delete-file " + f"--fs-name={storage_share_fs} --file-name=onlyStart.file'" + ) execute_command(cmd, timeout=180) sys.exit(0) sys.exit(1) @@ -238,8 +258,10 @@ def copy_version_yaml(deploy_user, deploy_group): chown_path(version_file, u_id, g_id) version_dir = os.path.dirname(version_file) storage_share_fs = get_value("storage_share_fs") - cmd = (f'su -s /bin/bash - "{deploy_user}" -c "dbstor --copy-file --import ' - f'--fs-name={storage_share_fs} --source-dir={version_dir} --target-dir=/ --file-name=versions.yml"') + cmd = ( + f'su -s /bin/bash - "{deploy_user}" -c "dbstor --copy-file --import ' + f'--fs-name={storage_share_fs} --source-dir={version_dir} --target-dir=/ --file-name=versions.yml"' + ) execute_command(cmd, timeout=180) @@ -249,10 +271,12 @@ def start_mysqld(deploy_user=None, deploy_group=None): if not deploy_group: deploy_group = get_info("deploy_group") LOG.info(f"Begin to start mysqld.") - cmd = (f"su -s /bin/bash - {deploy_user} -c 'python3 " - f"-B /opt/cantian/image/cantian_connector/CantianKernel/Cantian-DATABASE-CENTOS-64bit/install.py " - f"-U {deploy_user}:{deploy_group} -l /home/{deploy_user}/logs/install.log " - f"-M mysqld -m /opt/cantian/image/cantian_connector/cantian-connector-mysql/scripts/my.cnf -g withoutroot'") + cmd = ( + f"su -s /bin/bash - {deploy_user} -c 'python3 " + f"-B /opt/cantian/image/cantian_connector/CantianKernel/Cantian-DATABASE-CENTOS-64bit/install.py " + f"-U {deploy_user}:{deploy_group} -l /home/{deploy_user}/logs/install.log " + f"-M mysqld -m /opt/cantian/image/cantian_connector/cantian-connector-mysql/scripts/my.cnf -g withoutroot'" + ) execute_command(cmd, raise_flag=True, timeout=300) LOG.info(f"start mysqld success.") @@ -266,12 +290,16 @@ def dr_start_deploy(): count = 0 if deploy_mode == "dbstor": storage_fs = get_value("storage_share_fs") - cmd = (f"su -s /bin/bash - {deploy_user} -c 'dbstor --query-file " - f"--fs-name={storage_fs} --file-dir=/' | grep 'dr_deploy_param.json' | wc -l") + cmd = ( + f"su -s /bin/bash - {deploy_user} -c 'dbstor --query-file " + f"--fs-name={storage_fs} --file-dir=/' | grep 'dr_deploy_param.json' | wc -l" + ) code, count, err = execute_command(cmd, timeout=180) else: storage_fs = get_value("storage_metadata_fs") - if os.path.exists(f"/mnt/dbdata/remote/metadata_{storage_fs}/dr_deploy_param.json"): + if os.path.exists( + f"/mnt/dbdata/remote/metadata_{storage_fs}/dr_deploy_param.json" + ): count = 1 if not count.isdigit(): LOG.error("get file count failed.") @@ -284,28 +312,42 @@ def dr_start_deploy(): g_id = deploy_group_info.gr_gid os.chown(OPT_CONFIG_PATH, u_id, g_id) if deploy_mode == "dbstor": - cmd = (f"su -s /bin/bash - {deploy_user} -c 'dbstor --copy-file --export --fs-name={storage_fs} " - f"--source-dir=/ --target-dir={OPT_CONFIG_PATH} --file-name=dr_deploy_param.json'") + cmd = ( + f"su -s /bin/bash - {deploy_user} -c 'dbstor --copy-file --export --fs-name={storage_fs} " + f"--source-dir=/ --target-dir={OPT_CONFIG_PATH} --file-name=dr_deploy_param.json'" + ) execute_command(cmd, timeout=180) else: - copy_file(f"/mnt/dbdata/remote/metadata_{storage_fs}/dr_deploy_param.json", - f"{OPT_CONFIG_PATH}/dr_deploy_param.json") - copy_file(f"{OPT_CONFIG_PATH}/dr_deploy_param.json", f"{CONFIG_PATH}/dr_deploy_param.json") + copy_file( + f"/mnt/dbdata/remote/metadata_{storage_fs}/dr_deploy_param.json", + f"{OPT_CONFIG_PATH}/dr_deploy_param.json", + ) + copy_file( + f"{OPT_CONFIG_PATH}/dr_deploy_param.json", + f"{CONFIG_PATH}/dr_deploy_param.json", + ) if get_dr_status(dm_password) != "Normal": - msg = ("DR status is Abnormal. If you need, " - "please enter the container and manually execute the Dr_deploy process.") + msg = ( + "DR status is Abnormal. If you need, " + "please enter the container and manually execute the Dr_deploy process." + ) LOG.error(msg) cmd = f"sh {SCRIPT_PATH}/appctl.sh start" execute_command(cmd, raise_flag=True, timeout=3600) - if get_value("cantian_in_container") == "1" and get_value("M_RUNING_MODE") != "cantiand_with_mysql_in_cluster": + if ( + get_value("cantian_in_container") == "1" + and get_value("M_RUNING_MODE") != "cantiand_with_mysql_in_cluster" + ): start_mysqld(deploy_user, deploy_group) else: if role == "active": cmd = f"sh {SCRIPT_PATH}/appctl.sh start" execute_command(cmd, raise_flag=True, timeout=3600) - if (get_value("cantian_in_container") == "1" and - get_value("M_RUNING_MODE") != "cantiand_with_mysql_in_cluster"): + if ( + get_value("cantian_in_container") == "1" + and get_value("M_RUNING_MODE") != "cantiand_with_mysql_in_cluster" + ): start_mysqld(deploy_user, deploy_group) copy_version_yaml(deploy_user, deploy_group) LOG.info("dr_setup is True, executing dr_deploy tasks.") @@ -315,17 +357,19 @@ def dr_start_deploy(): def main(): - split_env = os.environ['LD_LIBRARY_PATH'].split(":") + split_env = os.environ["LD_LIBRARY_PATH"].split(":") if "/opt/cantian/dbstor/lib" not in split_env: - LOG.error(f"cantian-dbstor-lib not found, current envpath[{os.environ['LD_LIBRARY_PATH']}]") + LOG.error( + f"cantian-dbstor-lib not found, current envpath[{os.environ['LD_LIBRARY_PATH']}]" + ) raise Exception("cantian-dbstor-lib not found") action_dict = { "get_dm_password": get_dm_password, "get_dr_status": get_dr_status, - "start": dr_start_deploy + "start": dr_start_deploy, } - mysql_pwd = '' + mysql_pwd = "" delete_config = False if len(sys.argv) == 1: mysql_pwd = getpass.getpass("Please input mysql login password:") @@ -341,6 +385,7 @@ if __name__ == "__main__": try: main() except Exception as e: - LOG.error(f"execute failed, err[{str(e)}], traceback: [{traceback.format_exc(limit=-1)}]") + LOG.error( + f"execute failed, err[{str(e)}], traceback: [{traceback.format_exc(limit=-1)}]" + ) sys.exit(1) - diff --git a/pkg/deploy/action/docker/get_config_info.py b/pkg/deploy/action/docker/get_config_info.py index 9a4b278a..78c02eb4 100644 --- a/pkg/deploy/action/docker/get_config_info.py +++ b/pkg/deploy/action/docker/get_config_info.py @@ -35,7 +35,7 @@ def get_value(param): if param == "M_RUNING_MODE": return install_config.get("M_RUNING_MODE") - keys = param.split('.') + keys = param.split(".") value = info try: for key in keys: diff --git a/pkg/deploy/action/docker/pod_record.py b/pkg/deploy/action/docker/pod_record.py index 1801b424..717c474c 100644 --- a/pkg/deploy/action/docker/pod_record.py +++ b/pkg/deploy/action/docker/pod_record.py @@ -4,7 +4,7 @@ import sys from datetime import datetime from docker_common.file_utils import open_and_lock_csv, write_and_unlock_csv -sys.path.append('/ctdb/cantian_install/cantian_connector/action') +sys.path.append("/ctdb/cantian_install/cantian_connector/action") from delete_unready_pod import KubernetesService, get_pod_name_from_info from om_log import LOGGER as LOG @@ -17,27 +17,36 @@ RESTART_THRESHOLD = 6 def update_pod_restart_record(k8s_service, pod_name_full, pod_namespace): pod_record, file_handle = open_and_lock_csv(POD_RECORD_FILE_PATH) - record_dict = {rows[0]: {'restart_count': int(rows[1]), 'last_restart_time': rows[2]} for rows in pod_record} + record_dict = { + rows[0]: {"restart_count": int(rows[1]), "last_restart_time": rows[2]} + for rows in pod_record + } if pod_name_full not in record_dict: record_dict[pod_name_full] = { "restart_count": 1, - "last_restart_time": datetime.utcnow().strftime("%Y-%m-%dT%H:%M:%SZ") + "last_restart_time": datetime.utcnow().strftime("%Y-%m-%dT%H:%M:%SZ"), } else: record_dict[pod_name_full]["restart_count"] += 1 - record_dict[pod_name_full]["last_restart_time"] = datetime.utcnow().strftime("%Y-%m-%dT%H:%M:%SZ") + record_dict[pod_name_full]["last_restart_time"] = datetime.utcnow().strftime( + "%Y-%m-%dT%H:%M:%SZ" + ) restart_count = record_dict[pod_name_full]["restart_count"] if restart_count > RESTART_THRESHOLD: - LOG.info(f"Pod {pod_name_full} has restarted {restart_count} times, Deleting...") + LOG.info( + f"Pod {pod_name_full} has restarted {restart_count} times, Deleting..." + ) k8s_service.delete_pod(pod_name_full, pod_namespace) else: LOG.info("Cantian pod start record updated successfully.") - rows_to_write = [[pod_name, data['restart_count'], data['last_restart_time']] - for pod_name, data in record_dict.items()] + rows_to_write = [ + [pod_name, data["restart_count"], data["last_restart_time"]] + for pod_name, data in record_dict.items() + ] write_and_unlock_csv(rows_to_write, file_handle) @@ -76,4 +85,4 @@ if __name__ == "__main__": try: main() except Exception as err: - LOG.error(f"Error in pod_record.py: {err}") \ No newline at end of file + LOG.error(f"Error in pod_record.py: {err}") diff --git a/pkg/deploy/action/docker/resolve_pwd.py b/pkg/deploy/action/docker/resolve_pwd.py index 0ad6b79e..c1dc9688 100644 --- a/pkg/deploy/action/docker/resolve_pwd.py +++ b/pkg/deploy/action/docker/resolve_pwd.py @@ -18,8 +18,13 @@ def _exec_popen(cmd, values=None): if not values: values = [] bash_cmd = ["bash"] - pobj = subprocess.Popen(bash_cmd, shell=False, stdin=subprocess.PIPE, - stdout=subprocess.PIPE, stderr=subprocess.PIPE) + pobj = subprocess.Popen( + bash_cmd, + shell=False, + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + ) pobj.stdin.write(cmd.encode()) pobj.stdin.write(os.linesep.encode()) for value in values: @@ -48,10 +53,16 @@ def resolve_kmc_pwd(encrypt_pwd): try: passwd = kmc_adapter.decrypt(encrypt_pwd) except Exception as error: - raise Exception("Failed to decrypt password of user [sys]. Error: %s" % str(error)) from error - split_env = os.environ['LD_LIBRARY_PATH'].split(":") - filtered_env = [single_env for single_env in split_env if "/opt/cantian/dbstor/lib" not in single_env] - os.environ['LD_LIBRARY_PATH'] = ":".join(filtered_env) + raise Exception( + "Failed to decrypt password of user [sys]. Error: %s" % str(error) + ) from error + split_env = os.environ["LD_LIBRARY_PATH"].split(":") + filtered_env = [ + single_env + for single_env in split_env + if "/opt/cantian/dbstor/lib" not in single_env + ] + os.environ["LD_LIBRARY_PATH"] = ":".join(filtered_env) return passwd @@ -72,7 +83,9 @@ def kmc_to_ctencrypt_pwd(encrypt_pwd): stderr = str(stderr) stderr.replace(passwd, "****") if ret_code: - raise Exception("failed to get _SYS_PASSWORD by ctencrypt. output:%s" % str(stderr)) + raise Exception( + "failed to get _SYS_PASSWORD by ctencrypt. output:%s" % str(stderr) + ) return stdout @@ -82,7 +95,7 @@ def run_upgrade_modify_sys_tables_ctsql(encrypt_pwd): port = input() passwd = resolve_kmc_pwd(encrypt_pwd) try: - with open(sql_file_path, 'r') as sql_file: + with open(sql_file_path, "r") as sql_file: sql = "" line = sql_file.readline() while line: @@ -110,6 +123,6 @@ if __name__ == "__main__": "resolve_check_cert_pwd": resolve_check_cert_pwd, "kmc_to_ctencrypt_pwd": kmc_to_ctencrypt_pwd, "run_upgrade_modify_sys_tables_ctsql": run_upgrade_modify_sys_tables_ctsql, - "resolve_kmc_pwd": resolve_kmc_pwd + "resolve_kmc_pwd": resolve_kmc_pwd, } - print(options.get(action)(encrypt_pwd.strip())) \ No newline at end of file + print(options.get(action)(encrypt_pwd.strip())) diff --git a/pkg/deploy/action/docker/update_config.py b/pkg/deploy/action/docker/update_config.py index a6c4e586..6465f46b 100644 --- a/pkg/deploy/action/docker/update_config.py +++ b/pkg/deploy/action/docker/update_config.py @@ -2,6 +2,7 @@ import os import json import re import sys + CUR_PATH = os.path.dirname(os.path.realpath(__file__)) sys.path.append(os.path.join(CUR_PATH, "../")) from cantian.get_config_info import get_value @@ -10,7 +11,9 @@ from om_log import LOGGER as LOG CONFIG_PATH = os.path.join(CUR_PATH, "../../config") INIT_CONFIG_PATH = os.path.join(CONFIG_PATH, "container_conf/init_conf") -MY_CNF_FILE = "/opt/cantian/image/cantian_connector/cantian-connector-mysql/scripts/my.cnf" +MY_CNF_FILE = ( + "/opt/cantian/image/cantian_connector/cantian-connector-mysql/scripts/my.cnf" +) MYSQL_CONFIG_FILE = os.path.join(INIT_CONFIG_PATH, "mysql_config.json") MYSQL_PARAMS = ["max_connections", "table_open_cache", "table_open_cache_instances"] @@ -115,5 +118,7 @@ if __name__ == "__main__": LOG.error(f"An error occurred while executing '{action}': {e}") sys.exit(1) else: - LOG.error(f"Invalid action '{action}'. Available options: {', '.join(func_dict.keys())}") - sys.exit(1) \ No newline at end of file + LOG.error( + f"Invalid action '{action}'. Available options: {', '.join(func_dict.keys())}" + ) + sys.exit(1) diff --git a/pkg/deploy/action/docker/update_policy_params.py b/pkg/deploy/action/docker/update_policy_params.py index 0865e0a6..7d0dba27 100644 --- a/pkg/deploy/action/docker/update_policy_params.py +++ b/pkg/deploy/action/docker/update_policy_params.py @@ -2,20 +2,22 @@ import os import json import stat import sys + CUR_PATH = os.path.dirname(os.path.realpath(__file__)) sys.path.append(os.path.join(CUR_PATH, "..")) from om_log import LOGGER as LOG + dir_name, _ = os.path.split(os.path.abspath(__file__)) def parse_policy_config_file(): policy_path = os.path.join(dir_name, "../deploy_policy_config.json") try: - with open(policy_path, 'r', encoding='utf8') as file_path: + with open(policy_path, "r", encoding="utf8") as file_path: json_data = json.load(file_path) return json_data except Exception as error: - LOG.error('load %s error, error: %s', policy_path, str(error)) + LOG.error("load %s error, error: %s", policy_path, str(error)) return False @@ -23,11 +25,11 @@ def parse_cantian_config_file(): cantian_config_dir = os.path.join(dir_name, "../../config/container_conf/init_conf") cantian_config_path = os.path.join(cantian_config_dir, "deploy_param.json") try: - with open(cantian_config_path, 'r', encoding='utf8') as file_path: + with open(cantian_config_path, "r", encoding="utf8") as file_path: json_data = json.load(file_path) return json_data except Exception as error: - LOG.error('load %s error, error: %s', cantian_config_path, str(error)) + LOG.error("load %s error, error: %s", cantian_config_path, str(error)) return False @@ -36,7 +38,7 @@ def write_back_to_json(cantian_config_json): cantian_config_path = os.path.join(cantian_config_dir, "deploy_param.json") modes = stat.S_IRWXU | stat.S_IROTH | stat.S_IRGRP flag = os.O_RDWR | os.O_CREAT | os.O_TRUNC - with os.fdopen(os.open(cantian_config_path, flag, modes), 'w') as file_path: + with os.fdopen(os.open(cantian_config_path, flag, modes), "w") as file_path: config_params = json.dumps(cantian_config_json, indent=4) file_path.write(config_params) @@ -57,7 +59,7 @@ def main(): source_config_json["deploy_policy"] = "default" write_back_to_json(source_config_json) return - LOG.info("deploy policy is %s" % deploy_policy_key) + LOG.info("deploy policy is %s" % deploy_policy_key) # 如果配置方案未配置则返回失败，安装结束 deploy_policy_value = source_deploy_policy_json.get(deploy_policy_key, {}) @@ -72,8 +74,8 @@ def main(): write_back_to_json(source_config_json) -if __name__ == '__main__': +if __name__ == "__main__": try: main() except Exception as e: - exit(str(e)) \ No newline at end of file + exit(str(e)) diff --git a/pkg/deploy/action/docker/upgrade_version_check.py b/pkg/deploy/action/docker/upgrade_version_check.py index b808a2ef..d25c0c9b 100644 --- a/pkg/deploy/action/docker/upgrade_version_check.py +++ b/pkg/deploy/action/docker/upgrade_version_check.py @@ -3,25 +3,34 @@ import os import re from pathlib import Path from get_config_info import get_value + CUR_PATH = os.path.dirname(os.path.realpath(__file__)) sys.path.append(os.path.join(CUR_PATH, "..")) from om_log import LOGGER as LOG METADATA_FS = get_value("storage_metadata_fs") -VERSION_PREFIX = 'Version:' -SUB_VERSION_PREFIX = ('B', 'SP') +VERSION_PREFIX = "Version:" +SUB_VERSION_PREFIX = ("B", "SP") class UpgradeVersionCheck: def __init__(self): self.white_list_file = str(Path(os.path.join(CUR_PATH, "white_list.txt"))) - self.source_version_file = str(Path(os.path.join("/mnt/dbdata/remote/metadata_" + METADATA_FS, "versions.yml"))) + self.source_version_file = str( + Path( + os.path.join( + "/mnt/dbdata/remote/metadata_" + METADATA_FS, "versions.yml" + ) + ) + ) self.white_list_dict = {} - self.source_version = '' - self.err = {'read_failed': 'white list or source version read failed', - 'match_failed': 'source version not in white list', - 'updata_system_version_failed': 'update system version failed'} + self.source_version = "" + self.err = { + "read_failed": "white list or source version read failed", + "match_failed": "source version not in white list", + "updata_system_version_failed": "update system version failed", + } @staticmethod def update_system_version(): @@ -31,10 +40,10 @@ class UpgradeVersionCheck: @staticmethod def execption_handler(err_msg): LOG.error(err_msg) - return 'False {}'.format(err_msg) - + return "False {}".format(err_msg) + def process_white_list(self): - with open(self.white_list_file, 'r', encoding='utf-8') as file: + with open(self.white_list_file, "r", encoding="utf-8") as file: white_list_info = file.readlines() for white_list_detail in white_list_info[1:]: @@ -44,8 +53,8 @@ class UpgradeVersionCheck: self.white_list_dict[details[0]] = [details[1], details[2]] def read_source_version_info(self): - version = '' - with open(self.source_version_file, 'r', encoding='utf-8') as file: + version = "" + with open(self.source_version_file, "r", encoding="utf-8") as file: source_version_info = file.readlines() for line in source_version_info: @@ -56,21 +65,22 @@ class UpgradeVersionCheck: def source_version_check(self): for white_list_version, white_list_detail in self.white_list_dict.items(): - *white_main_version, white_sub_version = white_list_version.split('.') - *source_main_version, source_sub_version = self.source_version.split('.') + *white_main_version, white_sub_version = white_list_version.split(".") + *source_main_version, source_sub_version = self.source_version.split(".") if source_main_version != white_main_version: continue - if white_sub_version == '*' or white_sub_version == source_sub_version: + if white_sub_version == "*" or white_sub_version == source_sub_version: if "rollup" in white_list_detail[0]: - return 'True {} rollup'.format(white_list_detail[1]) - return 'True {} offline'.format(white_list_detail[1]) + return "True {} rollup".format(white_list_detail[1]) + return "True {} offline".format(white_list_detail[1]) err_msg = "source version '{}' not in white list.".format(self.source_version) return self.execption_handler(err_msg) -if __name__ == '__main__': + +if __name__ == "__main__": version_check = UpgradeVersionCheck() version_check.read_source_version_info() version_check.process_white_list() - print(version_check.source_version_check()) \ No newline at end of file + print(version_check.source_version_check()) diff --git a/pkg/deploy/action/dss/config.py b/pkg/deploy/action/dss/config.py index f55b931b..899d285f 100644 --- a/pkg/deploy/action/dss/config.py +++ b/pkg/deploy/action/dss/config.py @@ -1,8 +1,4 @@ -VG_CONFIG = { - "vg1": "/dev/dss-disk1", - "vg2": "/dev/dss-disk2", - "vg3": "/dev/dss-disk3" -} +VG_CONFIG = {"vg1": "/dev/dss-disk1", "vg2": "/dev/dss-disk2", "vg3": "/dev/dss-disk3"} INST_CONFIG = { "INST_ID": 0, @@ -12,5 +8,5 @@ INST_CONFIG = { "LSNR_PATH": "", "_LOG_BACKUP_FILE_COUNT": "40", "_LOG_MAX_FILE_SIZE": "120M", - "DSS_CM_SO_NAME": "libdsslock.so" -} \ No newline at end of file + "DSS_CM_SO_NAME": "libdsslock.so", +} diff --git a/pkg/deploy/action/dss/dssctl.py b/pkg/deploy/action/dss/dssctl.py index 9b52af8b..f064bae3 100644 --- a/pkg/deploy/action/dss/dssctl.py +++ b/pkg/deploy/action/dss/dssctl.py @@ -45,9 +45,7 @@ def setup(): log = logging.getLogger("dss") for handler in list(log.handlers): log.removeHandler(handler) - file_log = handlers.RotatingFileHandler( - LOG_FILE, maxBytes=6291456, - backupCount=5) + file_log = handlers.RotatingFileHandler(LOG_FILE, maxBytes=6291456, backupCount=5) log.addHandler(file_log) log.addHandler(console) @@ -55,8 +53,10 @@ def setup(): handler.setFormatter( logging.Formatter( fmt="%(asctime)s %(levelname)s [pid:%(process)d] [%(threadName)s]" - " [tid:%(thread)d] [%(filename)s:%(lineno)d %(funcName)s] %(message)s", - datefmt="%Y-%m-%d %H:%M:%S")) + " [tid:%(thread)d] [%(filename)s:%(lineno)d %(funcName)s] %(message)s", + datefmt="%Y-%m-%d %H:%M:%S", + ) + ) log.setLevel(logging.INFO) return log @@ -72,13 +72,13 @@ class ComOpt: content.append("{}{}{}".format(key, split, contents[key])) modes = stat.S_IWRITE | stat.S_IRUSR flags = os.O_WRONLY | os.O_TRUNC | os.O_CREAT - with os.fdopen(os.open(file_path, flags, modes), 'w', encoding='utf-8') as file: + with os.fdopen(os.open(file_path, flags, modes), "w", encoding="utf-8") as file: file.write("\n".join(content)) os.chmod(file_path, 0o640) @staticmethod def read_ini(file_path: str) -> str: - with open(file_path, 'r', encoding="utf-8") as file: + with open(file_path, "r", encoding="utf-8") as file: return file.read() @@ -105,14 +105,15 @@ class DssCtl(object): :param action: add/delete :return: """ - home_directory = os.path.expanduser('~') - bashrc_path = os.path.join(home_directory, '.bashrc') - with open(bashrc_path, 'r') as bashrc_file: + home_directory = os.path.expanduser("~") + bashrc_path = os.path.join(home_directory, ".bashrc") + with open(bashrc_path, "r") as bashrc_file: bashrc_content = bashrc_file.readlines() env = [ "export DSS_HOME=%s\n" % DSS_HOME, - "export LD_LIBRARY_PATH=%s:$LD_LIBRARY_PATH\n" % os.path.join(DSS_HOME, "lib"), - "export PATH=%s:$PATH\n" % os.path.join(DSS_HOME, "bin") + "export LD_LIBRARY_PATH=%s:$LD_LIBRARY_PATH\n" + % os.path.join(DSS_HOME, "lib"), + "export PATH=%s:$PATH\n" % os.path.join(DSS_HOME, "bin"), ] for line in env: if action == "add": @@ -124,7 +125,9 @@ class DssCtl(object): modes = stat.S_IWRITE | stat.S_IRUSR flags = os.O_WRONLY | os.O_TRUNC | os.O_CREAT - with os.fdopen(os.open(bashrc_path, flags, modes), 'w', encoding='utf-8') as bashrc_file: + with os.fdopen( + os.open(bashrc_path, flags, modes), "w", encoding="utf-8" + ) as bashrc_file: bashrc_file.writelines(bashrc_content) @staticmethod @@ -136,12 +139,15 @@ class DssCtl(object): LOG.info(err_msg) if stdout: LOG.info("dss server pid is[%s].", stdout) - for line in re.split(r'\n\s', stdout): + for line in re.split(r"\n\s", stdout): kill_cmd = "kill -9 %s" % line.strip() return_code, stdout, stderr = exec_popen(kill_cmd, timeout=TIMEOUT) if return_code: output = stdout + stderr - err_msg = "Exec kill cmd[%s] failed, details: %s" % (cmd, str(output)) + err_msg = "Exec kill cmd[%s] failed, details: %s" % ( + cmd, + str(output), + ) LOG.error(err_msg) def wait_dss_instance_started(self): @@ -151,16 +157,26 @@ class DssCtl(object): time.sleep(5) timeout = timeout - 5 if os.path.exists(self.log_file): - with open(self.log_file, 'r', errors='ignore') as f: + with open(self.log_file, "r", errors="ignore") as f: all_the_text = f.read() - succ_pattern = re.compile(r'.*(\d{4}\-\d{2}\-\d{2} \d{2}\:\d{2}\:\d{2}).*?DSS SERVER STARTED.*?', - re.IGNORECASE) - fail_pattern = re.compile(r'.*(\d{4}\-\d{2}\-\d{2} \d{2}\:\d{2}\:\d{2}).*?dss failed to startup.*?', - re.IGNORECASE) + succ_pattern = re.compile( + r".*(\d{4}\-\d{2}\-\d{2} \d{2}\:\d{2}\:\d{2}).*?DSS SERVER STARTED.*?", + re.IGNORECASE, + ) + fail_pattern = re.compile( + r".*(\d{4}\-\d{2}\-\d{2} \d{2}\:\d{2}\:\d{2}).*?dss failed to startup.*?", + re.IGNORECASE, + ) succ_timestamps = re.findall(succ_pattern, all_the_text) fail_timestamps = re.findall(fail_pattern, all_the_text) - is_instance_started = len(succ_timestamps) != 0 and max(succ_timestamps) >= self.begin_time - is_instance_failed = len(fail_timestamps) != 0 and max(fail_timestamps) >= self.begin_time + is_instance_started = ( + len(succ_timestamps) != 0 + and max(succ_timestamps) >= self.begin_time + ) + is_instance_failed = ( + len(fail_timestamps) != 0 + and max(fail_timestamps) >= self.begin_time + ) if is_instance_started: LOG.info("DSS server started successfully.") return @@ -182,10 +198,15 @@ class DssCtl(object): LOG.info("Start to exec dsscmd cv.") dsscmd = "source ~/.bashrc && dsscmd cv -g %s -v %s" for key, value in VG_CONFIG.items(): - return_code, stdout, stderr = exec_popen(dsscmd % (key, value), timeout=TIMEOUT) + return_code, stdout, stderr = exec_popen( + dsscmd % (key, value), timeout=TIMEOUT + ) if return_code: output = stdout + stderr - err_msg = "Dsscmd cv cmd[%s] exec failed, details: %s" % (dsscmd % (key, value), str(output)) + err_msg = "Dsscmd cv cmd[%s] exec failed, details: %s" % ( + dsscmd % (key, value), + str(output), + ) raise Exception(err_msg) LOG.info("Success to exec dsscmd cv.") else: @@ -200,10 +221,15 @@ class DssCtl(object): LOG.info("start to init lun.") init_cmd = "dd if=/dev/zero of=%s bs=1M count=1 conv=notrunc" for key, value in VG_CONFIG.items(): - return_code, stdout, stderr = exec_popen(init_cmd % value, timeout=TIMEOUT) + return_code, stdout, stderr = exec_popen( + init_cmd % value, timeout=TIMEOUT + ) if return_code: output = stdout + stderr - err_msg = "Init lun cmd[%s] exec failed, details: %s" % (init_cmd % value, str(output)) + err_msg = "Init lun cmd[%s] exec failed, details: %s" % ( + init_cmd % value, + str(output), + ) raise Exception(err_msg) LOG.info("Init lun cmd[] exec success.", init_cmd) LOG.info("Success to init lun.") @@ -221,7 +247,9 @@ class DssCtl(object): os.makedirs(DSS_CFG, exist_ok=True) ComOpt.write_ini(self.dss_vg_cfg, VG_CONFIG, split=":") INST_CONFIG["INST_ID"] = self.node_id - INST_CONFIG["DSS_NODES_LIST"] = "0:{}:1811,1:{}:1811".format(self.cms_ip.split(";")[0], self.cms_ip.split(";")[1]) + INST_CONFIG["DSS_NODES_LIST"] = "0:{}:1811,1:{}:1811".format( + self.cms_ip.split(";")[0], self.cms_ip.split(";")[1] + ) INST_CONFIG["LSNR_PATH"] = DSS_HOME INST_CONFIG["LOG_HOME"] = DSS_LOG ComOpt.write_ini(self.dss_inst_cfg, INST_CONFIG) @@ -251,8 +279,10 @@ class DssCtl(object): os.chmod(dss_contrl_path, 0o700) if self.node_id == "0": LOG.info("Start to add dss res.") - cmd = ("source ~/.bashrc && %s/bin/cms res -add dss -type dss -attr \"script=%s\"" - % (CMS_HOME, DSS_CTRL_SCRIPTS)) + cmd = ( + 'source ~/.bashrc && %s/bin/cms res -add dss -type dss -attr "script=%s"' + % (CMS_HOME, DSS_CTRL_SCRIPTS) + ) return_code, stdout, stderr = exec_popen(cmd, timeout=TIMEOUT) if return_code: output = stdout + stderr @@ -269,7 +299,7 @@ class DssCtl(object): LOG.info("Start to config perctrl permission.") cap_mode = f"{CAP_ADM},{CAP_WIO}" path = f"{DSS_HOME}/bin/perctrl" - cmd = f'sudo setcap {cap_mode}+ep {path}' + cmd = f"sudo setcap {cap_mode}+ep {path}" return_code, stdout, stderr = exec_popen(cmd, timeout=TIMEOUT) if return_code: output = stdout + stderr @@ -282,7 +312,11 @@ class DssCtl(object): check current node is reg. :return: """ - kick_cmd = "source ~/.bashrc && %s/bin/dsscmd inq_reg -i %s -D %s" % (DSS_HOME, self.node_id, DSS_HOME) + kick_cmd = "source ~/.bashrc && %s/bin/dsscmd inq_reg -i %s -D %s" % ( + DSS_HOME, + self.node_id, + DSS_HOME, + ) return_code, stdout, stderr = exec_popen(kick_cmd, timeout=TIMEOUT) if return_code: output = stdout + stderr @@ -296,7 +330,10 @@ class DssCtl(object): :return: """ LOG.info("Start to kick node.") - kick_cmd = "source ~/.bashrc && %s/bin/dsscmd unreghl -D %s" % (DSS_HOME, DSS_HOME) + kick_cmd = "source ~/.bashrc && %s/bin/dsscmd unreghl -D %s" % ( + DSS_HOME, + DSS_HOME, + ) return_code, stdout, stderr = exec_popen(kick_cmd, timeout=TIMEOUT) if return_code: output = stdout + stderr @@ -360,14 +397,14 @@ class DssCtl(object): self.prepare_source() self.cms_add_dss_res() self.config_perctrl_permission() - + with open(INSTALL_FILE, encoding="utf-8") as f: _tmp = f.read() info = json.loads(_tmp) dss_install_type = info.get("install_type", "") - + LOG.info("dss_install_type is %s", dss_install_type) - + if dss_install_type != "reserve": self.prepare_dss_dick() self.reghl_dss_disk() @@ -377,10 +414,14 @@ class DssCtl(object): LOG.info("Start backup.") if not os.path.exists(BACKUP_NAME): os.makedirs(BACKUP_NAME, exist_ok=True) - shutil.copytree(DSS_CFG, BACKUP_NAME, - symlinks=False, ignore=None, - copy_function=shutil.copy2, - ignore_dangling_symlinks=False) + shutil.copytree( + DSS_CFG, + BACKUP_NAME, + symlinks=False, + ignore=None, + copy_function=shutil.copy2, + ignore_dangling_symlinks=False, + ) if not os.path.exists(os.path.join(BACKUP_NAME, "scripts")): os.makedirs(os.path.join(BACKUP_NAME, "scripts")) shutil.copytree(SCRIPTS_DIR, os.path.join(BACKUP_NAME, "scripts")) @@ -412,7 +453,9 @@ class DssCtl(object): self.reghl_dss_disk() self.begin_time = str(datetime.datetime.now()).split(".")[0] dssserver_cmd = "source ~/.bashrc && nohup dssserver -D %s &" % DSS_HOME - subprocess.Popen(dssserver_cmd, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE) + subprocess.Popen( + dssserver_cmd, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE + ) self.wait_dss_instance_started() if self.check_status(): LOG.info("Success to start dss server.") @@ -451,9 +494,20 @@ class DssCtl(object): def main(): parse = argparse.ArgumentParser() - parse.add_argument("--action", type=str, - choices=["install", "uninstall", "start", "stop", "pre_install", - "upgrade", "rollback", "pre_upgrade"]) + parse.add_argument( + "--action", + type=str, + choices=[ + "install", + "uninstall", + "start", + "stop", + "pre_install", + "upgrade", + "rollback", + "pre_upgrade", + ], + ) parse.add_argument("--mode", required=False, dest="mode", default="") arg = parse.parse_args() act = arg.action @@ -467,4 +521,4 @@ if __name__ == "__main__": except Exception as err: LOG.error(str(err)) exit(str(err)) - exit(0) \ No newline at end of file + exit(0) diff --git a/pkg/deploy/action/fetch_cls_stat.py b/pkg/deploy/action/fetch_cls_stat.py index ed289ad0..fc4ed290 100644 --- a/pkg/deploy/action/fetch_cls_stat.py +++ b/pkg/deploy/action/fetch_cls_stat.py @@ -21,8 +21,13 @@ def _exec_popen(cmd, values=None): if not values: values = [] bash_cmd = ["bash"] - pobj = subprocess.Popen(bash_cmd, shell=False, stdin=subprocess.PIPE, - stdout=subprocess.PIPE, stderr=subprocess.PIPE) + pobj = subprocess.Popen( + bash_cmd, + shell=False, + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + ) py_version = platform.python_version() if py_version[0] == "3": @@ -53,17 +58,17 @@ def _exec_popen(cmd, values=None): def get_user(): config_path = Path(CONFIG_PATH) config_list = json.loads(config_path.read_text()) - return config_list["deploy_user"].split(':')[0] + return config_list["deploy_user"].split(":")[0] def parse_node_stat(node_stat): - keys = ['NODE_ID', 'NAME', 'STAT', 'PRE_STAT'] + keys = ["NODE_ID", "NAME", "STAT", "PRE_STAT"] values = node_stat.split() stat_json = {} - for (idx, key) in enumerate(keys): + for idx, key in enumerate(keys): stat_json[key] = values[idx] online = False - if stat_json.get('STAT') == 'ONLINE': + if stat_json.get("STAT") == "ONLINE": online = True return online, stat_json @@ -72,7 +77,7 @@ def fetch_cms_stat(): user = CANTIAN_USER cmd = 'su - %s -s /bin/bash -c "source ~/.bashrc && cms stat" | tail -n +2' % user _, output, _ = _exec_popen(cmd) - output = output.split('\n') + output = output.split("\n") cms_stat_json = {} if len(output) <= 1: return (False, cms_stat_json) @@ -83,19 +88,19 @@ def fetch_cms_stat(): detail_json.append(stat_json) if online: online_cnt += 1 - cms_stat_json['DETAIL'] = detail_json + cms_stat_json["DETAIL"] = detail_json if online_cnt == 0: - cms_stat_json['STATUS'] = 'OFFLINE' + cms_stat_json["STATUS"] = "OFFLINE" elif online_cnt == len(output): - cms_stat_json['STATUS'] = 'ONLINE' + cms_stat_json["STATUS"] = "ONLINE" else: - cms_stat_json['STATUS'] = 'PARTIALLY_ONLINE' + cms_stat_json["STATUS"] = "PARTIALLY_ONLINE" return (True, cms_stat_json) def gen_fault_result(): result_json = {} - result_json['RESULT'] = -1 + result_json["RESULT"] = -1 return json.dumps(result_json) @@ -104,10 +109,10 @@ def fetch_cls_stat(): if not success: return gen_fault_result() status_json = {} - status_json['CMS_STAT'] = cms_stat_json - status_json['RESULT'] = 0 + status_json["CMS_STAT"] = cms_stat_json + status_json["RESULT"] = 0 return json.dumps(status_json) -if __name__ == '__main__': +if __name__ == "__main__": print(fetch_cls_stat()) diff --git a/pkg/deploy/action/find_next_page.py b/pkg/deploy/action/find_next_page.py index baa63730..bc385136 100644 --- a/pkg/deploy/action/find_next_page.py +++ b/pkg/deploy/action/find_next_page.py @@ -3,7 +3,6 @@ import sys class Finder: - """ information of page 4 page head info { @@ -59,15 +58,15 @@ class Finder: def get_next(line_string): next_page = line_string.split(" ")[2] return next_page - + def get_line_array(self): - with open(self.file_name, 'r') as f: + with open(self.file_name, "r") as f: self.line_array = f.readlines() - + def get_hard_damage_and_checksum(self): - if(self.get_hard_damage(self.line_array[self.line_num]) == "0"): + if self.get_hard_damage(self.line_array[self.line_num]) == "0": self.line_num += 2 - if (self.get_checksum(self.line_array[self.line_num]) == "success"): + if self.get_checksum(self.line_array[self.line_num]) == "success": return 1 return 0 @@ -82,21 +81,28 @@ class Finder: tmp_next = "0-0" tmp_page_type_corr_and_not_damage = 0 while self.line_num < len(self.line_array): - if (self.line_array[self.line_num].startswith(self.page_id_start)): + if self.line_array[self.line_num].startswith(self.page_id_start): tmp_page_id = self.get_page_id(self.line_array[self.line_num]) - if (self.get_type(self.line_array[self.line_num]) == self.page_pcr_heap and - self.get_hard_damage_and_checksum() == 1): + if ( + self.get_type(self.line_array[self.line_num]) == self.page_pcr_heap + and self.get_hard_damage_and_checksum() == 1 + ): tmp_page_type_corr_and_not_damage = 1 - - if (tmp_page_type_corr_and_not_damage == 1 and - self.line_array[self.line_num].startswith(self.page_next_start)): + + if tmp_page_type_corr_and_not_damage == 1 and self.line_array[ + self.line_num + ].startswith(self.page_next_start): tmp_next = self.get_next(self.line_array[self.line_num]) - if (tmp_next == self.target_page_id): + if tmp_next == self.target_page_id: self.prev_page_id = tmp_page_id - print_message("find the prev corr page: " + str(self.prev_page_id) + ", whose next is " + - str(self.target_page_id)) + print_message( + "find the prev corr page: " + + str(self.prev_page_id) + + ", whose next is " + + str(self.target_page_id) + ) return - if (self.line_array[self.line_num].startswith(self.page_start)): + if self.line_array[self.line_num].startswith(self.page_start): return self.line_num += 1 return @@ -110,20 +116,33 @@ class Finder: tmp_page_id = "0-0" tmp_find_next = 0 while self.line_num < len(self.line_array): - if (self.line_array[self.line_num].startswith(self.page_id_start) and - self.get_page_id(self.line_array[self.line_num]) == self.target_page_id): + if ( + self.line_array[self.line_num].startswith(self.page_id_start) + and self.get_page_id(self.line_array[self.line_num]) + == self.target_page_id + ): tmp_find_next = 1 tmp_page_id = self.get_page_id(self.line_array[self.line_num]) - if (self.get_hard_damage_and_checksum() == 1): + if self.get_hard_damage_and_checksum() == 1: self.correct_page_id = tmp_page_id - print_message("the target page[" + str(tmp_page_id) + "] is correct!") + print_message( + "the target page[" + str(tmp_page_id) + "] is correct!" + ) return - if (tmp_find_next == 1 and self.line_array[self.line_num].startswith(self.page_next_start)): + if tmp_find_next == 1 and self.line_array[self.line_num].startswith( + self.page_next_start + ): self.target_page_id = self.get_next(self.line_array[self.line_num]) - print_message("the bad page[" + str(tmp_page_id) + "] is bad, continue to find [" + - str(tmp_page_id) + "]'s next: " + str(self.target_page_id)) + print_message( + "the bad page[" + + str(tmp_page_id) + + "] is bad, continue to find [" + + str(tmp_page_id) + + "]'s next: " + + str(self.target_page_id) + ) return - if (self.line_array[self.line_num].startswith(self.page_start)): + if self.line_array[self.line_num].startswith(self.page_start): return self.line_num += 1 return @@ -131,12 +150,12 @@ class Finder: def get(self): self.get_line_array() while self.line_num < len(self.line_array): - if (self.line_array[self.line_num].startswith(self.page_id_start)): + if self.line_array[self.line_num].startswith(self.page_id_start): if self.prev_page_id == "0-0": self.handle_page() else: self.handle_bad_page() - if (self.correct_page_id != "0-0"): + if self.correct_page_id != "0-0": print_message("find the correct next: " + str(self.correct_page_id)) break self.line_num += 1 @@ -148,6 +167,7 @@ class Options(object): self.file_name = "" self.target_page_id = "" + g_opts = Options() @@ -170,12 +190,12 @@ def parse_parameter(): g_opts.target_page_id = _value else: raise Exception("error") - + except getopt.GetoptError as err: raise Exception("Parameter input error: " + err.msg) from err - if (g_opts.file_name == ""): + if g_opts.file_name == "": raise Exception("please input the correct file_name.") - if (g_opts.target_page_id == ""): + if g_opts.target_page_id == "": raise Exception("please input the correct page_id.") @@ -191,18 +211,25 @@ def main(): parse_parameter() finder = Finder(g_opts.file_name, g_opts.target_page_id) finder.get() - if (finder.correct_page_id != "0-0"): - print_message("please set [" + str(finder.prev_page_id) + "]'s next to [" + - str(finder.correct_page_id) + "].") - elif (finder.prev_page_id == "0-0"): - print_message("the bad page [" + str(finder.target_page_id) + "] has no prev_page.") + if finder.correct_page_id != "0-0": + print_message( + "please set [" + + str(finder.prev_page_id) + + "]'s next to [" + + str(finder.correct_page_id) + + "]." + ) + elif finder.prev_page_id == "0-0": + print_message( + "the bad page [" + str(finder.target_page_id) + "] has no prev_page." + ) else: print_message("Cannot find the next correct page.") - + if __name__ == "__main__": try: main() except Exception as ex: print_message("Error: " + str(ex)) - exit(1) \ No newline at end of file + exit(1) diff --git a/pkg/deploy/action/get_config_info.py b/pkg/deploy/action/get_config_info.py index 751ed255..92e645c4 100644 --- a/pkg/deploy/action/get_config_info.py +++ b/pkg/deploy/action/get_config_info.py @@ -15,19 +15,21 @@ def get_value(param): _tmp = f.read() info = json.loads(_tmp) # deploy_user 格式为：用户:用户组 - if param == 'deploy_user': - return info.get('deploy_user').split(':')[0] + if param == "deploy_user": + return info.get("deploy_user").split(":")[0] - if param == 'deploy_group': - return info.get('deploy_user').split(':')[1] + if param == "deploy_group": + return info.get("deploy_user").split(":")[1] if param == "cluster_scale": return len(info.get("cms_ip").split(";")) if param == "deploy_mode": - if info.get('deploy_mode', ""): - return info.get('deploy_mode') - return "dbstor" if info.get('deploy_policy', "") in ["ModeB", "ModeC"] else "file" + if info.get("deploy_mode", ""): + return info.get("deploy_mode") + return ( + "dbstor" if info.get("deploy_policy", "") in ["ModeB", "ModeC"] else "file" + ) return info.get(param, "") diff --git a/pkg/deploy/action/get_source_version.py b/pkg/deploy/action/get_source_version.py index 759cbca3..3c44a6b6 100644 --- a/pkg/deploy/action/get_source_version.py +++ b/pkg/deploy/action/get_source_version.py @@ -2,12 +2,12 @@ from upgrade_version_check import UpgradeVersionCheck from om_log import LOGGER as LOG -if __name__ == '__main__': +if __name__ == "__main__": version_check = UpgradeVersionCheck() try: version_check.read_source_version_info() except Exception as err: - LOG.error(f'obtain source version failed with error: {str(err)}') - exit('') + LOG.error(f"obtain source version failed with error: {str(err)}") + exit("") print(version_check.source_version) diff --git a/pkg/deploy/action/implement/certificate_update_and_revocation.py b/pkg/deploy/action/implement/certificate_update_and_revocation.py index 9e6bd3dd..1ce13ac7 100644 --- a/pkg/deploy/action/implement/certificate_update_and_revocation.py +++ b/pkg/deploy/action/implement/certificate_update_and_revocation.py @@ -14,11 +14,12 @@ CUR_PATH = os.path.dirname(os.path.realpath(__file__)) sys.path.append(os.path.join(CUR_PATH, "../")) from logic.common_func import exec_popen from om_log import LOGGER as LOG + ENV_FILE = "/opt/cantian/action/env.sh" def get_param_value(param): - with open(ENV_FILE, 'r', encoding='utf-8') as file: + with open(ENV_FILE, "r", encoding="utf-8") as file: env_config = file.readlines() if param == "cantian_user": for line in env_config: @@ -120,11 +121,13 @@ class CertificateUpdateAndRevocation(object): """ 更新证书密码 """ - cmd = "su -s /bin/bash - cantian -c \"" + cmd = 'su -s /bin/bash - cantian -c "' cmd += "tmp_path=${LD_LIBRARY_PATH};export LD_LIBRARY_PATH=/opt/cantian/dbstor/lib:${LD_LIBRARY_PATH};" - cmd += f"echo -e '{passwd}' | python3 -B /opt/cantian/action/implement" \ - f"/update_cantian_passwd.py update_mes_key_pwd;" - cmd += "export LD_LIBRARY_PATH=${tmp_path}\"" + cmd += ( + f"echo -e '{passwd}' | python3 -B /opt/cantian/action/implement" + f"/update_cantian_passwd.py update_mes_key_pwd;" + ) + cmd += 'export LD_LIBRARY_PATH=${tmp_path}"' ret_code, _, stderr = exec_popen(cmd) stderr = str(stderr) stderr.replace(passwd, "****") @@ -208,6 +211,7 @@ class CertificateUpdateAndRevocation(object): raise Exception("query certifcate info failed, output:%s" % str(stderr)) print(str(stdout)) + if __name__ == "__main__": cert_update_and_revocation = CertificateUpdateAndRevocation() _args = [] @@ -217,13 +221,15 @@ if __name__ == "__main__": try: getattr(cert_update_and_revocation, action) except AttributeError as err: - err_msg = "Currently, you can modify the certificate revocation list,"\ - " update certificates, and query certificate information.\n"\ - "example:\n"\ - "query_crt_info\n"\ - "update_crt_key cert_file_path, key_file_path\n"\ - "update_ca_crt_key ca_file_path, cert_file_path, key_file_path\n"\ - "update_certificate_crl crl_file_path" + err_msg = ( + "Currently, you can modify the certificate revocation list," + " update certificates, and query certificate information.\n" + "example:\n" + "query_crt_info\n" + "update_crt_key cert_file_path, key_file_path\n" + "update_ca_crt_key ca_file_path, cert_file_path, key_file_path\n" + "update_certificate_crl crl_file_path" + ) exit(err_msg) try: getattr(cert_update_and_revocation, action)(*_args) diff --git a/pkg/deploy/action/implement/check_deploy_param.py b/pkg/deploy/action/implement/check_deploy_param.py index c2129363..60037349 100644 --- a/pkg/deploy/action/implement/check_deploy_param.py +++ b/pkg/deploy/action/implement/check_deploy_param.py @@ -3,7 +3,7 @@ import os CUR_PATH = os.path.dirname(os.path.realpath(__file__)) -DEPLOY_PARAM_FILE = os.path.join(CUR_PATH, '../../config/deploy_param.json') +DEPLOY_PARAM_FILE = os.path.join(CUR_PATH, "../../config/deploy_param.json") CHECK_LIST = [ "cluster_id", "cluster_name", @@ -16,7 +16,7 @@ CHECK_LIST = [ "mes_ssl_switch", "link_type", "db_type", - "mysql_metadata_in_cantian" + "mysql_metadata_in_cantian", ] @@ -29,11 +29,16 @@ def read_file(file_path): def check_deploy_param(): local_deploy_params = read_file(DEPLOY_PARAM_FILE) storage_metadata_fs = local_deploy_params.get("storage_metadata_fs") - remote_deploy_file = f"/mnt/dbdata/remote/metadata_{storage_metadata_fs}/deploy_param.json" + remote_deploy_file = ( + f"/mnt/dbdata/remote/metadata_{storage_metadata_fs}/deploy_param.json" + ) if not os.path.exists(remote_deploy_file): - err_msg = "%s is not exists, please check:\n" \ - "\t1、node 0 has been successfully installed.\n" \ - "\t2、storage_metadata_fs field in the configuration file same as node 0." % remote_deploy_file + err_msg = ( + "%s is not exists, please check:\n" + "\t1、node 0 has been successfully installed.\n" + "\t2、storage_metadata_fs field in the configuration file same as node 0." + % remote_deploy_file + ) raise Exception(err_msg) remote_deploy_params = read_file(remote_deploy_file) check_failed_list = [] @@ -41,8 +46,10 @@ def check_deploy_param(): if local_deploy_params.get(check_key) != remote_deploy_params.get(check_key): check_failed_list.append(check_key) if check_failed_list: - err_msg = "The configuration items of the current node are different from " \ - "those of node 0, details:%s" % check_failed_list + err_msg = ( + "The configuration items of the current node are different from " + "those of node 0, details:%s" % check_failed_list + ) raise Exception(err_msg) diff --git a/pkg/deploy/action/implement/check_nfs4_mandatory_lock_switch.py b/pkg/deploy/action/implement/check_nfs4_mandatory_lock_switch.py index f40f43cc..94343a43 100644 --- a/pkg/deploy/action/implement/check_nfs4_mandatory_lock_switch.py +++ b/pkg/deploy/action/implement/check_nfs4_mandatory_lock_switch.py @@ -10,7 +10,9 @@ from utils.client.ssh_client import SshClient from utils.client.rest_client import read_helper -DEPLOY_PARAMS = os.path.join(str(pathlib.Path(CUR_PATH).parent.parent), "config/deploy_param.json") +DEPLOY_PARAMS = os.path.join( + str(pathlib.Path(CUR_PATH).parent.parent), "config/deploy_param.json" +) class CheckLockSwitch(object): @@ -42,7 +44,9 @@ class CheckLockSwitch(object): if "NFSV4 Mandatory Lock Switch" in key and "Enabled" in value: break else: - err_msg = "Current NFSV4 Mandatory Lock Switch is disabled, details:%s" % res + err_msg = ( + "Current NFSV4 Mandatory Lock Switch is disabled, details:%s" % res + ) LOGGER.error(err_msg) raise Exception(err_msg) LOGGER.info("Success to check NFSV4 Mandatory Lock Switch") diff --git a/pkg/deploy/action/implement/check_pwd.py b/pkg/deploy/action/implement/check_pwd.py index d112036e..5dc36845 100644 --- a/pkg/deploy/action/implement/check_pwd.py +++ b/pkg/deploy/action/implement/check_pwd.py @@ -1,6 +1,7 @@ import json import sys import os + CUR_PATH = os.path.dirname(os.path.realpath(__file__)) sys.path.append(os.path.join(CUR_PATH, "../")) from logic.common_func import exec_popen @@ -12,7 +13,7 @@ PWD_LEN = 8 class PassWordChecker: def __init__(self, pwd): self.pwd = pwd - self.user = 'ctcliuser' + self.user = "ctcliuser" @staticmethod def check_key_passwd(key_file_path, passwd): @@ -25,7 +26,7 @@ class PassWordChecker: stderr.replace(passwd, "****") if ret_code: raise Exception("The password is incorrect.") - + @staticmethod def get_crt_modulus(cert_file_path): """ @@ -36,7 +37,7 @@ class PassWordChecker: if ret_code: raise Exception("Failed to get crt modulus, output:%s" % str(stderr)) return str(stdout) - + @staticmethod def get_key_modulus(key_file_path, passwd): """ @@ -72,14 +73,19 @@ class PassWordChecker: if passwd_set & cases: types += 1 if types < 3: - LOG.error("Error: Password must contains at least three different types of characters.") + LOG.error( + "Error: Password must contains at least three different types of characters." + ) return 1 # Only can contains enumerated cases all_cases = upper_cases | lower_cases | digits | special_cases un_cases = passwd_set - all_cases if un_cases: - LOG.error("Error: There are characters that are not allowed in the password: '%s'", "".join(un_cases)) + LOG.error( + "Error: There are characters that are not allowed in the password: '%s'", + "".join(un_cases), + ) return 1 return 0 @@ -103,13 +109,13 @@ class PassWordChecker: raise Exception("The certificate and private key do not match.") -if __name__ == '__main__': +if __name__ == "__main__": pwd_checker = PassWordChecker(input()) action = "check_pwd" if len(sys.argv) > 1: action = sys.argv[1] operator = { "check_pwd": pwd_checker.verify_new_passwd, - "check_cert_pwd": pwd_checker.check_cert_passwd + "check_cert_pwd": pwd_checker.check_cert_passwd, } exit(operator.get(action)()) diff --git a/pkg/deploy/action/implement/config_opt.py b/pkg/deploy/action/implement/config_opt.py index 8516d18d..a8a1b5c9 100644 --- a/pkg/deploy/action/implement/config_opt.py +++ b/pkg/deploy/action/implement/config_opt.py @@ -4,13 +4,14 @@ import json import argparse import traceback + def opt_ini_conf(file_path, action, key, value): - with open(file_path, "r", encoding="utf-8")as fp: + with open(file_path, "r", encoding="utf-8") as fp: config = fp.readlines() for i, item in enumerate(config): if "=" not in item: continue - _key, _value = item.split('=', maxsplit=1) + _key, _value = item.split("=", maxsplit=1) if key == _key.strip(" "): if action == "modify": config[i] = f"{key} = {value}\n" @@ -40,11 +41,12 @@ def cms_opt_ini_conf(action, key, value): def main(): update_parse = argparse.ArgumentParser() - update_parse.add_argument("-c", "--component", dest="component", - choices=["cms", "cantian"], - required=True) - update_parse.add_argument("-a", "--action", dest="action", choices=["query", "modify"], - required=True) + update_parse.add_argument( + "-c", "--component", dest="component", choices=["cms", "cantian"], required=True + ) + update_parse.add_argument( + "-a", "--action", dest="action", choices=["query", "modify"], required=True + ) update_parse.add_argument("-k", "--key", dest="key", required=True) update_parse.add_argument("-v", "--value", dest="value", required=False) args = update_parse.parse_args() @@ -63,4 +65,4 @@ if __name__ == "__main__": try: main() except Exception as err: - exit(str(traceback.format_exc(limit=1))) \ No newline at end of file + exit(str(traceback.format_exc(limit=1))) diff --git a/pkg/deploy/action/implement/get_source_version.py b/pkg/deploy/action/implement/get_source_version.py index 19c9e68a..56413184 100644 --- a/pkg/deploy/action/implement/get_source_version.py +++ b/pkg/deploy/action/implement/get_source_version.py @@ -1,17 +1,18 @@ import sys import os + CUR_PATH = os.path.dirname(os.path.realpath(__file__)) sys.path.append(os.path.join(CUR_PATH, "../")) from implement.upgrade_version_check import UpgradeVersionCheck from om_log import LOGGER as LOG -if __name__ == '__main__': +if __name__ == "__main__": version_check = UpgradeVersionCheck() try: version_check.read_source_version_info() except Exception as err: - LOG.error('obtain source version failed with error: %s', str(err)) - exit('') + LOG.error("obtain source version failed with error: %s", str(err)) + exit("") print(version_check.source_version) diff --git a/pkg/deploy/action/implement/update_cantian_passwd.py b/pkg/deploy/action/implement/update_cantian_passwd.py index 55a2d1f8..b5839357 100644 --- a/pkg/deploy/action/implement/update_cantian_passwd.py +++ b/pkg/deploy/action/implement/update_cantian_passwd.py @@ -3,7 +3,9 @@ import sys import os import json -sys.path.append(os.path.join(os.path.dirname(os.path.abspath(__file__)), "..", "dbstor")) +sys.path.append( + os.path.join(os.path.dirname(os.path.abspath(__file__)), "..", "dbstor") +) from kmc_adapter import CApiWrapper @@ -37,7 +39,9 @@ def update_mes_key_pwd(plain_text): try: ret_pwd = kmc_adapter.encrypt(plain_text) except Exception as error: - raise Exception("Failed to encrypt password of user [sys]. Error: %s" % str(error)) from error + raise Exception( + "Failed to encrypt password of user [sys]. Error: %s" % str(error) + ) from error cantian_config = "/mnt/dbdata/local/cantian/tmp/data/cfg/cantiand.ini" cms_config = "/opt/cantian/cms/cfg/cms.ini" @@ -49,7 +53,5 @@ def update_mes_key_pwd(plain_text): if __name__ == "__main__": passwd = input() action = sys.argv[1] - options = { - "update_mes_key_pwd": update_mes_key_pwd - } + options = {"update_mes_key_pwd": update_mes_key_pwd} options.get(action)(passwd.strip()) diff --git a/pkg/deploy/action/implement/upgrade_version_check.py b/pkg/deploy/action/implement/upgrade_version_check.py index 96933b3d..d93a1399 100644 --- a/pkg/deploy/action/implement/upgrade_version_check.py +++ b/pkg/deploy/action/implement/upgrade_version_check.py @@ -7,8 +7,8 @@ CUR_PATH = os.path.dirname(os.path.realpath(__file__)) sys.path.append(os.path.join(CUR_PATH, "../")) from om_log import LOGGER as LOG -VERSION_PREFIX = 'Version:' -SUB_VERSION_PREFIX = ('B', 'SP') +VERSION_PREFIX = "Version:" +SUB_VERSION_PREFIX = ("B", "SP") class UpgradeVersionCheck: @@ -16,12 +16,14 @@ class UpgradeVersionCheck: def __init__(self, white_list=None, upgrade_mode=None): self.white_list_file = white_list self.upgrade_mode = upgrade_mode - self.source_version_file = str(Path('/opt/cantian/versions.yml')) - self.white_list_dict = {} # 格式：{SOURCE-VERSION: [UPGRADE-MODE, CHANGE-SYSTEM]} - self.source_version = '' + self.source_version_file = str(Path("/opt/cantian/versions.yml")) + self.white_list_dict = ( + {} + ) # 格式：{SOURCE-VERSION: [UPGRADE-MODE, CHANGE-SYSTEM]} + self.source_version = "" def process_white_list(self): - with open(self.white_list_file, 'r', encoding='utf-8') as file: + with open(self.white_list_file, "r", encoding="utf-8") as file: white_list_info = file.readlines() for white_list_detail in white_list_info[1:]: @@ -31,8 +33,8 @@ class UpgradeVersionCheck: self.white_list_dict[details[0]] = [details[1], details[2]] def read_source_version_info(self): - version = '' - with open(self.source_version_file, 'r', encoding='utf-8') as file: + version = "" + with open(self.source_version_file, "r", encoding="utf-8") as file: source_version_info = file.readlines() for line in source_version_info: @@ -46,52 +48,62 @@ class UpgradeVersionCheck: for white_list_version, white_list_detail in self.white_list_dict.items(): if self.upgrade_mode not in white_list_detail[0]: continue - *white_main_version, white_sub_version = white_list_version.split('.') - *source_main_version, source_sub_version = self.source_version.split('.') + *white_main_version, white_sub_version = white_list_version.split(".") + *source_main_version, source_sub_version = self.source_version.split(".") if source_main_version != white_main_version: continue - if white_sub_version == '*' or white_sub_version == source_sub_version: - result = "{} {} {}".format(self.source_version, white_list_detail[0], white_list_detail[1]) + if white_sub_version == "*" or white_sub_version == source_sub_version: + result = "{} {} {}".format( + self.source_version, white_list_detail[0], white_list_detail[1] + ) break - if '-' in white_sub_version: - min_version, max_version = white_sub_version.split('-') - trans_map = str.maketrans('', '', digits) + if "-" in white_sub_version: + min_version, max_version = white_sub_version.split("-") + trans_map = str.maketrans("", "", digits) source_pre_fix = source_sub_version.translate(trans_map) - if source_pre_fix not in SUB_VERSION_PREFIX: # 源版本号开头不是B或者SPH返回结果为空 + if ( + source_pre_fix not in SUB_VERSION_PREFIX + ): # 源版本号开头不是B或者SPH返回结果为空 break - sub_version_min_num = min_version.replace(source_pre_fix, '') - sub_version_max_num = max_version.replace(source_pre_fix, '') - sub_source_version_num = source_sub_version.replace(source_pre_fix, '') - if sub_version_min_num.isdigit() \ - and sub_version_max_num.isdigit() \ - and int(sub_version_max_num) >= int(sub_source_version_num) >= int(sub_version_min_num): - result = "{} {} {}".format(self.source_version, white_list_detail[0], white_list_detail[1]) + sub_version_min_num = min_version.replace(source_pre_fix, "") + sub_version_max_num = max_version.replace(source_pre_fix, "") + sub_source_version_num = source_sub_version.replace(source_pre_fix, "") + if ( + sub_version_min_num.isdigit() + and sub_version_max_num.isdigit() + and int(sub_version_max_num) + >= int(sub_source_version_num) + >= int(sub_version_min_num) + ): + result = "{} {} {}".format( + self.source_version, white_list_detail[0], white_list_detail[1] + ) break return result -if __name__ == '__main__': +if __name__ == "__main__": white_list_input = sys.argv[1] upgrade_mode_input = sys.argv[2] version_check = UpgradeVersionCheck(white_list_input, upgrade_mode_input) try: version_check.process_white_list() except Exception as err: - LOG.error('obtain source version white list failed with error: %s', str(err)) - exit('') + LOG.error("obtain source version white list failed with error: %s", str(err)) + exit("") try: version_check.read_source_version_info() except Exception as err: - LOG.error('obtain source version failed with error: %s', str(err)) - exit('') + LOG.error("obtain source version failed with error: %s", str(err)) + exit("") try: print(version_check.source_version_check()) except Exception as err: - LOG.error('source version check failed with error: %s', str(err)) - exit('') + LOG.error("source version check failed with error: %s", str(err)) + exit("") diff --git a/pkg/deploy/action/inspection/declear_env.py b/pkg/deploy/action/inspection/declear_env.py index 3c0a2d90..2a55d0f0 100644 --- a/pkg/deploy/action/inspection/declear_env.py +++ b/pkg/deploy/action/inspection/declear_env.py @@ -26,9 +26,9 @@ class DeclearEnv: string: cantian or mysql """ if os.path.exists(self.version_file): - return 'cantian' + return "cantian" - return 'mysql' + return "mysql" def get_executor(self): """ diff --git a/pkg/deploy/action/inspection/generate_html_results.py b/pkg/deploy/action/inspection/generate_html_results.py index 7b4770c4..2b367f42 100644 --- a/pkg/deploy/action/inspection/generate_html_results.py +++ b/pkg/deploy/action/inspection/generate_html_results.py @@ -3,7 +3,7 @@ import os import stat from pathlib import Path -TEMPLETE_BODY_EN = ''' +TEMPLETE_BODY_EN = """ @@ -15,9 +15,9 @@ TEMPLETE_BODY_EN = ''' {} -''' +""" -TEMPLETE_BODY_ZH = ''' +TEMPLETE_BODY_ZH = """ @@ -29,8 +29,8 @@ TEMPLETE_BODY_ZH = ''' {} -''' -TEMPLETE_DEV = ''' +""" +TEMPLETE_DEV = """

{inspection_name}

@@ -86,7 +86,7 @@ TEMPLETE_DEV = '''
-''' +""" class GenHtmlRes(object): @@ -97,10 +97,10 @@ class GenHtmlRes(object): def write_file(self, content, lang="zh"): file_name = "{}_inspection_result_{}.html".format(self.node_info, lang) - file_path = str(Path(self.file_path + '/' + file_name)) + file_path = str(Path(self.file_path + "/" + file_name)) modes = stat.S_IWRITE | stat.S_IRUSR flags = os.O_WRONLY | os.O_TRUNC | os.O_CREAT - with os.fdopen(os.open(file_path, flags, modes), 'w', encoding='utf-8') as file: + with os.fdopen(os.open(file_path, flags, modes), "w", encoding="utf-8") as file: file.write(content) def generate_html_zh(self): @@ -108,50 +108,72 @@ class GenHtmlRes(object): for item in self.inspect_res: resource_zh = item.get("resource_zh") result = "成功" if item.get("inspection_result") == "success" else "失败" - display = "none" if item.get("inspection_result") == "success" else 'contents' - inspection_detail = json.dumps(item.get("inspection_detail"), indent=4, ensure_ascii=False).strip("\"") - method_detail = json.dumps(resource_zh.get("检查步骤"), indent=4, ensure_ascii=False).strip("\"") - criteria = json.dumps(resource_zh.get("检查方法"), indent=4, ensure_ascii=False).strip("\"") - suggestion = json.dumps(resource_zh.get("修复建议"), indent=4, ensure_ascii=False).strip("\"") - dev_one = TEMPLETE_DEV.format(inspection_name=item.get("description_zn"), - information="原始信息", - information_detail=inspection_detail.replace('\\n', "
"), - method="检查步骤", - method_detail=method_detail.replace('\\n', "
"), - criteria="检查方法", - criteria_detail=criteria.replace('\\n', "
"), - suggestion="修复建议", - suggestion_detail=suggestion.replace('\\n', "
"), - result="检查结果", - result_detail=result, - display=display - ) + display = ( + "none" if item.get("inspection_result") == "success" else "contents" + ) + inspection_detail = json.dumps( + item.get("inspection_detail"), indent=4, ensure_ascii=False + ).strip('"') + method_detail = json.dumps( + resource_zh.get("检查步骤"), indent=4, ensure_ascii=False + ).strip('"') + criteria = json.dumps( + resource_zh.get("检查方法"), indent=4, ensure_ascii=False + ).strip('"') + suggestion = json.dumps( + resource_zh.get("修复建议"), indent=4, ensure_ascii=False + ).strip('"') + dev_one = TEMPLETE_DEV.format( + inspection_name=item.get("description_zn"), + information="原始信息", + information_detail=inspection_detail.replace("\\n", "
"), + method="检查步骤", + method_detail=method_detail.replace("\\n", "
"), + criteria="检查方法", + criteria_detail=criteria.replace("\\n", "
"), + suggestion="修复建议", + suggestion_detail=suggestion.replace("\\n", "
"), + result="检查结果", + result_detail=result, + display=display, + ) dev_list += dev_one zh_html_res = TEMPLETE_BODY_ZH.format(dev_list) - self.write_file(zh_html_res, 'zh') + self.write_file(zh_html_res, "zh") def generate_html_en(self): dev_list = "" for item in self.inspect_res: resource_en = item.get("resource_en") - display = "none" if item.get("inspection_result") == "success" else 'contents' - inspection_detail = json.dumps(item.get("inspection_detail"), indent=4, ensure_ascii=False).strip("\"") - method_detail = json.dumps(resource_en.get("method"), indent=4, ensure_ascii=False).strip("\"") - criteria = json.dumps(resource_en.get("description"), indent=4, ensure_ascii=False).strip("\"") - suggestion = json.dumps(resource_en.get("suggestion"), indent=4, ensure_ascii=False).strip("\"") - dev_one = TEMPLETE_DEV.format(inspection_name=item.get("description_en"), - information="information", - information_detail=inspection_detail.replace('\\n', "
"), - method="method", - method_detail=method_detail.replace('\\n', "
"), - criteria="description", - criteria_detail=criteria.replace('\\n', "
"), - suggestion="suggestion", - suggestion_detail=suggestion.replace('\\n', "
"), - result="result", - result_detail=item.get("inspection_result"), - display=display - ) + display = ( + "none" if item.get("inspection_result") == "success" else "contents" + ) + inspection_detail = json.dumps( + item.get("inspection_detail"), indent=4, ensure_ascii=False + ).strip('"') + method_detail = json.dumps( + resource_en.get("method"), indent=4, ensure_ascii=False + ).strip('"') + criteria = json.dumps( + resource_en.get("description"), indent=4, ensure_ascii=False + ).strip('"') + suggestion = json.dumps( + resource_en.get("suggestion"), indent=4, ensure_ascii=False + ).strip('"') + dev_one = TEMPLETE_DEV.format( + inspection_name=item.get("description_en"), + information="information", + information_detail=inspection_detail.replace("\\n", "
"), + method="method", + method_detail=method_detail.replace("\\n", "
"), + criteria="description", + criteria_detail=criteria.replace("\\n", "
"), + suggestion="suggestion", + suggestion_detail=suggestion.replace("\\n", "
"), + result="result", + result_detail=item.get("inspection_result"), + display=display, + ) dev_list += dev_one en_html_res = TEMPLETE_BODY_EN.format(dev_list) - self.write_file(en_html_res, 'en') + self.write_file(en_html_res, "en") diff --git a/pkg/deploy/action/inspection/inspection_cantian.py b/pkg/deploy/action/inspection/inspection_cantian.py index 7805a252..7ea06e32 100644 --- a/pkg/deploy/action/inspection/inspection_cantian.py +++ b/pkg/deploy/action/inspection/inspection_cantian.py @@ -8,9 +8,9 @@ from pathlib import Path from declear_env import DeclearEnv from inspection_task import InspectionTask -DEPLOY_CONFIG_FILE = str(Path('/opt/cantian/config/deploy_param.json')) +DEPLOY_CONFIG_FILE = str(Path("/opt/cantian/config/deploy_param.json")) DIR_NAME, _ = os.path.split(os.path.abspath(__file__)) -INSPECTION_JSON_FILE = str(Path('{}/inspection_config.json'.format(DIR_NAME))) +INSPECTION_JSON_FILE = str(Path("{}/inspection_config.json".format(DIR_NAME))) class CantianInspection(InspectionTask): @@ -38,10 +38,10 @@ class CantianInspection(InspectionTask): :return: string: node id """ - with open(DEPLOY_CONFIG_FILE, encoding='utf-8') as file: + with open(DEPLOY_CONFIG_FILE, encoding="utf-8") as file: deploy_info = json.load(file) - node_id = deploy_info.get('node_id').split(':')[0] - cms_ip = deploy_info.get('cms_ip').split(';') + node_id = deploy_info.get("node_id").split(":")[0] + cms_ip = deploy_info.get("cms_ip").split(";") node_ip = cms_ip[int(node_id)] return "cantian_" + node_ip @@ -51,13 +51,13 @@ class CantianInspection(InspectionTask): raise ValueError(f"inspection must be executed by {self.deply_user}") def task_execute_single(self, inspection_detail, name_pwd, ip_port): - echo_sentence = '' - _ip = '' - _port = '' - inspection_item_file = inspection_detail.get('inspection_file_path') - inspection_item_input = inspection_detail.get('script_input') - component_belong = inspection_detail.get('component') - time_out = int(inspection_detail.get('time_out')) + echo_sentence = "" + _ip = "" + _port = "" + inspection_item_file = inspection_detail.get("inspection_file_path") + inspection_item_input = inspection_detail.get("script_input") + component_belong = inspection_detail.get("component") + time_out = int(inspection_detail.get("time_out")) if name_pwd: echo_sentence = f'echo -e "{name_pwd[0]}\n{name_pwd[1]}"' @@ -67,31 +67,59 @@ class CantianInspection(InspectionTask): _port = ip_port[1] if component_belong not in self.user_map.keys(): - raise Exception(f'Module {component_belong} not exist') + raise Exception(f"Module {component_belong} not exist") if inspection_item_input: if echo_sentence: echo_cmd = shlex.split(echo_sentence) - echo_popen = subprocess.Popen(echo_cmd, stdout=subprocess.PIPE, shell=False) - single_inspection_popen = subprocess.Popen([f'/usr/bin/python3', inspection_item_file, - inspection_item_input, _ip, _port], stdin=echo_popen.stdout, - stdout=subprocess.PIPE, shell=False) + echo_popen = subprocess.Popen( + echo_cmd, stdout=subprocess.PIPE, shell=False + ) + single_inspection_popen = subprocess.Popen( + [ + f"/usr/bin/python3", + inspection_item_file, + inspection_item_input, + _ip, + _port, + ], + stdin=echo_popen.stdout, + stdout=subprocess.PIPE, + shell=False, + ) else: - single_inspection_popen = subprocess.Popen([f'{echo_sentence}/usr/bin/python3', inspection_item_file, - inspection_item_input, _ip, _port], - stdout=subprocess.PIPE, shell=False) + single_inspection_popen = subprocess.Popen( + [ + f"{echo_sentence}/usr/bin/python3", + inspection_item_file, + inspection_item_input, + _ip, + _port, + ], + stdout=subprocess.PIPE, + shell=False, + ) else: if echo_sentence: echo_cmd = shlex.split(echo_sentence) - echo_popen = subprocess.Popen(echo_cmd, stdout=subprocess.PIPE, shell=False) - single_inspection_popen = subprocess.Popen([f'/usr/bin/python3', inspection_item_file, - _ip, _port], stdin=echo_popen.stdout, - stdout=subprocess.PIPE, shell=False) + echo_popen = subprocess.Popen( + echo_cmd, stdout=subprocess.PIPE, shell=False + ) + single_inspection_popen = subprocess.Popen( + [f"/usr/bin/python3", inspection_item_file, _ip, _port], + stdin=echo_popen.stdout, + stdout=subprocess.PIPE, + shell=False, + ) else: - single_inspection_popen = subprocess.Popen([f'/usr/bin/python3', inspection_item_file, - _ip, _port], - stdout=subprocess.PIPE, shell=False) + single_inspection_popen = subprocess.Popen( + [f"/usr/bin/python3", inspection_item_file, _ip, _port], + stdout=subprocess.PIPE, + shell=False, + ) - single_inspection_result = single_inspection_popen.communicate(timeout=time_out)[0].decode('utf-8') + single_inspection_result = single_inspection_popen.communicate( + timeout=time_out + )[0].decode("utf-8") - return single_inspection_result \ No newline at end of file + return single_inspection_result diff --git a/pkg/deploy/action/inspection/inspection_mysql.py b/pkg/deploy/action/inspection/inspection_mysql.py index 51eb1b55..b4dcc508 100644 --- a/pkg/deploy/action/inspection/inspection_mysql.py +++ b/pkg/deploy/action/inspection/inspection_mysql.py @@ -9,7 +9,7 @@ from inspection_task import InspectionTask DIR_NAME, _ = os.path.split(os.path.abspath(__file__)) -INSPECTION_JSON_FILE = str(Path('{}/mysql_inspection_config.json'.format(DIR_NAME))) +INSPECTION_JSON_FILE = str(Path("{}/mysql_inspection_config.json".format(DIR_NAME))) DEPLOY_UID = 5000 LOCALHOST = "127.0.0.1" @@ -44,7 +44,7 @@ class MysqlInspection(InspectionTask): manage_ip = socket.gethostbyname(socket.gethostname()) except Exception as err: manage_ip = LOCALHOST - return 'mysql_' + str(manage_ip) + return "mysql_" + str(manage_ip) @staticmethod def check_executor(): @@ -52,21 +52,29 @@ class MysqlInspection(InspectionTask): raise ValueError(f"inspection must be executed by root") def task_execute_single(self, inspection_detail, name_pwd=None, ip_port=None): - inspection_item_file = inspection_detail.get('inspection_file_path') - inspection_item_input = inspection_detail.get('script_input') - component_belong = inspection_detail.get('component') - time_out = int(inspection_detail.get('time_out')) + inspection_item_file = inspection_detail.get("inspection_file_path") + inspection_item_input = inspection_detail.get("script_input") + component_belong = inspection_detail.get("component") + time_out = int(inspection_detail.get("time_out")) if component_belong not in self.user_map.keys(): - raise ValueError(f'Module {component_belong} not exist') + raise ValueError(f"Module {component_belong} not exist") if inspection_item_input: - single_inspection_popen = subprocess.Popen(['/usr/bin/python3', inspection_item_file, - inspection_item_input], stdout=subprocess.PIPE, shell=False) + single_inspection_popen = subprocess.Popen( + ["/usr/bin/python3", inspection_item_file, inspection_item_input], + stdout=subprocess.PIPE, + shell=False, + ) else: - single_inspection_popen = subprocess.Popen(['/usr/bin/python3', inspection_item_file], - stdout=subprocess.PIPE, shell=False) + single_inspection_popen = subprocess.Popen( + ["/usr/bin/python3", inspection_item_file], + stdout=subprocess.PIPE, + shell=False, + ) - single_inspection_result = single_inspection_popen.communicate(timeout=time_out)[0].decode('utf-8') + single_inspection_result = single_inspection_popen.communicate( + timeout=time_out + )[0].decode("utf-8") return single_inspection_result diff --git a/pkg/deploy/action/inspection/inspection_scripts/cms/cms_res_check.py b/pkg/deploy/action/inspection/inspection_scripts/cms/cms_res_check.py index 5376dba0..7c4a5106 100644 --- a/pkg/deploy/action/inspection/inspection_scripts/cms/cms_res_check.py +++ b/pkg/deploy/action/inspection/inspection_scripts/cms/cms_res_check.py @@ -5,33 +5,42 @@ import os import json import sys from pathlib import Path -sys.path.append('/opt/cantian/action/inspection') + +sys.path.append("/opt/cantian/action/inspection") from log_tool import setup from common_func import _exec_popen def parse_cmd_result(cmd_res): - keys = ['RESOURCE_NAME', 'RESOURCE_TYPE', 'RESOURCE_GROUP_NAME', 'START_TIMEOUT(ms)', 'STOP_TIMEOUT(ms)', - 'CHECK_TIMEOUT(ms)', 'CHECK_INTERVAL(ms)', 'HB_TIMEOUT(ms)'] + keys = [ + "RESOURCE_NAME", + "RESOURCE_TYPE", + "RESOURCE_GROUP_NAME", + "START_TIMEOUT(ms)", + "STOP_TIMEOUT(ms)", + "CHECK_TIMEOUT(ms)", + "CHECK_INTERVAL(ms)", + "HB_TIMEOUT(ms)", + ] values = cmd_res.split() stat_json = {} - for (idx, key) in enumerate(keys): + for idx, key in enumerate(keys): stat_json[key] = values[idx] res = True - if stat_json.get('HB_TIMEOUT(ms)') != '10000': - res = False - return (res) + if stat_json.get("HB_TIMEOUT(ms)") != "10000": + res = False + return res def fetch_cms_hbtime(logger): logger.info("cms res check start!") - cmd = 'source ~/.bashrc && cms res -list | tail -n +2' + cmd = "source ~/.bashrc && cms res -list | tail -n +2" ret_code, output, stderr = _exec_popen(cmd) - output = output.split('\n') + output = output.split("\n") if ret_code: logger.error("get cms res failed, std_err: %s", stderr) result_json = {} - result_json['data'] = {} + result_json["data"] = {} result_json["error"] = {} result_json["error"]["code"] = 0 result_json["error"]["description"] = "" @@ -39,28 +48,30 @@ def fetch_cms_hbtime(logger): result_json["error"]["code"] = -1 result_json["error"]["description"] = "cms res check error!" logger.error("cms res check error!") - return (result_json) + return result_json for cmd_res in output: (res) = parse_cmd_result(cmd_res) if res: - result_json['data']["RESULT"] = 'HB_TIMEOUT is 10 seconds' + result_json["data"]["RESULT"] = "HB_TIMEOUT is 10 seconds" else: result_json["error"]["code"] = -1 - result_json['error']["description"] = '[WAR]: HB_TIMEOUT greater than 10 seconds' - result_json['data']["RESULT"] = output + result_json["error"][ + "description" + ] = "[WAR]: HB_TIMEOUT greater than 10 seconds" + result_json["data"]["RESULT"] = output logger.info("cms res check succ!") - return (result_json) + return result_json def fetch_cls_stat(): # check if user is root - cantian_log = setup('cantian') - if(os.getuid() == 0): + cantian_log = setup("cantian") + if os.getuid() == 0: cantian_log.error("Cannot use root user for this operation!") sys.exit(1) (result_json) = fetch_cms_hbtime(cantian_log) return json.dumps(result_json, indent=1) -if __name__ == '__main__': +if __name__ == "__main__": print(fetch_cls_stat()) diff --git a/pkg/deploy/action/inspection/inspection_scripts/cms/cms_stat_check.py b/pkg/deploy/action/inspection/inspection_scripts/cms/cms_stat_check.py index a23151e7..45a7b52e 100644 --- a/pkg/deploy/action/inspection/inspection_scripts/cms/cms_stat_check.py +++ b/pkg/deploy/action/inspection/inspection_scripts/cms/cms_stat_check.py @@ -6,39 +6,49 @@ import json import sys from pathlib import Path -sys.path.append('/opt/cantian/action/inspection') +sys.path.append("/opt/cantian/action/inspection") from log_tool import setup from common_func import _exec_popen def parse_node_stat(node_stat): - keys = ['NODE_ID', 'NAME', 'STAT', 'PRE_STAT', 'TARGET_STAT', 'WORK_STAT', 'SESSION_ID', 'INSTANCE_ID', 'ROLE'] + keys = [ + "NODE_ID", + "NAME", + "STAT", + "PRE_STAT", + "TARGET_STAT", + "WORK_STAT", + "SESSION_ID", + "INSTANCE_ID", + "ROLE", + ] values = node_stat.split() stat_json = {} - node_info = {'NODE STAT': 'OFFLINE', 'NODE ROLE': 'NULL'} - for (idx, key) in enumerate(keys): + node_info = {"NODE STAT": "OFFLINE", "NODE ROLE": "NULL"} + for idx, key in enumerate(keys): stat_json[key] = values[idx] online = False reformer = False - if stat_json.get('STAT') == 'ONLINE': + if stat_json.get("STAT") == "ONLINE": online = True - node_info['NODE STAT'] = 'ONLINE' - if stat_json.get('ROLE') == 'REFORMER': + node_info["NODE STAT"] = "ONLINE" + if stat_json.get("ROLE") == "REFORMER": reformer = True - node_info['NODE ROLE'] = 'REFORMER' + node_info["NODE ROLE"] = "REFORMER" return online, reformer, node_info def fetch_cms_stat(logger): logger.info("cms stat check start!") - cmd = 'source ~/.bashrc && cms stat | tail -n +2' + cmd = "source ~/.bashrc && cms stat | tail -n +2" ret_code, output, stderr = _exec_popen(cmd) - output = output.split('\n') + output = output.split("\n") if ret_code: logger.error("get cms res information failed, std_err: %s", stderr) cluster_stat = {} result_json = {} - result_json['data'] = {} + result_json["data"] = {} result_json["error"] = {} result_json["error"]["code"] = 0 result_json["error"]["description"] = "" @@ -57,20 +67,20 @@ def fetch_cms_stat(logger): if reformer: refomer_stat = True if online_cnt == len(output) and refomer_stat is True: - result_json['data']['RESULT'] = 'CLUSTER STAT NORMAL' + result_json["data"]["RESULT"] = "CLUSTER STAT NORMAL" else: result_json["error"]["code"] = 1 result_json["error"]["description"] = detail_json return result_json detail_json.append(cluster_stat) - result_json['data']['DETAIL'] = detail_json + result_json["data"]["DETAIL"] = detail_json logger.info("cms stat check succ!") return result_json def fetch_cls_stat(): # check if user is root - cantian_log = setup('cantian') + cantian_log = setup("cantian") if os.getuid() == 0: cantian_log.error("Cannot use root user for this operation!") sys.exit(1) @@ -78,5 +88,5 @@ def fetch_cls_stat(): return json.dumps(result_json, indent=1) -if __name__ == '__main__': +if __name__ == "__main__": print(fetch_cls_stat()) diff --git a/pkg/deploy/action/inspection/inspection_scripts/cms/cms_version_check.py b/pkg/deploy/action/inspection/inspection_scripts/cms/cms_version_check.py index db44c08a..8e09abc7 100644 --- a/pkg/deploy/action/inspection/inspection_scripts/cms/cms_version_check.py +++ b/pkg/deploy/action/inspection/inspection_scripts/cms/cms_version_check.py @@ -5,14 +5,15 @@ import os import json import sys from pathlib import Path -sys.path.append('/opt/cantian/action/inspection') + +sys.path.append("/opt/cantian/action/inspection") from log_tool import setup from common_func import _exec_popen def fetch_cms_version(logger): logger.info("cms version check start!") - cmd = "source ~/.bashrc && cms -help |head -n 1 | grep -oP \"\K\d+\.\d+\"" + cmd = 'source ~/.bashrc && cms -help |head -n 1 | grep -oP "\K\d+\.\d+"' ret_code, output, stderr = _exec_popen(cmd) if ret_code: logger.error("get cms help information failed, std_err: %s", stderr) @@ -23,7 +24,7 @@ def fetch_cms_version(logger): raise Exception("Get cms version failed.") version = output.strip() result_json = {} - result_json['data'] = {} + result_json["data"] = {} result_json["error"] = {} result_json["error"]["code"] = 0 result_json["error"]["description"] = "" @@ -31,21 +32,23 @@ def fetch_cms_version(logger): if version != output_yml: logger.error("cms version is different from the version.yml") result_json["error"]["code"] = 1 - result_json["error"]["description"] = "get cms help information failed, std_err " \ - "or cms version is different from the version.yml" + result_json["error"]["description"] = ( + "get cms help information failed, std_err " + "or cms version is different from the version.yml" + ) logger.info("cms version check succ!") - return (result_json) + return result_json def fetch_cls_stat(): # check if user is root - cantian_log = setup('cantian') - if(os.getuid() == 0): + cantian_log = setup("cantian") + if os.getuid() == 0: cantian_log.error("Cannot use root user for this operation!") sys.exit(1) (result_json) = fetch_cms_version(cantian_log) return json.dumps(result_json, indent=1) -if __name__ == '__main__': +if __name__ == "__main__": print(fetch_cls_stat()) diff --git a/pkg/deploy/action/inspection/inspection_scripts/cms/common_func.py b/pkg/deploy/action/inspection/inspection_scripts/cms/common_func.py index 633f8522..f93d227d 100644 --- a/pkg/deploy/action/inspection/inspection_scripts/cms/common_func.py +++ b/pkg/deploy/action/inspection/inspection_scripts/cms/common_func.py @@ -15,8 +15,13 @@ def _exec_popen(cmd, values=None): if not values: values = [] bash_cmd = ["bash"] - pobj = subprocess.Popen(bash_cmd, shell=False, stdin=subprocess.PIPE, - stdout=subprocess.PIPE, stderr=subprocess.PIPE) + pobj = subprocess.Popen( + bash_cmd, + shell=False, + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + ) py_version = platform.python_version() if py_version[0] == "3": @@ -41,4 +46,4 @@ def _exec_popen(cmd, values=None): if stderr[-1:] == os.linesep: stderr = stderr[:-1] - return pobj.returncode, stdout, stderr \ No newline at end of file + return pobj.returncode, stdout, stderr diff --git a/pkg/deploy/action/inspection/inspection_scripts/ct_om/cantian_om_check.py b/pkg/deploy/action/inspection/inspection_scripts/ct_om/cantian_om_check.py index f6563749..d411070c 100644 --- a/pkg/deploy/action/inspection/inspection_scripts/ct_om/cantian_om_check.py +++ b/pkg/deploy/action/inspection/inspection_scripts/ct_om/cantian_om_check.py @@ -8,114 +8,140 @@ from pathlib import Path class OmChecker: def __init__(self): - self.decode_mod = 'utf-8' - self.component_check_order = ['cms', 'cantian', 'cantian_exporter'] - self.check_file_parent_path = '/opt/cantian/action' - self.check_file = 'check_status.sh' - self.check_daemon_cmd = 'pgrep -f cantian_daemon' - self.check_timer_cmd = 'systemctl is-active cantian.timer' + self.decode_mod = "utf-8" + self.component_check_order = ["cms", "cantian", "cantian_exporter"] + self.check_file_parent_path = "/opt/cantian/action" + self.check_file = "check_status.sh" + self.check_daemon_cmd = "pgrep -f cantian_daemon" + self.check_timer_cmd = "systemctl is-active cantian.timer" self.check_res_flag = True self.check_note = { - 'cms': 'unknown', - 'cantian': 'unknown', - 'ct_om': 'unknown', - 'cantian_exporter': 'unknown', - 'cantian_daemon': 'unknown', - 'cantian_timer': 'unknown' - } - self.format_output = { - 'data': {}, - 'error': { - 'code': 0, - 'description': '' - } + "cms": "unknown", + "cantian": "unknown", + "ct_om": "unknown", + "cantian_exporter": "unknown", + "cantian_daemon": "unknown", + "cantian_timer": "unknown", } + self.format_output = {"data": {}, "error": {"code": 0, "description": ""}} def check_ctom(self): - key_file = 'ctmgr/uds_server.py' - check_popen = subprocess.Popen(['/usr/bin/pgrep', '-f', key_file], - stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=False) + key_file = "ctmgr/uds_server.py" + check_popen = subprocess.Popen( + ["/usr/bin/pgrep", "-f", key_file], + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + shell=False, + ) ct_om_pid, _ = check_popen.communicate(timeout=60) if ct_om_pid.decode(self.decode_mod): - self.check_note['ct_om'] = 'online' + self.check_note["ct_om"] = "online" else: - self.check_note['ct_om'] = 'offline' + self.check_note["ct_om"] = "offline" self.check_res_flag = False def check_components(self): for component in self.component_check_order: - script_path = str(Path(os.path.join(self.check_file_parent_path, component, self.check_file))) - check_popen = subprocess.Popen(['/usr/bin/bash', script_path], - stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=False) + script_path = str( + Path( + os.path.join( + self.check_file_parent_path, component, self.check_file + ) + ) + ) + check_popen = subprocess.Popen( + ["/usr/bin/bash", script_path], + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + shell=False, + ) _, err = check_popen.communicate(timeout=60) if err.decode(self.decode_mod): continue check_result = check_popen.returncode if check_result: - self.check_note[component] = 'offline' + self.check_note[component] = "offline" self.check_res_flag = False else: - self.check_note[component] = 'online' + self.check_note[component] = "online" def check_daemon(self): - daemon = subprocess.Popen(shlex.split(self.check_daemon_cmd), stdout=subprocess.PIPE, stderr=subprocess.PIPE, - shell=False) + daemon = subprocess.Popen( + shlex.split(self.check_daemon_cmd), + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + shell=False, + ) output, err = daemon.communicate(timeout=60) if not err.decode(self.decode_mod): if output.decode(self.decode_mod): - self.check_note['cantian_daemon'] = 'online' + self.check_note["cantian_daemon"] = "online" else: - self.check_note['cantian_daemon'] = 'offline' + self.check_note["cantian_daemon"] = "offline" self.check_res_flag = False def check_cantian_timer(self): - daemon = subprocess.Popen(shlex.split(self.check_timer_cmd), stdout=subprocess.PIPE, - stderr=subprocess.PIPE, shell=False) + daemon = subprocess.Popen( + shlex.split(self.check_timer_cmd), + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + shell=False, + ) output, err = daemon.communicate(timeout=60) if not err.decode(self.decode_mod): - if output.decode(self.decode_mod).strip() == 'active': - self.check_note['cantian_timer'] = 'active' + if output.decode(self.decode_mod).strip() == "active": + self.check_note["cantian_timer"] = "active" - if output.decode(self.decode_mod).strip() == 'inactive': - self.check_note['cantian_timer'] = 'inactive' + if output.decode(self.decode_mod).strip() == "inactive": + self.check_note["cantian_timer"] = "inactive" self.check_res_flag = False def get_format_output(self): try: self.check_components() except Exception as err: - self.format_output['error']['code'] = 1 - self.format_output['error']['description'] = "check components failed with err: {}".format(str(err)) + self.format_output["error"]["code"] = 1 + self.format_output["error"]["description"] = ( + "check components failed with err: {}".format(str(err)) + ) return self.format_output try: self.check_ctom() except Exception as err: - self.format_output['error']['code'] = 1 - self.format_output['error']['description'] = "check ct_om status failed with err: {}".format(str(err)) + self.format_output["error"]["code"] = 1 + self.format_output["error"]["description"] = ( + "check ct_om status failed with err: {}".format(str(err)) + ) return self.format_output try: self.check_daemon() except Exception as err: - self.format_output['error']['code'] = 1 - self.format_output['error']['description'] = "check cantian_daemon failed with err: {}".format(str(err)) + self.format_output["error"]["code"] = 1 + self.format_output["error"]["description"] = ( + "check cantian_daemon failed with err: {}".format(str(err)) + ) return self.format_output try: self.check_cantian_timer() except Exception as err: - self.format_output['error']['code'] = 1 - self.format_output['error']['description'] = "check cantian timer fained with err: {}".format(str(err)) + self.format_output["error"]["code"] = 1 + self.format_output["error"]["description"] = ( + "check cantian timer fained with err: {}".format(str(err)) + ) return self.format_output if not self.check_res_flag: - self.format_output['error']['code'] = 1 - self.format_output['error']['description'] = "check cantian status failed, details: %s" % self.check_note - self.format_output['data'] = self.check_note + self.format_output["error"]["code"] = 1 + self.format_output["error"]["description"] = ( + "check cantian status failed, details: %s" % self.check_note + ) + self.format_output["data"] = self.check_note return self.format_output -if __name__ == '__main__': +if __name__ == "__main__": oc = OmChecker() print(json.dumps(oc.get_format_output())) diff --git a/pkg/deploy/action/inspection/inspection_scripts/ct_om/cantian_om_logicrep_check.py b/pkg/deploy/action/inspection/inspection_scripts/ct_om/cantian_om_logicrep_check.py index 067e684b..64781c5b 100644 --- a/pkg/deploy/action/inspection/inspection_scripts/ct_om/cantian_om_logicrep_check.py +++ b/pkg/deploy/action/inspection/inspection_scripts/ct_om/cantian_om_logicrep_check.py @@ -2,7 +2,7 @@ import json import os import sys -sys.path.append('/opt/cantian/action/inspection') +sys.path.append("/opt/cantian/action/inspection") from om_common_function import exec_popen @@ -13,21 +13,15 @@ FLAG_FILE = "/opt/software/tools/logicrep/enable.success" class LogicrepChecker: def __init__(self): - self.check_process_cmd = 'ps -ef | grep ZLogCatcherMain | grep -v grep' + self.check_process_cmd = "ps -ef | grep ZLogCatcherMain | grep -v grep" self.collect_cmd = f'grep -E "{CHECK_PATTERN}" {LOG_FILE_PATH}' self.flag = 0 self.check_note = { - 'Logicrep master node': 'unknown', - 'Logicrep process': 'unknown', - 'Warning log': "unknown", - } - self.format_output = { - 'data': dict(), - 'error': { - 'code': 0, - 'description': '' - } + "Logicrep master node": "unknown", + "Logicrep process": "unknown", + "Warning log": "unknown", } + self.format_output = {"data": dict(), "error": {"code": 0, "description": ""}} @staticmethod def select_error_info(info_list): @@ -57,18 +51,18 @@ class LogicrepChecker: def check_node(self): if os.path.exists(FLAG_FILE): - self.check_note['Logicrep master node'] = 'true' + self.check_note["Logicrep master node"] = "true" else: - self.check_note['Logicrep master node'] = 'false' + self.check_note["Logicrep master node"] = "false" self.flag += 1 def check_process(self): - self.check_note['Logicrep process'] = 'online' + self.check_note["Logicrep process"] = "online" try: _ = self._run_cmd(self.check_process_cmd) except Exception as err: if self.check_process_cmd in str(err): - self.check_note['Logicrep process'] = 'offline' + self.check_note["Logicrep process"] = "offline" self.flag += 1 else: raise @@ -81,53 +75,60 @@ class LogicrepChecker: if self.collect_cmd in str(err): pass if not stdout: - self.check_note['Warning log'] = 'none' + self.check_note["Warning log"] = "none" return stdout_list = self.select_error_info(stdout.split("\n")) if not stdout_list: - self.check_note['Warning log'] = 'none' + self.check_note["Warning log"] = "none" return - self.check_note['Warning log'] = "\n".join(stdout_list) + self.check_note["Warning log"] = "\n".join(stdout_list) raise Exception(str(self.check_note)) def get_format_output(self): try: self.check_node() except Exception as err: - self.format_output['error']['code'] = -1 - self.format_output['error']['description'] = "check logicrep master node failed with err: {}".format( - str(err)) - self.format_output['data'] = self.check_note + self.format_output["error"]["code"] = -1 + self.format_output["error"]["description"] = ( + "check logicrep master node failed with err: {}".format(str(err)) + ) + self.format_output["data"] = self.check_note return self.format_output try: self.check_process() except Exception as err: - self.format_output['error']['code'] = -1 - self.format_output['error']['description'] = "check logicrep process failed with err: {}".format(str(err)) - self.format_output['data'] = self.check_note + self.format_output["error"]["code"] = -1 + self.format_output["error"]["description"] = ( + "check logicrep process failed with err: {}".format(str(err)) + ) + self.format_output["data"] = self.check_note return self.format_output if self.flag == 2: - self.check_note['Warning log'] = 'none' - self.format_output['data'] = self.check_note + self.check_note["Warning log"] = "none" + self.format_output["data"] = self.check_note return self.format_output elif self.flag: - self.format_output['error']['code'] = -1 - self.format_output['error']['description'] = f"directory or process not found\n{str(self.check_note)}" - self.format_output['data'] = self.check_note + self.format_output["error"]["code"] = -1 + self.format_output["error"][ + "description" + ] = f"directory or process not found\n{str(self.check_note)}" + self.format_output["data"] = self.check_note return self.format_output try: self.collect_log() except Exception as err: - self.format_output['error']['code'] = -1 - self.format_output['error']['description'] = "collect warning info failed with err: {}".format(str(err)) - self.format_output['data'] = self.check_note + self.format_output["error"]["code"] = -1 + self.format_output["error"]["description"] = ( + "collect warning info failed with err: {}".format(str(err)) + ) + self.format_output["data"] = self.check_note return self.format_output - self.format_output['data'] = self.check_note + self.format_output["data"] = self.check_note return self.format_output -if __name__ == '__main__': +if __name__ == "__main__": far = LogicrepChecker() print(json.dumps(far.get_format_output())) diff --git a/pkg/deploy/action/inspection/inspection_scripts/ct_om/cantian_om_ntp_check.py b/pkg/deploy/action/inspection/inspection_scripts/ct_om/cantian_om_ntp_check.py index 83623982..58d7205a 100644 --- a/pkg/deploy/action/inspection/inspection_scripts/ct_om/cantian_om_ntp_check.py +++ b/pkg/deploy/action/inspection/inspection_scripts/ct_om/cantian_om_ntp_check.py @@ -3,7 +3,7 @@ import re import sys from datetime import datetime -sys.path.append('/opt/cantian/action/inspection') +sys.path.append("/opt/cantian/action/inspection") from log_tool import setup from om_common_function import exec_popen @@ -20,21 +20,15 @@ LOG_ZIP_FILE_PATH = "/opt/cantian/log/cantian/run/*tar.gz" class NtpChecker: def __init__(self): - self.check_ntp_cmd = 'timedatectl' + self.check_ntp_cmd = "timedatectl" self.check_cmd = f'grep -E "%s" {LOG_FILE_PATH}' self.check_zip_cmd = f'tar -Oxzf {LOG_ZIP_FILE_PATH} | grep -E "%s"' self.check_note = { - 'System clock synchronized': 'unknown', - 'NTP service': "unknown", - 'pitr warning': "unknown", - } - self.format_output = { - 'data': dict(), - 'error': { - 'code': 0, - 'description': '' - } + "System clock synchronized": "unknown", + "NTP service": "unknown", + "pitr warning": "unknown", } + self.format_output = {"data": dict(), "error": {"code": 0, "description": ""}} @staticmethod def get_ntp_result(output): @@ -59,9 +53,9 @@ class NtpChecker: stdout = self._run_cmd(self.check_ntp_cmd) flag = self.get_ntp_result(stdout) if flag: - self.check_note['System clock synchronized'] = 'yes' + self.check_note["System clock synchronized"] = "yes" else: - self.check_note['System clock synchronized'] = 'no' + self.check_note["System clock synchronized"] = "no" raise Exception("System clock synchronized is no") if "NTP service: active" in str(stdout): self.check_note["NTP service"] = "active" @@ -69,7 +63,6 @@ class NtpChecker: self.check_note["NTP service"] = "inactive" raise Exception("NTP service is inactive.") - def check_time_interval(self): check_list = [self.check_cmd, self.check_zip_cmd] check_result = [] @@ -87,7 +80,9 @@ class NtpChecker: timestamp = self._get_log_time(item) check_result.append((timestamp, item)) if check_result: - self.check_note["pitr warning"] = "\n".join([item[1] for item in sorted(check_result)][-20:]) + self.check_note["pitr warning"] = "\n".join( + [item[1] for item in sorted(check_result)][-20:] + ) raise Exception(str(self.check_note)) else: self.check_note["pitr warning"] = "success" @@ -96,22 +91,25 @@ class NtpChecker: try: self.check_ntp() except Exception as err: - self.format_output['error']['code'] = -1 - self.format_output['error']['description'] = "check ntp server failed with err: {}".format(str(err)) - self.format_output['data'] = self.check_note + self.format_output["error"]["code"] = -1 + self.format_output["error"]["description"] = ( + "check ntp server failed with err: {}".format(str(err)) + ) + self.format_output["data"] = self.check_note return self.format_output try: self.check_time_interval() except Exception as err: - self.format_output['error']['code'] = -1 - self.format_output['error']['description'] = "check time interval failed with err: {}".format(str(err)) - self.format_output['data'] = self.check_note + self.format_output["error"]["code"] = -1 + self.format_output["error"]["description"] = ( + "check time interval failed with err: {}".format(str(err)) + ) + self.format_output["data"] = self.check_note return self.format_output - self.format_output['data'] = self.check_note + self.format_output["data"] = self.check_note return self.format_output -if __name__ == '__main__': +if __name__ == "__main__": far = NtpChecker() print(json.dumps(far.get_format_output())) - diff --git a/pkg/deploy/action/inspection/inspection_scripts/ct_om/om_common_function.py b/pkg/deploy/action/inspection/inspection_scripts/ct_om/om_common_function.py index 947f76c1..5e4fde4c 100644 --- a/pkg/deploy/action/inspection/inspection_scripts/ct_om/om_common_function.py +++ b/pkg/deploy/action/inspection/inspection_scripts/ct_om/om_common_function.py @@ -12,8 +12,13 @@ def exec_popen(cmd, values=None): if not values: values = [] bash_cmd = ["bash"] - pobj = subprocess.Popen(bash_cmd, shell=False, stdin=subprocess.PIPE, - stdout=subprocess.PIPE, stderr=subprocess.PIPE) + pobj = subprocess.Popen( + bash_cmd, + shell=False, + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + ) py_version = platform.python_version() if py_version[0] == "3": @@ -39,4 +44,3 @@ def exec_popen(cmd, values=None): stderr = stderr[:-1] return pobj.returncode, stdout, stderr - diff --git a/pkg/deploy/action/inspection/inspection_scripts/kernal/check_archive_status.py b/pkg/deploy/action/inspection/inspection_scripts/kernal/check_archive_status.py index e28a296f..a4a9e679 100644 --- a/pkg/deploy/action/inspection/inspection_scripts/kernal/check_archive_status.py +++ b/pkg/deploy/action/inspection/inspection_scripts/kernal/check_archive_status.py @@ -6,23 +6,23 @@ import re from pathlib import Path from ct_check import CheckContext, BaseItem, ResultStatus -sys.path.append('/opt/cantian/action/inspection') +sys.path.append("/opt/cantian/action/inspection") from log_tool import setup -DEPLOY_CONFIG_FILE = str(Path('/opt/cantian/config/deploy_param.json')) +DEPLOY_CONFIG_FILE = str(Path("/opt/cantian/config/deploy_param.json")) UNIT_CONVERSION_MAP = { "P": 1024 * 1024 * 1024 * 1024, "T": 1024 * 1024 * 1024, "G": 1024 * 1024, "M": 1024, - "K": 1 + "K": 1, } class CheckArchiveStatus(BaseItem): - ''' + """ check version of database - ''' + """ def __init__(self): super(CheckArchiveStatus, self).__init__(self.__class__.__name__) @@ -38,7 +38,9 @@ class CheckArchiveStatus(BaseItem): values = {} if self.db_type == 0 or self.db_type == "0": self.result.rst = ResultStatus.WARNING - self.result.sug = "The backup function is invalid. You are advised to enable it" + self.result.sug = ( + "The backup function is invalid. You are advised to enable it" + ) values["result"] = "db_type is %s" % self.db_type self.result.val = json.dumps(values) return False @@ -52,9 +54,11 @@ class CheckArchiveStatus(BaseItem): return if self.deploy_mode == "dbstor": self.result.rst = ResultStatus.NG - values["result"] = "Deploy mode is %s, please check whether the "\ - "remaining capacity of the file system meets "\ - "the requirements by self." % self.deploy_mode + values["result"] = ( + "Deploy mode is %s, please check whether the " + "remaining capacity of the file system meets " + "the requirements by self." % self.deploy_mode + ) self.result.val = json.dumps(values) return @@ -62,8 +66,10 @@ class CheckArchiveStatus(BaseItem): ret_code, str_out = self.get_cmd_result(cmd) if ret_code: self.result.rst = ResultStatus.ERROR - self.result.sug = "1) Check whether the network link is normal\n " \ - "2) If the link is normal, contact technical support engineers" + self.result.sug = ( + "1) Check whether the network link is normal\n " + "2) If the link is normal, contact technical support engineers" + ) values["except"] = "can not connect to %s" % self.archive_ip self.result.val = json.dumps(values) return @@ -90,37 +96,49 @@ class CheckArchiveStatus(BaseItem): used_capacity_number_str = re.sub("[A-z]", "", used_capacity) max_capacity_number_str = re.sub("[A-z]", "", max_capacity) - used_capacity_num = float(used_capacity_number_str) * UNIT_CONVERSION_MAP.get(used_capacity_unit_str, 0) - max_capacity_num = float(max_capacity_number_str) * UNIT_CONVERSION_MAP.get(max_capacity_unit_str, 0) + used_capacity_num = float(used_capacity_number_str) * UNIT_CONVERSION_MAP.get( + used_capacity_unit_str, 0 + ) + max_capacity_num = float(max_capacity_number_str) * UNIT_CONVERSION_MAP.get( + max_capacity_unit_str, 0 + ) if self.max_archive_size > max_capacity_num * 0.45: self.result.rst = ResultStatus.ERROR - values["except"] = "The archive configuration capacity must be less than or " \ - "equal to 45% of the maximum archive file system capacity" + values["except"] = ( + "The archive configuration capacity must be less than or " + "equal to 45% of the maximum archive file system capacity" + ) self.result.sug = "Please modify the archive file" self.result.val = json.dumps(values) return if used_capacity_num > self.max_archive_size * 0.95: self.result.rst = ResultStatus.WARNING - values["result"] = "The used archive file capacity exceeds the threshold, causing archive failure" + values["result"] = ( + "The used archive file capacity exceeds the threshold, causing archive failure" + ) self.result.sug = "Contact technical support engineers" self.result.val = json.dumps(values) -if __name__ == '__main__': - ''' +if __name__ == "__main__": + """ main - ''' + """ # check if user is root - cantian_log = setup('cantian') + cantian_log = setup("cantian") if os.getuid() == 0: cantian_log.error("Cannot use root user for this operation!") sys.exit(1) archive_object = CheckArchiveStatus() - with os.fdopen(os.open(DEPLOY_CONFIG_FILE, os.O_RDONLY | os.O_EXCL, stat.S_IWUSR | stat.S_IRUSR), "r") \ - as file_handle: + with os.fdopen( + os.open( + DEPLOY_CONFIG_FILE, os.O_RDONLY | os.O_EXCL, stat.S_IWUSR | stat.S_IRUSR + ), + "r", + ) as file_handle: json_data = json.load(file_handle) archive_object.max_archive_size_str = json_data.get("MAX_ARCH_FILES_SIZE", "") archive_object.db_type = json_data.get("db_type", 0) @@ -130,7 +148,9 @@ if __name__ == '__main__': unit_str = re.compile("[A-z]").findall(archive_object.max_archive_size_str)[0] number_str = re.sub("[A-z]", "", archive_object.max_archive_size_str) if UNIT_CONVERSION_MAP.get(unit_str, 0) != 0: - archive_object.max_archive_size = float(number_str) * UNIT_CONVERSION_MAP.get(unit_str) + archive_object.max_archive_size = float( + number_str + ) * UNIT_CONVERSION_MAP.get(unit_str) checker_context = CheckContext() archive_object.run_check(checker_context, cantian_log) diff --git a/pkg/deploy/action/inspection/inspection_scripts/kernal/check_db_version.py b/pkg/deploy/action/inspection/inspection_scripts/kernal/check_db_version.py index ee0ac82f..198b49b2 100644 --- a/pkg/deploy/action/inspection/inspection_scripts/kernal/check_db_version.py +++ b/pkg/deploy/action/inspection/inspection_scripts/kernal/check_db_version.py @@ -6,33 +6,37 @@ import sys from ct_check import CheckContext from ct_check import BaseItem from ct_check import ResultStatus -sys.path.append('/opt/cantian/action/inspection') + +sys.path.append("/opt/cantian/action/inspection") from log_tool import setup class CheckDBVersion(BaseItem): - ''' + """ check version of database - ''' + """ + def __init__(self): super(CheckDBVersion, self).__init__(self.__class__.__name__) self.title = "Check the database version" def do_check(self): - ''' + """ function : Check version of database input : NA output : NA - ''' + """ vals = {} self.result.rst = ResultStatus.OK - cmd = "source ~/.bashrc && %s/cantiand -v | grep -oP \"\K\d+\.\d+\"" % \ - os.path.join(self.context.app_path, "bin") + cmd = ( + 'source ~/.bashrc && %s/cantiand -v | grep -oP "\K\d+\.\d+"' + % os.path.join(self.context.app_path, "bin") + ) self.result.raw = cmd status, output = self.get_cmd_result(cmd, self.user) - if (status == 0): + if status == 0: vals["db_version"] = output cmd = "cat /opt/cantian/versions.yml | grep -oP 'Version: \K\d+\.\d+'" status, output = self.get_cmd_result(cmd, self.user) @@ -49,12 +53,12 @@ class CheckDBVersion(BaseItem): self.result.val = json.dumps(vals) -if __name__ == '__main__': - ''' +if __name__ == "__main__": + """ main - ''' + """ # check if user is root - cantian_log = setup('cantian') + cantian_log = setup("cantian") if os.getuid() == 0: cantian_log.error("Cannot use root user for this operation!") sys.exit(1) diff --git a/pkg/deploy/action/inspection/inspection_scripts/kernal/check_drc_res_ratio.py b/pkg/deploy/action/inspection/inspection_scripts/kernal/check_drc_res_ratio.py index a3303b81..32f85de6 100644 --- a/pkg/deploy/action/inspection/inspection_scripts/kernal/check_drc_res_ratio.py +++ b/pkg/deploy/action/inspection/inspection_scripts/kernal/check_drc_res_ratio.py @@ -6,13 +6,16 @@ import sys from ct_check import CheckContext from ct_check import BaseItem from ct_check import ResultStatus -sys.path.append('/opt/cantian/action/inspection') + +sys.path.append("/opt/cantian/action/inspection") from log_tool import setup + class CheckDRCResRatio(BaseItem): - ''' + """ check DRC res ratio - ''' + """ + def __init__(self): super(CheckDRCResRatio, self).__init__(self.__class__.__name__) self.suggestion = "If DRC resource ratio is too high, try checkpoint" @@ -21,11 +24,11 @@ class CheckDRCResRatio(BaseItem): self.epv = 0.9375 def do_check(self): - ''' + """ function : Check for DRC res ratio input : NA output : NA - ''' + """ self.result.epv = self.epv self.result.rst = ResultStatus.OK @@ -36,16 +39,16 @@ class CheckDRCResRatio(BaseItem): # Execute sql command status, records = self.get_sql_result(sql) - if (status == 0): - res_ratio_dict['PAGE_BUF'] = records["records"][0][0] - res_ratio_dict['GLOBAL_LOCK'] = records["records"][1][0] - res_ratio_dict['LOCAL_LOCK'] = records["records"][2][0] - res_ratio_dict['LOCAL_TXN'] = records["records"][3][0] - res_ratio_dict['GLOBAL_TXN'] = records["records"][4][0] - res_ratio_dict['LOCK_ITEM'] = records["records"][5][0] + if status == 0: + res_ratio_dict["PAGE_BUF"] = records["records"][0][0] + res_ratio_dict["GLOBAL_LOCK"] = records["records"][1][0] + res_ratio_dict["LOCAL_LOCK"] = records["records"][2][0] + res_ratio_dict["LOCAL_TXN"] = records["records"][3][0] + res_ratio_dict["GLOBAL_TXN"] = records["records"][4][0] + res_ratio_dict["LOCK_ITEM"] = records["records"][5][0] self.result.rst = ResultStatus.OK for value in res_ratio_dict.values(): - if (float(value) >= self.result.epv) : + if float(value) >= self.result.epv: self.result.rst = ResultStatus.NG else: self.result.rst = ResultStatus.ERROR @@ -55,13 +58,13 @@ class CheckDRCResRatio(BaseItem): self.result.val = json.dumps(res_ratio_dict) -if __name__ == '__main__': - ''' +if __name__ == "__main__": + """ main - ''' + """ # check if user is root - cantian_log = setup('cantian') - if(os.getuid() == 0): + cantian_log = setup("cantian") + if os.getuid() == 0: cantian_log.error("Cannot use root user for this operation!") sys.exit(1) diff --git a/pkg/deploy/action/inspection/inspection_scripts/kernal/check_redundant_links.py b/pkg/deploy/action/inspection/inspection_scripts/kernal/check_redundant_links.py index 2759c831..633dda24 100644 --- a/pkg/deploy/action/inspection/inspection_scripts/kernal/check_redundant_links.py +++ b/pkg/deploy/action/inspection/inspection_scripts/kernal/check_redundant_links.py @@ -6,52 +6,56 @@ import sys from ct_check import CheckContext from ct_check import BaseItem from ct_check import ResultStatus -sys.path.append('/opt/cantian/action/inspection') + +sys.path.append("/opt/cantian/action/inspection") from log_tool import setup - - + + class CheckRedundantLinks(BaseItem): - ''' + """ check version of database - ''' + """ + def __init__(self): super(CheckRedundantLinks, self).__init__(self.__class__.__name__) self.title = "Check redundant links" - + def do_check(self): - ''' + """ function : Check redundant links input : NA output : NA - ''' - + """ + vals = {} self.result.rst = ResultStatus.OK - + cmd = "sh /opt/cantian/action/inspection/inspection_scripts/kernal/check_link_cnt.sh" self.result.raw = cmd status, output = self.get_cmd_result(cmd, self.user) - if (status == 0): + if status == 0: vals["success"] = "Have redundant link." else: self.result.rst = ResultStatus.ERROR - vals["except"] = "Do not have redundant link, for details, see the /opt/cantian/dbstor/cgwshowdev.log ." - + vals["except"] = ( + "Do not have redundant link, for details, see the /opt/cantian/dbstor/cgwshowdev.log ." + ) + # add resault to json self.result.val = json.dumps(vals) - - -if __name__ == '__main__': - ''' + + +if __name__ == "__main__": + """ main - ''' + """ # check if user is root - cantian_log = setup('cantian') - if(os.getuid() == 0): + cantian_log = setup("cantian") + if os.getuid() == 0: cantian_log.error("Cannot use root user for this operation!") sys.exit(1) - + # main function checker = CheckRedundantLinks() checker_context = CheckContext() - checker.run_check(checker_context, cantian_log) \ No newline at end of file + checker.run_check(checker_context, cantian_log) diff --git a/pkg/deploy/action/inspection/inspection_scripts/kernal/check_session.py b/pkg/deploy/action/inspection/inspection_scripts/kernal/check_session.py index ff1e3445..680810bb 100644 --- a/pkg/deploy/action/inspection/inspection_scripts/kernal/check_session.py +++ b/pkg/deploy/action/inspection/inspection_scripts/kernal/check_session.py @@ -6,65 +6,65 @@ import sys from ct_check import CheckContext from ct_check import BaseItem from ct_check import ResultStatus -sys.path.append('/opt/cantian/action/inspection') + +sys.path.append("/opt/cantian/action/inspection") from log_tool import setup class CheckSession(BaseItem): - ''' + """ check resource of host - ''' + """ + def __init__(self): super(CheckSession, self).__init__(self.__class__.__name__) - self.suggestion = \ - "The SQL client connection is less than or equal to 80% of the configured value." + self.suggestion = "The SQL client connection is less than or equal to 80% of the configured value." self.standard = "The SQL client connection is less than or equal to 80% of the configured value." self.title = "Check the number of database connections" self.epv = "[0, 80]" def execute_sql(self, sql, vals): - ''' + """ function : excute sql input : NA output : NA - ''' - if not (self.context.db_user and self.context.db_passwd and - self.context.port): + """ + if not (self.context.db_user and self.context.db_passwd and self.context.port): vals["except"] = "Database connection failed" self.result.rst = ResultStatus.ERROR self.result.val = json.dumps(vals) return -1, 0 status, records = self.get_sql_result(sql) - if status : + if status: self.result.rst = ResultStatus.ERROR vals["except"] = records return status, records def get_db_session(self, vals): - ''' + """ function : get session count of database input : NA output : int - ''' + """ if self.copyright: sql = "SELECT COUNT(1) FROM DV_SESSIONS;" else: sql = "SELECT COUNT(1) FROM V\$SESSION;" self.result.raw += "SESSION: " + sql.replace("\$", "$") + "\n" status, records = self.execute_sql(sql, vals) - if (status == 0): + if status == 0: session_count = int(records["records"][0][0]) else: session_count = 0 return status, session_count def get_conf_session(self, vals): - ''' + """ function : get session value of confFile input : dict output : int - ''' + """ session_value = 1500 status = 0 conf_file = "%s/cfg/cantiand.ini" % self.data_path @@ -74,7 +74,7 @@ class CheckSession(BaseItem): return status, 0 content = "" - with open(conf_file, 'r') as fp: + with open(conf_file, "r") as fp: content = fp.readlines() for line in content: @@ -88,11 +88,11 @@ class CheckSession(BaseItem): return status, session_value - 5 def check_session(self, vals): - ''' + """ function : check session input : dict output : NA - ''' + """ status, db_session = self.get_db_session(vals) if status: return @@ -103,16 +103,16 @@ class CheckSession(BaseItem): if usage > 80.0: self.result.rst = ResultStatus.NG - vals['db_session'] = db_session - vals['conf_session'] = conf_session - vals['usage'] = "%.2f%%" % usage + vals["db_session"] = db_session + vals["conf_session"] = conf_session + vals["usage"] = "%.2f%%" % usage def do_check(self): - ''' + """ function : Check for status input : NA output : NA - ''' + """ vals = {} self.result.rst = ResultStatus.OK @@ -122,13 +122,13 @@ class CheckSession(BaseItem): self.result.val = json.dumps(vals) -if __name__ == '__main__': - ''' +if __name__ == "__main__": + """ main - ''' + """ # check if user is root - cantian_log = setup('cantian') - if(os.getuid() == 0): + cantian_log = setup("cantian") + if os.getuid() == 0: cantian_log.error("Cannot use root user for this operation!") sys.exit(1) @@ -144,5 +144,5 @@ if __name__ == '__main__': for argv in sys.argv[1:]: setattr(checker_context, context_attr[item_index], argv) item_index += 1 - + checker.run_check(checker_context, cantian_log) diff --git a/pkg/deploy/action/inspection/inspection_scripts/kernal/check_transaction.py b/pkg/deploy/action/inspection/inspection_scripts/kernal/check_transaction.py index 177183e5..c506b77e 100644 --- a/pkg/deploy/action/inspection/inspection_scripts/kernal/check_transaction.py +++ b/pkg/deploy/action/inspection/inspection_scripts/kernal/check_transaction.py @@ -6,7 +6,8 @@ import sys from ct_check import CheckContext from ct_check import BaseItem from ct_check import ResultStatus -sys.path.append('/opt/cantian/action/inspection') + +sys.path.append("/opt/cantian/action/inspection") from log_tool import setup # minutes converted to microseconds @@ -14,24 +15,26 @@ MAX_MICRO_TIME = 3 * 60 * 1000000 class CheckTransaction(BaseItem): - ''' + """ check transaction of database - ''' + """ + def __init__(self): super(CheckTransaction, self).__init__(self.__class__.__name__) - self.suggestion = \ - "Configurable time, if there is a long transaction, \ + self.suggestion = "Configurable time, if there is a long transaction, \ it is recommended that the user modify the SQL statement." - self.standard = "Check transactions greater than 3 minutes, check if they do not exist." + self.standard = ( + "Check transactions greater than 3 minutes, check if they do not exist." + ) self.title = "check for long time transactions" self.epv = 0 def do_check(self): - ''' + """ function : Check for transaction of long time input : NA output : NA - ''' + """ vals = {} self.result.epv = self.epv @@ -48,7 +51,7 @@ it is recommended that the user modify the SQL statement." # Execute sql command status, records = self.get_sql_result(sql) - if (status == 0): + if status == 0: count = records["records"][0][0] # expect value : 0 if int(count) == self.result.epv: @@ -64,14 +67,13 @@ it is recommended that the user modify the SQL statement." self.result.val = json.dumps(vals) - -if __name__ == '__main__': - ''' +if __name__ == "__main__": + """ main - ''' + """ # check if user is root - cantian_log = setup('cantian') - if(os.getuid() == 0): + cantian_log = setup("cantian") + if os.getuid() == 0: cantian_log.error("Cannot use root user for this operation!") sys.exit(1) diff --git a/pkg/deploy/action/inspection/inspection_scripts/kernal/ct_check.py b/pkg/deploy/action/inspection/inspection_scripts/kernal/ct_check.py index 140bac6a..143c2588 100644 --- a/pkg/deploy/action/inspection/inspection_scripts/kernal/ct_check.py +++ b/pkg/deploy/action/inspection/inspection_scripts/kernal/ct_check.py @@ -33,7 +33,7 @@ gPyVersion = platform.python_version() if gPyVersion[0] == "3": import importlib -if CURRENT_OS != 'Linux': +if CURRENT_OS != "Linux": raise ValueError("Error:Check os failed:current os is not linux") @@ -48,7 +48,7 @@ class CheckContext: """ # Initialize the self.clusterInfo variable curr_path = os.path.realpath(__file__) - self.base_path = os.path.join(os.path.split(curr_path)[0], 'inspection') + self.base_path = os.path.join(os.path.split(curr_path)[0], "inspection") self.user = None self.support_items = {} self.support_scenes = {} @@ -82,9 +82,9 @@ class CheckContext: # Exception class # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # class CheckException(Exception): - ''' + """ base class of exception - ''' + """ def __init__(self, content): super(CheckException, self).__init__(self) @@ -96,9 +96,9 @@ class CheckException(Exception): class CheckNAException(CheckException): - ''' + """ NA checkItem exception - ''' + """ def __init__(self, item): super(CheckNAException, self).__init__(self.__class__.__name__) @@ -108,9 +108,9 @@ class CheckNAException(CheckException): class TimeoutException(CheckException): - ''' + """ timeout exception - ''' + """ def __init__(self, second): super(TimeoutException, self).__init__(self.__class__.__name__) @@ -120,20 +120,41 @@ class TimeoutException(CheckException): def get_current_user(): - ''' + """ get current user - ''' + """ # Get the current user return pwd.getpwuid(os.getuid())[0] def check_legality(parameter_string): - ''' + """ Check for illegal characters - ''' + """ # the list of invalid characters value_check_list = [ - "|", ";", "&", "$", "<", ">", "`", "\\", "'", "\"", "{", "}", "(", ")", "[", "]", "~", "*", "?", "!", "\n", " " + "|", + ";", + "&", + "$", + "<", + ">", + "`", + "\\", + "'", + '"', + "{", + "}", + "(", + ")", + "[", + "]", + "~", + "*", + "?", + "!", + "\n", + " ", ] # judge illegal characters for ch in value_check_list: @@ -143,9 +164,9 @@ def check_legality(parameter_string): class SharedFuncs: - ''' + """ defined tools for executing cmd and sql - ''' + """ def __init__(self): pass @@ -163,37 +184,40 @@ class SharedFuncs: ssh_base = "%s -o NumberOfPasswordPrompts=1 %s@%s" % (ssh, user, ip) process = Execution(ssh_base) - idx = process.expect(['(P|p)assword:']) + idx = process.expect(["(P|p)assword:"]) if idx == 0: process.sendLine(passwd) - done_flag = 'ct_check done' + done_flag = "ct_check done" process.sendLine("%s; echo 'ct_check done'" % ssh_sql) - escape = 'unicode-escape' + escape = "unicode-escape" while True: - idx = process.expect([done_flag, "Please enter password"], - timeout=50) + idx = process.expect([done_flag, "Please enter password"], timeout=50) if idx == 0: - process.sendLine('exit') + process.sendLine("exit") status = 0 - output = str(process.context_before.decode(escape).split(done_flag)[0]) + output = str( + process.context_before.decode(escape).split(done_flag)[0] + ) break elif idx == 1: process.sendLine(db_passwd) else: - process.sendLine('exit') + process.sendLine("exit") status = 1 - output = str(process.context_buffer.decode(escape).split(done_flag)[0]) + output = str( + process.context_buffer.decode(escape).split(done_flag)[0] + ) break except Exception as err: if process: - process.sendLine('exit') + process.sendLine("exit") status = 1 output = f"{err}\n{process.context_buffer.decode('unicode-escape').split(done_flag)[0]}" return status, output @staticmethod def get_abs_path(self, _file): - for _path in os.environ["PATH"].split(':'): + for _path in os.environ["PATH"].split(":"): abs_file = os.path.normpath(os.path.join(_path, _file)) if os.path.exists(abs_file): return 0, abs_file @@ -209,8 +233,13 @@ class SharedFuncs: if not values: values = [] bash_cmd = ["bash"] - pobj = subprocess.Popen(bash_cmd, shell=False, stdin=subprocess.PIPE, - stdout=subprocess.PIPE, stderr=subprocess.PIPE) + pobj = subprocess.Popen( + bash_cmd, + shell=False, + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + ) py_version = platform.python_version() if py_version[0] == "3": @@ -253,9 +282,9 @@ class SharedFuncs: return 0, "\n".join(result) def run_shell_cmd(self, cmd, user=None, mpprc_file="", array=(0,)): - ''' + """ defined tools for cmd - ''' + """ if not isinstance(array, tuple): return -1, "parameter [array] is illegal" @@ -267,12 +296,12 @@ class SharedFuncs: if max(array) > len(cmd.split("|")) - 1: return -1, "parameter [array] is illegal" - if (mpprc_file): + if mpprc_file: cmd = "source '%s'; %s" % (mpprc_file, cmd) cmd = cmd + "; echo ${PIPESTATUS[*]}" # change user but can not be root user - if (user and user != get_current_user()): + if user and user != get_current_user(): cmd = "su -s /bin/bash - %s -c 'source ~/.bashrc; %s'" % (user, cmd) returncode, stdout, stderr = self.exec_open(cmd) @@ -299,21 +328,24 @@ class SharedFuncs: return 1, "CantianDB 100 V300R001" ctsql_path = os.path.join(app_path, "bin") - sql_cmd = "source ~/.bashrc && echo -e %s| %s/ctsql %s@%s:%s -q -c \"%s\"" % (db_passwd, ctsql_path, - db_user, - db_addr, - port, - sql) + sql_cmd = 'source ~/.bashrc && echo -e %s| %s/ctsql %s@%s:%s -q -c "%s"' % ( + db_passwd, + ctsql_path, + db_user, + db_addr, + port, + sql, + ) returncode, stdout, stderr = self.exec_open(sql_cmd) output = stdout + stderr return returncode, output def get_sql_result(self, *args): - ''' + """ get records of sql output success:state, {'records':[[list1], [list2]], "title":[list_title]} failed: state, string_errormsg - ''' + """ sql, db_user, db_passwd, db_addr, port, app_path = args params = [sql, db_user, db_passwd, db_addr, port, app_path] status, output = self.run_sql_cmd(*params) @@ -326,18 +358,18 @@ class SharedFuncs: num_records = int(re.findall(r"(\d+) rows fetched.", output)[0]) end_index = result_list.index(f"{num_records} rows fetched.") result_records["title"] = result_list[end_index - 2 - num_records].split() - for result_line in result_list[end_index - num_records: end_index]: + for result_line in result_list[end_index - num_records : end_index]: result_records["records"].append(result_line.split()) return status, result_records else: return status, "\n".join([line for line in output.split("\n") if line]) def verify_conn(self, *args_): - ''' + """ function : get ip type input :string, string, string, string, string, string output : iptype - ''' + """ sql, dbuser, dbpwd, db_addr, port, app_path = args_ params = [sql, dbuser, dbpwd, db_addr, port, app_path] if not (dbuser and dbpwd): @@ -364,9 +396,10 @@ def get_validity(path, path_desc): class ResultStatus(object): - ''' + """ define result status - ''' + """ + OK = "OK" NA = "NA" WARNING = "WARNING" @@ -375,9 +408,9 @@ class ResultStatus(object): class LocalItemResult(object): - ''' + """ the check result running on one host - ''' + """ def __init__(self, name, host): self.name = name @@ -409,29 +442,32 @@ class LocalItemResult(object): " {sug}\n" "[REFER ]\n" " {raw}\n" - ) val = self.val if self.val else "" raw = self.raw if self.raw else "" try: - content = output_doc.format(name=self.name, - rst=self.rst, - host=self.host, - val=val, - epv=self.epv, - des=self.des, - sug=self.sug, - raw=raw) + content = output_doc.format( + name=self.name, + rst=self.rst, + host=self.host, + val=val, + epv=self.epv, + des=self.des, + sug=self.sug, + raw=raw, + ) except Exception: - output_utf8 = output_doc.encode('utf-8') - content = output_utf8.format(name=self.name, - rst=self.rst, - host=self.host, - val=val, - epv=self.epv, - des=self.des, - sug=self.sug, - raw=raw.decode('utf-8')) + output_utf8 = output_doc.encode("utf-8") + content = output_utf8.format( + name=self.name, + rst=self.rst, + host=self.host, + val=val, + epv=self.epv, + des=self.des, + sug=self.sug, + raw=raw.decode("utf-8"), + ) return content def to_json(self): @@ -443,45 +479,49 @@ class LocalItemResult(object): "EXPECT": self.epv, "SUGGEST": self.sug, "DESCRIPT_EN": self.des, - "REFER": [raw.strip() for raw in self.raw.split("\n") if raw] + "REFER": [raw.strip() for raw in self.raw.split("\n") if raw], }, - "error": { - "code": 0, - "description": "" - } + "error": {"code": 0, "description": ""}, } - if (self.rst == ResultStatus.NA): + if self.rst == ResultStatus.NA: rst = "\033[0;37m%s\033[0m" % "NONE" - elif (self.rst == ResultStatus.WARNING or - self.rst == ResultStatus.ERROR or - self.rst == ResultStatus.NG): + elif ( + self.rst == ResultStatus.WARNING + or self.rst == ResultStatus.ERROR + or self.rst == ResultStatus.NG + ): rst = "\033[0;31m%s\033[0m" % self.rst else: rst = "\033[0;32m%s\033[0m" % ResultStatus.OK val = json.loads(self.val) - if (self.rst == ResultStatus.NG): + if self.rst == ResultStatus.NG: detail_result["error"]["code"] = 1 - detail_result["error"]["description"] = \ - "{} is failed, expect val: {} current val: {}".format(self.des, self.epv, val) - elif (self.rst == ResultStatus.ERROR): + detail_result["error"]["description"] = ( + "{} is failed, expect val: {} current val: {}".format( + self.des, self.epv, val + ) + ) + elif self.rst == ResultStatus.ERROR: detail_result["error"]["code"] = -1 - detail_result["error"]["description"] = "{} is failed, error msg: \"{}\"".format(self.des, val["except"]) + detail_result["error"]["description"] = ( + '{} is failed, error msg: "{}"'.format(self.des, val["except"]) + ) json_dump = json.dumps(detail_result, ensure_ascii=False, indent=2) print(json_dump) class BaseItem(object): - ''' + """ base class of check item - ''' + """ def __init__(self, name): - ''' + """ Constructor - ''' + """ self.name = name self.title = None self.suggestion = None @@ -489,10 +529,10 @@ class BaseItem(object): self.time = int(time.time()) self.standard = None self.threshold = {} - self.category = 'other' - self.permission = 'user' - self.analysis = 'default' - self.scope = 'all' + self.category = "other" + self.permission = "user" + self.analysis = "default" + self.scope = "all" self.cluster = None self.user = None self.nodes = None @@ -514,15 +554,15 @@ class BaseItem(object): @abstractmethod def do_check(self): - ''' + """ check script for each item - ''' + """ pass def init_form(self, context): - ''' + """ initialize the check item from context - ''' + """ self.context = context self.user = context.user self.nodes = context.nodes @@ -543,38 +583,40 @@ class BaseItem(object): self.result.epv = self.epv self.result.des = self.title # new host without cluster installed - if (not self.user): + if not self.user: self.host = socket.gethostname() self.result.host = socket.gethostname() def get_cmd_result(self, cmd, user=None): - ''' + """ get cmd result - ''' + """ if not user: user = self.user status, output = SharedFuncs().run_shell_cmd(cmd, user) return status, output def get_sql_result(self, sql): - ''' + """ get sql result - ''' + """ if self.context.port: - status, output = SharedFuncs().get_sql_result(sql, - self.context.db_user, - self.context.db_passwd, - self.context.db_addr, - self.context.port, - self.context.app_path) + status, output = SharedFuncs().get_sql_result( + sql, + self.context.db_user, + self.context.db_passwd, + self.context.db_addr, + self.context.port, + self.context.app_path, + ) return status, output else: return "-1", "miss parameter [-P]" def run_check(self, context, logger): - ''' + """ main process for checking - ''' + """ content = "" except_val = {} self.init_form(context) @@ -590,8 +632,12 @@ class BaseItem(object): except_val["except"] = str(e) self.result.rst = ResultStatus.ERROR self.result.val = json.dumps(except_val) - logger.error("Exception occur when running %s:\n%s:traceback%s", self.name, str(e), - traceback.format_exc()) + logger.error( + "Exception occur when running %s:\n%s:traceback%s", + self.name, + str(e), + traceback.format_exc(), + ) finally: # output result content = self.result.output(context.out_path) @@ -600,9 +646,9 @@ class BaseItem(object): class ItemResult(object): - ''' + """ inspection inspection framework - ''' + """ def __init__(self, name): self.name = name @@ -612,26 +658,26 @@ class ItemResult(object): self.suggestion = "" self.epv = "" self.des = "" - self.category = 'other' + self.category = "other" self.analysis = "" def __iter__(self): - ''' + """ make iterable - ''' + """ return iter(self._items) def __getitem__(self, idx): - ''' + """ get item - ''' + """ return self._items[idx] @staticmethod def parse(output): - ''' + """ parse output - ''' + """ item_result = None local_item_result = None lines = output.splitlines() @@ -644,43 +690,49 @@ class ItemResult(object): if not line: continue - if line.startswith('[HOST ]'): + if line.startswith("[HOST ]"): host = line.split()[-1] - if line.startswith('[NAME ]'): + if line.startswith("[NAME ]"): name = line.split()[-1] if item_result is None: item_result = ItemResult(name) else: item_result.append(local_item_result) local_item_result = LocalItemResult(name, host) - if line.startswith('[RESULT ]'): + if line.startswith("[RESULT ]"): local_item_result.rst = line.split()[-1] value = ItemResult.__parse_multi_line(lines, idx) - if line.startswith('[VALUE ]'): + if line.startswith("[VALUE ]"): local_item_result.val = value - if line.startswith('[EXPECT ]'): + if line.startswith("[EXPECT ]"): local_item_result.epv = value - if line.startswith('[DESCRIPT_EN]'): + if line.startswith("[DESCRIPT_EN]"): local_item_result.des = value - if line.startswith('[SUGGEST ]'): + if line.startswith("[SUGGEST ]"): local_item_result.sug = value - if line.startswith('[REFER ]'): + if line.startswith("[REFER ]"): local_item_result.raw = value return item_result @staticmethod def __parse_multi_line(lines, start_idx): - ''' + """ parse line by line - ''' + """ vals = [] starter = ( - '[HOST ]', '[NAME ]', '[RESULT ]', '[VALUE ]', '[REFER ]', '[EXPECT ]', - '[DESCRIPT_EN]', '[SUGGEST ]' + "[HOST ]", + "[NAME ]", + "[RESULT ]", + "[VALUE ]", + "[REFER ]", + "[EXPECT ]", + "[DESCRIPT_EN]", + "[SUGGEST ]", ) - for line in lines[start_idx + 1:]: + for line in lines[start_idx + 1 :]: if line.strip().startswith(starter): break else: @@ -688,7 +740,7 @@ class ItemResult(object): return "\n".join(vals) def append(self, val): - ''' + """ append item - ''' + """ self._items.append(val) diff --git a/pkg/deploy/action/inspection/inspection_scripts/kernal/gs_check.py b/pkg/deploy/action/inspection/inspection_scripts/kernal/gs_check.py index ccee2186..0bf5f201 100644 --- a/pkg/deploy/action/inspection/inspection_scripts/kernal/gs_check.py +++ b/pkg/deploy/action/inspection/inspection_scripts/kernal/gs_check.py @@ -11,6 +11,7 @@ import socket import pwd import json from abc import abstractmethod + sys.path.append("/opt/cantian/action/cantian") from cantian_funclib import Execution from cantian_funclib import get_abs_path @@ -30,7 +31,7 @@ gPyVersion = platform.python_version() if gPyVersion[0] == "3": import importlib -if CURRENT_OS != 'Linux': +if CURRENT_OS != "Linux": print("Error:Check os failed:current os is not linux") raise ValueError("Error:Check os failed:current os is not linux") @@ -46,7 +47,7 @@ class CheckContext: """ # Initialize the self.clusterInfo variable curr_path = os.path.realpath(__file__) - self.base_path = os.path.join(os.path.split(curr_path)[0], 'inspection') + self.base_path = os.path.join(os.path.split(curr_path)[0], "inspection") self.user = None self.support_items = {} self.support_scenes = {} @@ -80,9 +81,10 @@ class CheckContext: # Exception class # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # class CheckException(Exception): - ''' + """ base class of exception - ''' + """ + def __init__(self, content): super(CheckException, self).__init__(self) self.code = "CANTIAN-53000" @@ -93,9 +95,10 @@ class CheckException(Exception): class CheckNAException(CheckException): - ''' + """ NA checkItem exception - ''' + """ + def __init__(self, item): super(CheckNAException, self).__init__(self.__class__.__name__) self.code = "CANTIAN-53033" @@ -104,9 +107,10 @@ class CheckNAException(CheckException): class TimeoutException(CheckException): - ''' + """ timeout exception - ''' + """ + def __init__(self, second): super(TimeoutException, self).__init__(self.__class__.__name__) self.code = "CANTIAN-53028" @@ -115,21 +119,42 @@ class TimeoutException(CheckException): def get_current_user(): - ''' + """ get current user - ''' + """ # Get the current user return pwd.getpwuid(os.getuid())[0] def check_legality(parameter_string): - ''' + """ Check for illegal characters - ''' + """ # the list of invalid characters - value_check_list = ["|", ";", "&", "$", "<", ">", "`", "\\", "'", "\"", - "{", "}", "(", ")", "[", "]", "~", "*", - "?", "!", "\n", " "] + value_check_list = [ + "|", + ";", + "&", + "$", + "<", + ">", + "`", + "\\", + "'", + '"', + "{", + "}", + "(", + ")", + "[", + "]", + "~", + "*", + "?", + "!", + "\n", + " ", + ] # judge illegal characters for ch in value_check_list: if parameter_string.find(ch) >= 0: @@ -138,17 +163,18 @@ def check_legality(parameter_string): class SharedFuncs: - ''' + """ defined tools for executing cmd and sql - ''' + """ + def __init__(self): pass @staticmethod def run_shell_cmd(cmd, user=None, mpprc_file="", array=(0,)): - ''' + """ defined tools for cmd - ''' + """ if not isinstance(array, tuple): return -1, "parameter [array] is illegal" @@ -160,18 +186,18 @@ class SharedFuncs: if max(array) > len(cmd.split("|")) - 1: return -1, "parameter [array] is illegal" - if (mpprc_file): + if mpprc_file: cmd = "source '%s'; %s" % (mpprc_file, cmd) cmd = cmd + "; echo ${PIPESTATUS[*]}" # change user but can not be root user - if (user and user != get_current_user()): + if user and user != get_current_user(): cmd = "su -s /bin/bash - %s -c 'source ~/.bashrc; %s'" % (user, cmd) # execute cmd - p = subprocess.Popen(['bash', '-c', cmd], - stdout=subprocess.PIPE, - stderr=subprocess.PIPE) + p = subprocess.Popen( + ["bash", "-c", cmd], stdout=subprocess.PIPE, stderr=subprocess.PIPE + ) # get normal and abnormal output (stdoutdata, stderrdata) = p.communicate() @@ -179,11 +205,11 @@ class SharedFuncs: status = p.returncode if gPyVersion[0] == "3": - ''' + """ python3's Popen returned Byte python2's Popen returned str convert to str if python3 - ''' + """ stdoutdata = stdoutdata.decode() stderrdata = stderrdata.decode() @@ -207,7 +233,7 @@ class SharedFuncs: return status, stdoutdata + stderrdata def get_abs_path(self, _file): - for _path in os.environ["PATH"].split(':'): + for _path in os.environ["PATH"].split(":"): abs_file = os.path.normpath(os.path.join(_path, _file)) if os.path.exists(abs_file): return abs_file @@ -223,20 +249,24 @@ class SharedFuncs: return 1, "CantianDB 100 V300R001" ctsql_path = os.path.join(app_path, "bin") - sql_cmd = "source ~/.bashrc && %s/ctsql %s@%s:%s -q -c \"%s\"" % (ctsql_path, - db_user, - db_addr, - port, - sql) + sql_cmd = 'source ~/.bashrc && %s/ctsql %s@%s:%s -q -c "%s"' % ( + ctsql_path, + db_user, + db_addr, + port, + sql, + ) if gPyVersion[0] == "3": sql_cmd = sql_cmd.encode() - p = subprocess.Popen(['bash', '-c', sql_cmd], - shell=False, - stdin=subprocess.PIPE, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE) + p = subprocess.Popen( + ["bash", "-c", sql_cmd], + shell=False, + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + ) db_passwd += "\n" @@ -262,22 +292,19 @@ class SharedFuncs: process = None ssh = get_abs_path("ssh") try: - ssh_base = "%s -o NumberOfPasswordPrompts=1 %s@%s" % (ssh, - user, - ip) + ssh_base = "%s -o NumberOfPasswordPrompts=1 %s@%s" % (ssh, user, ip) process = Execution(ssh_base) - idx = process.expect(['(P|p)assword:']) + idx = process.expect(["(P|p)assword:"]) if idx == 0: process.sendLine(passwd) - done_flag = 'gs_check done' + done_flag = "gs_check done" process.sendLine("%s; echo 'gs_check done'" % ssh_sql) - escape = 'unicode-escape' + escape = "unicode-escape" while True: - idx = process.expect([done_flag, "Please enter password"], - timeout=50) + idx = process.expect([done_flag, "Please enter password"], timeout=50) if idx == 0: - process.sendLine('exit') + process.sendLine("exit") status = 0 output = str(process.context_before.decode(escape)) output = output.split(done_flag)[0] @@ -285,35 +312,32 @@ class SharedFuncs: elif idx == 1: process.sendLine(db_passwd) else: - process.sendLine('exit') + process.sendLine("exit") status = 1 context_buffer = process.context_buffer.decode(escape) output = str(context_buffer).split(done_flag)[0] break except Exception as err: if process: - process.sendLine('exit') + process.sendLine("exit") status = 1 output = str(err) output += os.linesep - context_buffer = process.context_buffer.decode('unicode-escape') + context_buffer = process.context_buffer.decode("unicode-escape") output += context_buffer.split(done_flag)[0] return status, output def get_sql_result(self, sql, db_user, db_passwd, db_addr, port, app_path): - ''' + """ get records of sql output success:state, {'records':[[list1], [list2]], "title":[list_title]} failed: state, string_errormsg - ''' - status, output = self.run_sql_cmd(sql, - db_user, - db_passwd, - db_addr, - port, - app_path) + """ + status, output = self.run_sql_cmd( + sql, db_user, db_passwd, db_addr, port, app_path + ) if not status: - if (output.find("CT-") != -1) : + if output.find("CT-") != -1: status = -1 return status, output.split("\n")[2] result_records = {"title": [], "records": []} @@ -328,21 +352,16 @@ class SharedFuncs: return status, "\n".join([_f for _f in output.split("\n") if _f]) def verify_conn(self, sql, dbuser, dbpwd, db_addr, port, app_path): - ''' + """ function : get ip type input :string, string, string, string, string, string output : iptype - ''' + """ if not (dbuser and dbpwd): raise Exception("[ERROR]: Username and password cannot be empty") check_legality(dbuser) # execute the command to verify the database connection - status, output = self.run_sql_cmd(sql, - dbuser, - dbpwd, - db_addr, - port, - app_path) + status, output = self.run_sql_cmd(sql, dbuser, dbpwd, db_addr, port, app_path) if status: raise Exception("[ERROR]: %s" % output) @@ -357,8 +376,7 @@ def get_validity(path, path_desc): if not os.path.exists(real_path): raise Exception("The %s %s is not exists." % (path_desc, real_path)) if not os.path.isdir(real_path): - raise Exception("The %s %s is not directory type." % (path_desc, - real_path)) + raise Exception("The %s %s is not directory type." % (path_desc, real_path)) return real_path @@ -370,9 +388,10 @@ def __print_on_screen(msg): class ResultStatus(object): - ''' + """ define result status - ''' + """ + OK = "OK" NA = "NA" WARNING = "WARNING" @@ -381,9 +400,9 @@ class ResultStatus(object): class LocalItemResult(object): - ''' + """ the check result running on one host - ''' + """ def __init__(self, name, host): self.name = name @@ -419,24 +438,28 @@ class LocalItemResult(object): val = self.val if self.val else "" raw = self.raw if self.raw else "" try: - content = output_doc.format(name=self.name, - rst=self.rst, - host=self.host, - val=val, - epv=self.epv, - des=self.des, - sug=self.sug, - raw=raw) + content = output_doc.format( + name=self.name, + rst=self.rst, + host=self.host, + val=val, + epv=self.epv, + des=self.des, + sug=self.sug, + raw=raw, + ) except Exception: - output_utf8 = output_doc.encode('utf-8') - content = output_utf8.format(name=self.name, - rst=self.rst, - host=self.host, - val=val, - epv=self.epv, - des=self.des, - sug=self.sug, - raw=raw.decode('utf-8')) + output_utf8 = output_doc.encode("utf-8") + content = output_utf8.format( + name=self.name, + rst=self.rst, + host=self.host, + val=val, + epv=self.epv, + des=self.des, + sug=self.sug, + raw=raw.decode("utf-8"), + ) return content def to_json(self): @@ -445,11 +468,13 @@ class LocalItemResult(object): detail_result["error"] = {} detail_result["error"]["code"] = 0 detail_result["error"]["description"] = "" - if (self.rst == ResultStatus.NA): + if self.rst == ResultStatus.NA: rst = "\033[0;37m%s\033[0m" % "NONE" - elif (self.rst == ResultStatus.WARNING or - self.rst == ResultStatus.ERROR or - self.rst == ResultStatus.NG): + elif ( + self.rst == ResultStatus.WARNING + or self.rst == ResultStatus.ERROR + or self.rst == ResultStatus.NG + ): rst = "\033[0;31m%s\033[0m" % self.rst else: rst = "\033[0;32m%s\033[0m" % ResultStatus.OK @@ -468,13 +493,18 @@ class LocalItemResult(object): detail_result["data"]["REFER"] = [] for raw in self.raw.split("\n"): self.add_refer(raw, detail_result["data"]["REFER"]) - if (self.rst == ResultStatus.NG) : + if self.rst == ResultStatus.NG: detail_result["error"]["code"] = 1 - detail_result["error"]["description"] = \ - "{} is failed, expect val: {} current val: {}".format(self.des, self.epv, val) - elif (self.rst == ResultStatus.ERROR) : + detail_result["error"]["description"] = ( + "{} is failed, expect val: {} current val: {}".format( + self.des, self.epv, val + ) + ) + elif self.rst == ResultStatus.ERROR: detail_result["error"]["code"] = -1 - detail_result["error"]["description"] = "{} is failed, error msg: \"{}\"".format(self.des, val["except"]) + detail_result["error"]["description"] = ( + '{} is failed, error msg: "{}"'.format(self.des, val["except"]) + ) json_dump = json.dumps(detail_result, ensure_ascii=False, indent=2) print(json_dump) @@ -488,14 +518,14 @@ class LocalItemResult(object): class BaseItem(object): - ''' + """ base class of check item - ''' + """ def __init__(self, name): - ''' + """ Constructor - ''' + """ self.name = name self.title = None self.suggestion = None @@ -503,10 +533,10 @@ class BaseItem(object): self.time = int(time.time()) self.standard = None self.threshold = {} - self.category = 'other' - self.permission = 'user' - self.analysis = 'default' - self.scope = 'all' + self.category = "other" + self.permission = "user" + self.analysis = "default" + self.scope = "all" self.cluster = None self.user = None self.nodes = None @@ -528,15 +558,15 @@ class BaseItem(object): @abstractmethod def do_check(self): - ''' + """ check script for each item - ''' + """ pass def init_form(self, context): - ''' + """ initialize the check item from context - ''' + """ self.context = context self.user = context.user self.nodes = context.nodes @@ -557,38 +587,40 @@ class BaseItem(object): self.result.epv = self.epv self.result.des = self.title # new host without cluster installed - if (not self.user): + if not self.user: self.host = socket.gethostname() self.result.host = socket.gethostname() def get_cmd_result(self, cmd, user=None): - ''' + """ get cmd result - ''' + """ if not user: user = self.user status, output = SharedFuncs().run_shell_cmd(cmd, user) return status, output def get_sql_result(self, sql): - ''' + """ get sql result - ''' + """ if self.context.port: - status, output = SharedFuncs().get_sql_result(sql, - self.context.db_user, - self.context.db_passwd, - self.context.db_addr, - self.context.port, - self.context.app_path) + status, output = SharedFuncs().get_sql_result( + sql, + self.context.db_user, + self.context.db_passwd, + self.context.db_addr, + self.context.port, + self.context.app_path, + ) return status, output else: return "-1", "miss prameter [-P]" def run_check(self, context, logger): - ''' + """ main process for checking - ''' + """ content = "" except_val = {} self.init_form(context) @@ -604,7 +636,9 @@ class BaseItem(object): except_val["except"] = str(Exception) self.result.rst = ResultStatus.ERROR self.result.val = json.dumps(except_val) - logger.error("Exception occur when running {}:\n{}".format(self.name, str(Exception))) + logger.error( + "Exception occur when running {}:\n{}".format(self.name, str(Exception)) + ) finally: # output result content = self.result.output(context.out_path) @@ -613,9 +647,10 @@ class BaseItem(object): class ItemResult(object): - ''' + """ inspection inspection framework - ''' + """ + def __init__(self, name): self.name = name self._items = [] @@ -624,86 +659,91 @@ class ItemResult(object): self.suggestion = "" self.epv = "" self.des = "" - self.category = 'other' + self.category = "other" self.analysis = "" def __iter__(self): - ''' + """ make iterable - ''' + """ return iter(self._items) def __getitem__(self, idx): - ''' + """ get item - ''' + """ return self._items[idx] def append(self, val): - ''' + """ append item - ''' + """ self._items.append(val) @staticmethod def parse(output): - ''' + """ parse output - ''' + """ item_result = None local_item_result = None host = None idx = 0 for line in output.splitlines(): idx += 1 - if (idx == len(output.splitlines()) and - local_item_result is not None): + if idx == len(output.splitlines()) and local_item_result is not None: item_result.append(local_item_result) current = line.strip() - if (not current): + if not current: continue - if (current.startswith('[HOST ]')): + if current.startswith("[HOST ]"): host = current.split()[-1].strip() - if (current.startswith('[NAME ]')): + if current.startswith("[NAME ]"): name = current.split()[-1].strip() - if (item_result is None): + if item_result is None: item_result = ItemResult(name) - if (local_item_result is not None): + if local_item_result is not None: item_result.append(local_item_result) - local_item_result = LocalItemResult(current.split()[-1].strip(), - host) - if (current.startswith('[RESULT ]')): + local_item_result = LocalItemResult(current.split()[-1].strip(), host) + if current.startswith("[RESULT ]"): local_item_result.rst = current.split()[-1].strip() - if (current.startswith('[VALUE ]')): + if current.startswith("[VALUE ]"): value = ItemResult.__parse_multi_line(output.splitlines()[idx:]) local_item_result.val = value - if (current.startswith('[EXPECT ]')): + if current.startswith("[EXPECT ]"): exp = ItemResult.__parse_multi_line(output.splitlines()[idx:]) local_item_result.epv = exp - if (current.startswith('[DESCRIPT_EN]')): + if current.startswith("[DESCRIPT_EN]"): des_en = ItemResult.__parse_multi_line(output.splitlines()[idx:]) local_item_result.des = des_en - if (current.startswith('[SUGGEST ]')): + if current.startswith("[SUGGEST ]"): sug = ItemResult.__parse_multi_line(output.splitlines()[idx:]) local_item_result.sug = sug - if (current.startswith('[REFER ]')): + if current.startswith("[REFER ]"): refer = ItemResult.__parse_multi_line(output.splitlines()[idx:]) local_item_result.raw = refer return item_result @staticmethod def __parse_multi_line(lines): - ''' + """ parse line by line - ''' + """ vals = [] - starter = ('[HOST ]', '[NAME ]', '[RESULT ]', - '[VALUE ]', '[REFER ]', '[EXPECT ]', - '[DESCRIPT_EN]', '[SUGGEST ]') + starter = ( + "[HOST ]", + "[NAME ]", + "[RESULT ]", + "[VALUE ]", + "[REFER ]", + "[EXPECT ]", + "[DESCRIPT_EN]", + "[SUGGEST ]", + ) for line in lines: current = line.strip() - if (current.startswith(starter)): + if current.startswith(starter): break else: vals.append(current) - return "\n".join(vals) \ No newline at end of file + return "\n".join(vals) diff --git a/pkg/deploy/action/inspection/inspection_scripts/mysql/common_func.py b/pkg/deploy/action/inspection/inspection_scripts/mysql/common_func.py index 633f8522..f93d227d 100644 --- a/pkg/deploy/action/inspection/inspection_scripts/mysql/common_func.py +++ b/pkg/deploy/action/inspection/inspection_scripts/mysql/common_func.py @@ -15,8 +15,13 @@ def _exec_popen(cmd, values=None): if not values: values = [] bash_cmd = ["bash"] - pobj = subprocess.Popen(bash_cmd, shell=False, stdin=subprocess.PIPE, - stdout=subprocess.PIPE, stderr=subprocess.PIPE) + pobj = subprocess.Popen( + bash_cmd, + shell=False, + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + ) py_version = platform.python_version() if py_version[0] == "3": @@ -41,4 +46,4 @@ def _exec_popen(cmd, values=None): if stderr[-1:] == os.linesep: stderr = stderr[:-1] - return pobj.returncode, stdout, stderr \ No newline at end of file + return pobj.returncode, stdout, stderr diff --git a/pkg/deploy/action/inspection/inspection_scripts/mysql/mysql_connection_check.py b/pkg/deploy/action/inspection/inspection_scripts/mysql/mysql_connection_check.py index 57f78fd3..0a238174 100644 --- a/pkg/deploy/action/inspection/inspection_scripts/mysql/mysql_connection_check.py +++ b/pkg/deploy/action/inspection/inspection_scripts/mysql/mysql_connection_check.py @@ -4,21 +4,16 @@ import json import os import subprocess import sys -sys.path.append('/mf_connector/inspection') + +sys.path.append("/mf_connector/inspection") from log_tool import setup from inspection_scripts.mysql.mysql_diskspace_check import DiskSpaceCheck -class ConnectionCheck(): +class ConnectionCheck: def __init__(self): - self.result_json = { - 'data': {}, - 'error': { - 'code': 0, - 'description': '' - } - } + self.result_json = {"data": {}, "error": {"code": 0, "description": ""}} def connection_check(self, logger): logger.info("connection check start!") @@ -30,27 +25,28 @@ class ConnectionCheck(): if status: logger.error("can not get engines information") self.result_json["error"]["code"] = -1 - self.result_json['error']['description'] = output + self.result_json["error"]["description"] = output else: res = output.split("\n") res = res[1:] lines = [line.split("\t") for line in res] for line in lines: if line[0] == "CTC" and line[1] == "DEFAULT": - self.result_json["data"]["RESULT"] = 'check connection succ!' + self.result_json["data"]["RESULT"] = "check connection succ!" logger.info("check connection succ!") break else: logger.error("the connection check failed") self.result_json["error"]["code"] = -1 - self.result_json['error']['description'] = "the connection check failed, CTC is not default" + self.result_json["error"][ + "description" + ] = "the connection check failed, CTC is not default" return self.result_json return self.result_json -if __name__ == '__main__': +if __name__ == "__main__": mysql_log = setup("mysql") cn = ConnectionCheck() result_json = cn.connection_check(mysql_log) print(json.dumps(result_json, indent=1)) - diff --git a/pkg/deploy/action/inspection/inspection_scripts/mysql/mysql_diskspace_check.py b/pkg/deploy/action/inspection/inspection_scripts/mysql/mysql_diskspace_check.py index d82c5799..7a1a6145 100644 --- a/pkg/deploy/action/inspection/inspection_scripts/mysql/mysql_diskspace_check.py +++ b/pkg/deploy/action/inspection/inspection_scripts/mysql/mysql_diskspace_check.py @@ -4,21 +4,16 @@ import json import os import subprocess import sys -sys.path.append('/mf_connector/inspection') + +sys.path.append("/mf_connector/inspection") from log_tool import setup from common_func import _exec_popen -class DiskSpaceCheck(): +class DiskSpaceCheck: def __init__(self): - self.result_json = { - 'data': {}, - 'error': { - 'code': 0, - 'description': '' - } - } + self.result_json = {"data": {}, "error": {"code": 0, "description": ""}} @staticmethod def get_cmd_res(cmd): @@ -26,14 +21,14 @@ class DiskSpaceCheck(): return status, output def pwd_check(self, logger): - cmd_pwd = '/docker-entrypoint-initdb.d/encrypt -d $MYSQL_AGENT_PASSWORD' + cmd_pwd = "/docker-entrypoint-initdb.d/encrypt -d $MYSQL_AGENT_PASSWORD" pwd_code, pwd = self.get_cmd_res(cmd_pwd) if pwd_code == 0: return pwd else: logger.error("can not get password") self.result_json["error"]["code"] = -1 - self.result_json['error']['description'] = pwd + self.result_json["error"]["description"] = pwd return self.result_json def dir_check(self, logger): @@ -41,14 +36,18 @@ class DiskSpaceCheck(): if isinstance(temp, dict): return temp password = temp.strip() - cmd_dir = "mysql -uRDS_agent -p" + password + ''' -e"show global variables like 'datadir';"''' + cmd_dir = ( + "mysql -uRDS_agent -p" + + password + + ''' -e"show global variables like 'datadir';"''' + ) datadir_code, res = self.get_cmd_res(cmd_dir) if datadir_code == 0: return res else: logger.error("can not get data directory") self.result_json["error"]["code"] = -1 - self.result_json['error']['description'] = res + self.result_json["error"]["description"] = res return self.result_json def diskspace_check(self, logger): @@ -64,22 +63,26 @@ class DiskSpaceCheck(): if int(use_space) < 90: self.result_json["error"]["code"] = 0 self.result_json["error"]["description"] = "" - self.result_json["data"]["RESULT"] = ' the used disk space are less than 90%, succ!' + self.result_json["data"][ + "RESULT" + ] = " the used disk space are less than 90%, succ!" logger.info("the remaining disk spaces check succ!") return self.result_json else: logger.error("the remaining spaces are less than 10%") self.result_json["error"]["code"] = -1 - self.result_json['error']['description'] = "the remaining spaces are less than 10% " + self.result_json["error"][ + "description" + ] = "the remaining spaces are less than 10% " return self.result_json else: logger.error("can not get the disk spaces info") self.result_json["error"]["code"] = -1 - self.result_json['error']['description'] = value + self.result_json["error"]["description"] = value return self.result_json -if __name__ == '__main__': +if __name__ == "__main__": mysql_log = setup("mysql") ds = DiskSpaceCheck() result_json = ds.diskspace_check(mysql_log) diff --git a/pkg/deploy/action/inspection/inspection_scripts/mysql/mysql_file_check.py b/pkg/deploy/action/inspection/inspection_scripts/mysql/mysql_file_check.py index 75bbc205..b025a5e0 100644 --- a/pkg/deploy/action/inspection/inspection_scripts/mysql/mysql_file_check.py +++ b/pkg/deploy/action/inspection/inspection_scripts/mysql/mysql_file_check.py @@ -4,26 +4,30 @@ import json import os import subprocess import sys -sys.path.append('/mf_connector/inspection') + +sys.path.append("/mf_connector/inspection") from log_tool import setup -class SharedFileCheck(): +class SharedFileCheck: def __init__(self): self.path = "/dev/shm/" self.shm_list = [ - "cantian.0", "cantian.1", "cantian.2", "cantian.3", "cantian.4", "cantian.5", "cantian.6", - "cantian.7", "cantian.8", "cantian.shm_unix_sock", "cantian_shm_config_0.txt", - "cantian_shm_config_1.txt" + "cantian.0", + "cantian.1", + "cantian.2", + "cantian.3", + "cantian.4", + "cantian.5", + "cantian.6", + "cantian.7", + "cantian.8", + "cantian.shm_unix_sock", + "cantian_shm_config_0.txt", + "cantian_shm_config_1.txt", ] - self.result_json = { - 'data': {}, - 'error': { - 'code': 0, - 'description': '' - } - } + self.result_json = {"data": {}, "error": {"code": 0, "description": ""}} def file_check(self, logger): myfile = [] @@ -33,7 +37,7 @@ class SharedFileCheck(): else: logger.error("there are some files missed") self.result_json["error"]["code"] = -1 - self.result_json['error']['description'] = "not all shm files exist" + self.result_json["error"]["description"] = "not all shm files exist" break return myfile @@ -44,13 +48,14 @@ class SharedFileCheck(): return self.result_json for i in myfile: # 合法uid是6000， gid是5000 - if os.stat(self.path + i)[4] == 6000 and \ - os.stat(self.path + i)[5] == 5000: + if os.stat(self.path + i)[4] == 6000 and os.stat(self.path + i)[5] == 5000: uid.append(i) else: logger.error("not all uid of shm files are 5000") self.result_json["error"]["code"] = -1 - self.result_json['error']['description'] = "not all uid of shm files are 5000 " + self.result_json["error"][ + "description" + ] = "not all uid of shm files are 5000 " break return uid @@ -61,19 +66,24 @@ class SharedFileCheck(): if uid != self.shm_list: return self.result_json for i in uid: - if (os.access(self.path + i, os.R_OK) is True) and (os.access(self.path + i, os.W_OK) is True): + if (os.access(self.path + i, os.R_OK) is True) and ( + os.access(self.path + i, os.W_OK) is True + ): rights.append(i) else: logger.error("user does not have right to read or write") self.result_json["error"]["code"] = -1 - self.result_json['error']['description'] = "not all files can be read and written" + self.result_json["error"][ + "description" + ] = "not all files can be read and written" break if rights == self.shm_list: - self.result_json["data"]["RESULT"] = 'check shm files succ!' + self.result_json["data"]["RESULT"] = "check shm files succ!" return self.result_json -if __name__ == '__main__': + +if __name__ == "__main__": mysql_log = setup("mysql") sf = SharedFileCheck() result_json = sf.rights_check(mysql_log) - print(json.dumps(result_json, indent=1)) \ No newline at end of file + print(json.dumps(result_json, indent=1)) diff --git a/pkg/deploy/action/inspection/inspection_scripts/mysql/mysql_transaction_isolation_check.py b/pkg/deploy/action/inspection/inspection_scripts/mysql/mysql_transaction_isolation_check.py index f345a054..293a9049 100644 --- a/pkg/deploy/action/inspection/inspection_scripts/mysql/mysql_transaction_isolation_check.py +++ b/pkg/deploy/action/inspection/inspection_scripts/mysql/mysql_transaction_isolation_check.py @@ -3,53 +3,55 @@ import json import os import sys -sys.path.append('/mf_connector/inspection') + +sys.path.append("/mf_connector/inspection") from log_tool import setup from inspection_scripts.mysql.mysql_diskspace_check import DiskSpaceCheck -class TransactionIsolationCheck(): +class TransactionIsolationCheck: def __init__(self): - self.result_json = { - 'data': {}, - 'error': { - 'code': 0, - 'description': '' - } - } + self.result_json = {"data": {}, "error": {"code": 0, "description": ""}} def transaction_isolation_check(self, logger): logger.info("transaction isolation level check start!") password = DiskSpaceCheck().pwd_check(logger) if isinstance(password, dict): - self.result_json['error']['description'] = "Password is a dictionary." + self.result_json["error"]["description"] = "Password is a dictionary." return self.result_json - cmd = "mysql -uRDS_agent -p" + password + ''' -e"show variables like 'transaction_isolation';"''' + cmd = ( + "mysql -uRDS_agent -p" + + password + + ''' -e"show variables like 'transaction_isolation';"''' + ) status, output = DiskSpaceCheck().get_cmd_res(cmd) if status: logger.error("can not get transaction isolation information") self.result_json["error"]["code"] = -1 - self.result_json['error']['description'] = output + self.result_json["error"]["description"] = output else: res = output.split("\n") res = res[1:] lines = [line.split("\t") for line in res] for line in lines: if line[0] == "transaction_isolation" and line[1] == "READ-COMMITTED": - self.result_json["data"]["RESULT"] = 'check transaction isolation succ!' + self.result_json["data"][ + "RESULT" + ] = "check transaction isolation succ!" logger.info("check transaction isolation succeed!") break else: logger.error("the transaction isolation check failed") self.result_json["error"]["code"] = -1 - self.result_json['error']['description'] = \ - "the transaction isolation check failed, isolation level is not READ-COMMITTED" + self.result_json["error"][ + "description" + ] = "the transaction isolation check failed, isolation level is not READ-COMMITTED" return self.result_json -if __name__ == '__main__': +if __name__ == "__main__": mysql_log = setup("mysql") cn = TransactionIsolationCheck() result_json = cn.transaction_isolation_check(mysql_log) - print(json.dumps(result_json, indent=1)) \ No newline at end of file + print(json.dumps(result_json, indent=1)) diff --git a/pkg/deploy/action/inspection/inspection_task.py b/pkg/deploy/action/inspection/inspection_task.py index e84097a8..124d22ea 100644 --- a/pkg/deploy/action/inspection/inspection_task.py +++ b/pkg/deploy/action/inspection/inspection_task.py @@ -14,23 +14,23 @@ sys.path.append(CUR_PATH) import log_tool from generate_html_results import GenHtmlRes -LOG = log_tool.setup('om') +LOG = log_tool.setup("om") MAX_AUDIT_NUM = 10 DIR_NAME, _ = os.path.split(os.path.abspath(__file__)) -INSPECTION_PATH = str(Path('{}/inspections_log'.format(DIR_NAME))) -FAIL = 'fail' -SUCCESS = 'success' -SUCCESS_ENUM = [0, '0'] +INSPECTION_PATH = str(Path("{}/inspections_log".format(DIR_NAME))) +FAIL = "fail" +SUCCESS = "success" +SUCCESS_ENUM = [0, "0"] CTSQL_IP = "127.0.0.1" CTSQL_PORT = "1611" -LOG_DIRECTORY = f'{CUR_PATH}/inspection_task_log' +LOG_DIRECTORY = f"{CUR_PATH}/inspection_task_log" class InspectionTask: def __init__(self, _input_value, _use_smartkit=False): - self.input_param = 'all' + self.input_param = "all" self.inspection_map = self.read_inspection_config() self.use_smartkit = _use_smartkit @@ -46,12 +46,12 @@ class InspectionTask: self.fail_list = [] self.deply_user = self.get_depoly_user() self.user_map = { - 'cantian': self.deply_user, - 'cms': self.deply_user, - 'dbstor': self.deply_user, - 'mysql': self.deply_user, - 'ctmgr': 'ctmgruser', - 'ct_om': 'root' + "cantian": self.deply_user, + "cms": self.deply_user, + "dbstor": self.deply_user, + "mysql": self.deply_user, + "ctmgr": "ctmgruser", + "ct_om": "root", } @staticmethod @@ -63,27 +63,29 @@ class InspectionTask: return "" @staticmethod - def format_single_inspection_result(inspection_item, inspection_detail, execute_result, inspection_result): + def format_single_inspection_result( + inspection_item, inspection_detail, execute_result, inspection_result + ): return_value = { - 'inspection_item': inspection_item, - 'description_zn': inspection_detail.get("description_zn"), - 'description_en': inspection_detail.get("description_en"), - 'component': inspection_detail.get("component"), - 'inspection_result': execute_result, - 'inspection_detail': inspection_result.get('data'), - 'resource_en': inspection_detail.get("resource_en"), - 'resource_zh': inspection_detail.get("resource_zh") + "inspection_item": inspection_item, + "description_zn": inspection_detail.get("description_zn"), + "description_en": inspection_detail.get("description_en"), + "component": inspection_detail.get("component"), + "inspection_result": execute_result, + "inspection_detail": inspection_result.get("data"), + "resource_en": inspection_detail.get("resource_en"), + "resource_zh": inspection_detail.get("resource_zh"), } if inspection_result and isinstance(inspection_result, dict): - err_info = inspection_result.get('error', {}) - error_code = err_info.get('code') + err_info = inspection_result.get("error", {}) + error_code = err_info.get("code") if error_code is None: return return_value if error_code not in SUCCESS_ENUM: - return_value['inspection_result'] = FAIL - return_value['inspection_detail'] = { - 'error': inspection_result.get('error') + return_value["inspection_result"] = FAIL + return_value["inspection_detail"] = { + "error": inspection_result.get("error") } return return_value @@ -93,9 +95,11 @@ class InspectionTask: if not inspection_detail: raise ValueError("[error]: inspection item %s not exist" % inspection_item) - if not os.path.exists(inspection_detail.get('inspection_file_path')): - raise ValueError("[error]: inspection file: " - "%s not exist" % str(inspection_detail.get('inspection_file_path'))) + if not os.path.exists(inspection_detail.get("inspection_file_path")): + raise ValueError( + "[error]: inspection file: " + "%s not exist" % str(inspection_detail.get("inspection_file_path")) + ) @staticmethod def res_format_check(res_output): @@ -109,10 +113,12 @@ class InspectionTask: if not isinstance(res_output, dict): return False - if 'data' not in res_output or 'error' not in res_output: + if "data" not in res_output or "error" not in res_output: return False - if isinstance(res_output.get('error'), dict) and 'code' in res_output.get('error'): + if isinstance(res_output.get("error"), dict) and "code" in res_output.get( + "error" + ): return True return False @@ -125,7 +131,7 @@ class InspectionTask: True: the input username is legal False: the input username is illegal """ - reg_pattern = r'^\w+$' + reg_pattern = r"^\w+$" reg_match_res = re.findall(reg_pattern, user_name) if not reg_match_res or len(reg_match_res) >= 2: return False @@ -138,20 +144,27 @@ class InspectionTask: standby_keystore = "/opt/cantian/common/config/standby_keystore_bak.ks" sys.path.append("/opt/cantian/action/dbstor") from kmc_adapter import CApiWrapper - ctsql_ini_path = '/mnt/dbdata/local/cantian/tmp/data/cfg/ctsql.ini' - kmc_decrypt = CApiWrapper(primary_keystore=primary_keystore, standby_keystore=standby_keystore) + + ctsql_ini_path = "/mnt/dbdata/local/cantian/tmp/data/cfg/ctsql.ini" + kmc_decrypt = CApiWrapper( + primary_keystore=primary_keystore, standby_keystore=standby_keystore + ) kmc_decrypt.initialize() ctsql_ini_data = file_reader(ctsql_ini_path) - encrypt_pwd = ctsql_ini_data[ctsql_ini_data.find('=') + 1:].strip() + encrypt_pwd = ctsql_ini_data[ctsql_ini_data.find("=") + 1 :].strip() try: kmc_decrypt_pwd = kmc_decrypt.decrypt(encrypt_pwd) except Exception as error: - raise Exception('[result] decrypt ctsql passwd failed') from error + raise Exception("[result] decrypt ctsql passwd failed") from error finally: kmc_decrypt.finalize() - split_env = os.environ['LD_LIBRARY_PATH'].split(":") - filtered_env = [single_env for single_env in split_env if "/opt/cantian/dbstor/lib" not in single_env] - os.environ['LD_LIBRARY_PATH'] = ":".join(filtered_env) + split_env = os.environ["LD_LIBRARY_PATH"].split(":") + filtered_env = [ + single_env + for single_env in split_env + if "/opt/cantian/dbstor/lib" not in single_env + ] + os.environ["LD_LIBRARY_PATH"] = ":".join(filtered_env) return kmc_decrypt_pwd def read_inspection_config(self): @@ -159,7 +172,7 @@ class InspectionTask: reading inspection config file to obtain inspection component details :return: """ - with open(self.inspection_json_file, encoding='utf-8') as file: + with open(self.inspection_json_file, encoding="utf-8") as file: inspection_map = json.load(file) return inspection_map @@ -170,14 +183,16 @@ class InspectionTask: :param _input_value: all or [component names] :return: """ - if _input_value == 'all': + if _input_value == "all": return list(self.inspection_map.keys()) - if not _input_value.startswith('[') or not _input_value.endswith(']'): - LOG.error(f'input_value is: {_input_value}, format error') - raise ValueError('[error]: Input value is not correct; should be "all" or "[component1, component2, ...]".') + if not _input_value.startswith("[") or not _input_value.endswith("]"): + LOG.error(f"input_value is: {_input_value}, format error") + raise ValueError( + '[error]: Input value is not correct; should be "all" or "[component1, component2, ...]".' + ) - return _input_value[1:-1].split(',') + return _input_value[1:-1].split(",") def task_execute_single(self, inspection_detail, name_pwd, ip_port): """ @@ -197,8 +212,10 @@ class InspectionTask: utc_now = datetime.utcnow() cur_time = utc_now.replace(tzinfo=timezone.utc).astimezone(tz=None) node_info = self.get_node_ip() - audit_file = 'inspection_{}_{}'.format(node_info, str(cur_time.strftime("%Y%m%d%H%M%S"))) - audit_file_path = str(Path(self.audit_path + '/' + audit_file)) + audit_file = "inspection_{}_{}".format( + node_info, str(cur_time.strftime("%Y%m%d%H%M%S")) + ) + audit_file_path = str(Path(self.audit_path + "/" + audit_file)) if not os.path.exists(self.audit_path): os.mkdir(self.audit_path) @@ -213,8 +230,12 @@ class InspectionTask: shutil.rmtree(temp_path) audit_list.pop(0) # 生成html格式巡检结果 - GenHtmlRes(self.inspection_result, audit_file_path, node_info).generate_html_zh() - GenHtmlRes(self.inspection_result, audit_file_path, node_info).generate_html_en() + GenHtmlRes( + self.inspection_result, audit_file_path, node_info + ).generate_html_zh() + GenHtmlRes( + self.inspection_result, audit_file_path, node_info + ).generate_html_en() return audit_file @@ -223,11 +244,13 @@ class InspectionTask: user_name = "sys" system_pwd = self.decrypt_password() else: - user_name = input('Please input user:') + user_name = input("Please input user:") system_pwd = getpass.getpass("Please input password:") if not self.user_name_reg(user_name): - raise ValueError(f"[error] the input username '{user_name}' is illegal, " - f"please enter a correct username.") + raise ValueError( + f"[error] the input username '{user_name}' is illegal, " + f"please enter a correct username." + ) return system_pwd, user_name def get_user_pwd(self): @@ -238,7 +261,7 @@ class InspectionTask: required is False: empty tuple () """ for inspection_item in self.inspection_items: - if self.inspection_map.get(inspection_item, {}).get('need_pwd'): + if self.inspection_map.get(inspection_item, {}).get("need_pwd"): system_pwd, user_name = self.check_smartkit() return user_name, system_pwd @@ -252,7 +275,7 @@ class InspectionTask: required is False: empty tuple () """ for inspection_item in self.inspection_items: - if self.inspection_map.get(inspection_item, {}).get('need_ip'): + if self.inspection_map.get(inspection_item, {}).get("need_ip"): return CTSQL_IP, CTSQL_PORT return () @@ -272,28 +295,39 @@ class InspectionTask: raise ValueError(str(val_err)) from val_err try: - single_inspection_result = json.loads(self.task_execute_single(inspection_detail, name_pwd, ip_port)) + single_inspection_result = json.loads( + self.task_execute_single(inspection_detail, name_pwd, ip_port) + ) except Exception as _err: - LOG.error(f'execute item: {inspection_item} with {str(_err.__class__)} error: {str(_err)}') - print(f'[error]: inspection component: {inspection_item} execute it\'s inspection script failed') - formated_inspection_result = self.format_single_inspection_result(inspection_item, - inspection_detail, FAIL, {}) + LOG.error( + f"execute item: {inspection_item} with {str(_err.__class__)} error: {str(_err)}" + ) + print( + f"[error]: inspection component: {inspection_item} execute it's inspection script failed" + ) + formated_inspection_result = self.format_single_inspection_result( + inspection_item, inspection_detail, FAIL, {} + ) self.inspection_result.append(formated_inspection_result) self.fail_list.append(inspection_item) continue if not self.res_format_check(single_inspection_result): - print(f'[error]: inspection component: {inspection_item} obtain an unexpected result') - formated_inspection_result = self.format_single_inspection_result(inspection_item, - inspection_detail, FAIL, {}) + print( + f"[error]: inspection component: {inspection_item} obtain an unexpected result" + ) + formated_inspection_result = self.format_single_inspection_result( + inspection_item, inspection_detail, FAIL, {} + ) self.inspection_result.append(formated_inspection_result) self.fail_list.append(inspection_item) continue - formated_inspection_result = self.format_single_inspection_result(inspection_item, inspection_detail, - SUCCESS, single_inspection_result) + formated_inspection_result = self.format_single_inspection_result( + inspection_item, inspection_detail, SUCCESS, single_inspection_result + ) - if formated_inspection_result.get('inspection_result') == FAIL: + if formated_inspection_result.get("inspection_result") == FAIL: self.inspection_result.append(formated_inspection_result) self.fail_list.append(inspection_item) continue @@ -302,21 +336,27 @@ class InspectionTask: self.success_list.append(inspection_item) log_name = self.write_audit() - log_full_path = str(Path(f'{self.audit_path}/{log_name}')) + log_full_path = str(Path(f"{self.audit_path}/{log_name}")) if not self.fail_list: - raise ValueError(f'All components inspection execute success; \ninspection result file is {log_full_path}') + raise ValueError( + f"All components inspection execute success; \ninspection result file is {log_full_path}" + ) if not self.success_list: - raise ValueError(f'All components inspection execute failed; \ninspection result file is {log_full_path}') + raise ValueError( + f"All components inspection execute failed; \ninspection result file is {log_full_path}" + ) - raise ValueError(f'Component: [{", ".join(self.success_list)}] inspection execute success, ' - f'\ncomponent: [{", ".join(self.fail_list)} ]inspection execute failed; ' - f'\ninspection result file is {log_full_path}') + raise ValueError( + f'Component: [{", ".join(self.success_list)}] inspection execute success, ' + f'\ncomponent: [{", ".join(self.fail_list)} ]inspection execute failed; ' + f"\ninspection result file is {log_full_path}" + ) def file_reader(file_path): - with open(file_path, 'r') as file: + with open(file_path, "r") as file: return file.read() @@ -328,30 +368,35 @@ def main(input_val): # 运行期间修改日志路径为777，确保各模块可以在日志路径内创建日志文件 os.chmod(LOG_DIRECTORY, 0o777) from declear_env import DeclearEnv + # 获取当前为参天还是mysql容器 current_env = DeclearEnv().get_env_type() # 获取执行当前进程的用户名 current_executor = DeclearEnv().get_executor() if current_env == "cantian": from inspection_cantian import CantianInspection + cantian_inspection = CantianInspection(input_val, use_smartkit) cantian_inspection.task_execute() else: from inspection_mysql import MysqlInspection + mysql_inspection = MysqlInspection(input_val, use_smartkit) mysql_inspection.task_execute() -if __name__ == '__main__': +if __name__ == "__main__": input_value = None use_smartkit = False try: input_value = sys.argv[1] except Exception as err: _ = err - exit('[error]: Input format is not correct, missing input value;' - ' Input value could be "all" or "[component1,component2,...]".') + exit( + "[error]: Input format is not correct, missing input value;" + ' Input value could be "all" or "[component1,component2,...]".' + ) if len(sys.argv) == 3 and sys.argv[2] == "smartkit": use_smartkit = True sys.dont_write_bytecode = True diff --git a/pkg/deploy/action/inspection/log_tool.py b/pkg/deploy/action/inspection/log_tool.py index 701504b2..6f4150f3 100644 --- a/pkg/deploy/action/inspection/log_tool.py +++ b/pkg/deploy/action/inspection/log_tool.py @@ -13,9 +13,9 @@ CONSOLE_CONF = { "log_file_backup_count": 5, "log_date_format": "%Y-%m-%d %H:%M:%S", "logging_default_format_string": "%(asctime)s %(levelname)s [pid:%(process)d] [%(threadName)s] " - "[tid:%(thread)d] [%(filename)s:%(lineno)d %(funcName)s] %(message)s", + "[tid:%(thread)d] [%(filename)s:%(lineno)d %(funcName)s] %(message)s", "logging_context_format_string": "%(asctime)s %(levelname)s [pid:%(process)d] [%(threadName)s] " - "[tid:%(thread)d] [%(filename)s:%(lineno)d %(funcName)s] %(message)s" + "[tid:%(thread)d] [%(filename)s:%(lineno)d %(funcName)s] %(message)s", } } @@ -37,7 +37,7 @@ def _get_log_file_path(project): os.chown(logger_file, owner_uid, owner_gid) return logger_file - return '' + return "" def setup(project_name): @@ -55,8 +55,10 @@ def setup(project_name): log_path = _get_log_file_path(project_name) if log_path: file_log = handlers.RotatingFileHandler( - log_path, maxBytes=log_config.get("log_file_max_size"), - backupCount=log_config.get("log_file_backup_count")) + log_path, + maxBytes=log_config.get("log_file_max_size"), + backupCount=log_config.get("log_file_backup_count"), + ) log_root.addHandler(file_log) log_root.addHandler(console) @@ -64,7 +66,9 @@ def setup(project_name): handler.setFormatter( logging.Formatter( fmt=log_config.get("logging_context_format_string"), - datefmt=log_config.get("log_date_format"))) + datefmt=log_config.get("log_date_format"), + ) + ) if log_config.get("debug"): log_root.setLevel(logging.DEBUG) diff --git a/pkg/deploy/action/logic/common_func.py b/pkg/deploy/action/logic/common_func.py index 720a8c7c..9692b80f 100644 --- a/pkg/deploy/action/logic/common_func.py +++ b/pkg/deploy/action/logic/common_func.py @@ -17,11 +17,11 @@ def close_child_process(proc): os.killpg(proc.pid, signal.SIGKILL) except ProcessLookupError as err: _ = err - return 'success' + return "success" except Exception as err: return str(err) - return 'success' + return "success" def retry(retry_times, log, task, wait_times): @@ -40,7 +40,9 @@ def retry(retry_times, log, task, wait_times): continue else: raise err + return wrapper + return decorate @@ -51,8 +53,14 @@ def exec_popen(cmd, timeout=TIME_OUT): return: status code, standard output, error output """ bash_cmd = ["bash"] - pobj = subprocess.Popen(bash_cmd, shell=False, stdin=subprocess.PIPE, - stdout=subprocess.PIPE, stderr=subprocess.PIPE, preexec_fn=os.setsid) + pobj = subprocess.Popen( + bash_cmd, + shell=False, + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + preexec_fn=os.setsid, + ) pobj.stdin.write(cmd.encode()) pobj.stdin.write(os.linesep.encode()) try: @@ -80,12 +88,12 @@ def read_json_config(file_path): def write_json_config(file_path, data): flags = os.O_WRONLY | os.O_CREAT | os.O_TRUNC modes = stat.S_IWUSR | stat.S_IRUSR - with os.fdopen(os.open(file_path, flags, modes), 'w') as fp: + with os.fdopen(os.open(file_path, flags, modes), "w") as fp: json.dump(data, fp, indent=4) def file_reader(file_path): - with open(file_path, 'r') as file: + with open(file_path, "r") as file: return file.read() diff --git a/pkg/deploy/action/logic/storage_operate.py b/pkg/deploy/action/logic/storage_operate.py index d64680bf..c48e2e41 100644 --- a/pkg/deploy/action/logic/storage_operate.py +++ b/pkg/deploy/action/logic/storage_operate.py @@ -60,8 +60,11 @@ class StorageInf(object): :param fs_name: 文件系统名称 :return: """ - mount_dir = f"/mnt/dbdata/remote/{fs_name}" if prefix is None \ + mount_dir = ( + f"/mnt/dbdata/remote/{fs_name}" + if prefix is None else f"/mnt/dbdata/remote/{prefix}_{fs_name}" + ) mount_cmd = f"mount -t nfs {params} {logic_ip}:/{fs_name} " + mount_dir mkdir_cmd = f"if [ ! -d {mount_dir} ];then mkdir -p {mount_dir};fi" return_code, _, stderr = exec_popen(mkdir_cmd) @@ -96,10 +99,10 @@ class StorageInf(object): err_msg = err_msg % (status_code, error_code, error_des) cls.handle_error_msg(err_msg) rsp_code, rsp_result, rsp_data = result.get_rsp_data() - error_code = rsp_result.get('code') + error_code = rsp_result.get("code") if rsp_code != 0 or error_code != 0: - error_des = rsp_result.get('description') - error_sgt = rsp_result.get('suggestion') + error_des = rsp_result.get("description") + error_sgt = rsp_result.get("suggestion") err_msg = err_msg % (error_code, error_des, error_sgt) cls.handle_error_msg(err_msg) return rsp_data @@ -109,10 +112,10 @@ class StorageInf(object): err_msg = err_msg + ", Detail:[%s]%s.Suggestion:%s" result = ResponseParse(res) rsp_code, rsp_result, rsp_data = result.get_omtask_rsp_data() - if rsp_code != 0 or (rsp_result.get('code') and rsp_result.get('code') != 0): - error_des = rsp_result.get('description') - error_sgt = rsp_result.get('suggestion') - err_msg = err_msg % (rsp_result.get('code'), error_des, error_sgt) + if rsp_code != 0 or (rsp_result.get("code") and rsp_result.get("code") != 0): + error_des = rsp_result.get("description") + error_sgt = rsp_result.get("suggestion") + err_msg = err_msg % (rsp_result.get("code"), error_des, error_sgt) cls.handle_error_msg(err_msg) return rsp_data @@ -125,8 +128,10 @@ class StorageInf(object): try: self.rest_client.login() except Exception as _err: - err_msg = "Login DM[float ip:%s] failed, details: %s " \ - % (self.ip, str(_err)) + err_msg = "Login DM[float ip:%s] failed, details: %s " % ( + self.ip, + str(_err), + ) self.handle_error_msg(err_msg) LOG.info("Login DM success.") @@ -159,21 +164,29 @@ class StorageInf(object): :return: None """ LOG.info("Query storage pool info start, pool_id:[%s]", pool_id) - url = Constant.QUERY_POOL.format(deviceId=self.rest_client.device_id) + f"?filter=ID:{pool_id}" + url = ( + Constant.QUERY_POOL.format(deviceId=self.rest_client.device_id) + + f"?filter=ID:{pool_id}" + ) res = self.rest_client.normal_request(url, "get") err_msg = f"Failed to query pool id:[{pool_id}]" resp_data = self.result_parse(err_msg, res) return resp_data def query_vstore_count(self, vstore_id): - url = Constant.QUERY_VSTORE.format(deviceId=self.rest_client.device_id) + f"?filter=ID:{vstore_id}" + url = ( + Constant.QUERY_VSTORE.format(deviceId=self.rest_client.device_id) + + f"?filter=ID:{vstore_id}" + ) res = self.rest_client.normal_request(url, "get") err_msg = f"Failed to query vstore id:[{vstore_id}]" rsp_data = self.result_parse(err_msg, res) return rsp_data def query_vstore_info(self, vstore_id): - url = Constant.DELETE_VSTORE.format(deviceId=self.rest_client.device_id, id=vstore_id) + url = Constant.DELETE_VSTORE.format( + deviceId=self.rest_client.device_id, id=vstore_id + ) res = self.rest_client.normal_request(url, "get") err_msg = f"Failed to query vstore info:[{vstore_id}]" rsp_data = self.result_parse(err_msg, res) @@ -193,10 +206,11 @@ class StorageInf(object): :param vstore_id: 租户id :return: rollback_rate：回顾进度， rollback_status：回滚状态 """ - query_url = Constant.QUERY_ROLLBACK_SNAPSHOT_PROCESS.format(deviceId=self.rest_client.device_id, - fs_name=fs_name) + query_url = Constant.QUERY_ROLLBACK_SNAPSHOT_PROCESS.format( + deviceId=self.rest_client.device_id, fs_name=fs_name + ) url = query_url + f"&&vstoreId={vstore_id}" - res = self.rest_client.normal_request(url, 'get') + res = self.rest_client.normal_request(url, "get") err_msg = f"Failed to query fs[{fs_name}] rollback snapshot process." rsp_data = self.result_parse(err_msg, res) return rsp_data @@ -244,10 +258,17 @@ class StorageInf(object): :param vstore_id: 租户ID :return: """ - query_url = Constant.NFS_SHARE_ADD_CLIENT.format(deviceId=self.rest_client.device_id) - url = query_url + f"?filter=PARENTID:{nfs_share_id}&vstoreId={vstore_id}&range=[0-1]" + query_url = Constant.NFS_SHARE_ADD_CLIENT.format( + deviceId=self.rest_client.device_id + ) + url = ( + query_url + + f"?filter=PARENTID:{nfs_share_id}&vstoreId={vstore_id}&range=[0-1]" + ) res = self.rest_client.normal_request(url, "get") - err_msg = f"Failed to query nfs share auth client, nfs_share_id:%s" % nfs_share_id + err_msg = ( + f"Failed to query nfs share auth client, nfs_share_id:%s" % nfs_share_id + ) return self.result_parse(err_msg, res) def query_logical_port_info(self, ip_addr, vstore_id=None): @@ -257,9 +278,14 @@ class StorageInf(object): :param vstore_id: 租户id :return: """ - query_url = Constant.QUERY_LOGIC_PORT_INFO.format(deviceId=self.rest_client.device_id) + query_url = Constant.QUERY_LOGIC_PORT_INFO.format( + deviceId=self.rest_client.device_id + ) if vstore_id: - url = query_url + f"?filter=IPV4ADDR:{ip_addr}&range=[0-100]&vstoreId={vstore_id}" + url = ( + query_url + + f"?filter=IPV4ADDR:{ip_addr}&range=[0-100]&vstoreId={vstore_id}" + ) else: url = query_url + f"?filter=IPV4ADDR:{ip_addr}&range=[0-100]" res = self.rest_client.normal_request(url, "get") @@ -272,7 +298,9 @@ class StorageInf(object): :param vstore_id: 租户id, 默认是系统租户 :return: """ - query_url = Constant.QUERY_LOGIC_PORT_INFO.format(deviceId=self.rest_client.device_id) + query_url = Constant.QUERY_LOGIC_PORT_INFO.format( + deviceId=self.rest_client.device_id + ) url = query_url + f"?vstoreId={vstore_id}" res = self.rest_client.normal_request(url, "get") err_msg = f"Failed to query lf info" @@ -305,12 +333,17 @@ class StorageInf(object): :param vstore_id: :return: 克隆文件系统ID """ - LOG.info("Begin to clone fs id[%s], new fs[%s], vstore id [%s]", parent_id, clone_fs_name, vstore_id) + LOG.info( + "Begin to clone fs id[%s], new fs[%s], vstore id [%s]", + parent_id, + clone_fs_name, + vstore_id, + ) url = Constant.CREATE_CLONE_FS.format(deviceId=self.rest_client.device_id) data = { "NAME": clone_fs_name, "PARENTFILESYSTEMID": parent_id, - "vstoreId": vstore_id + "vstoreId": vstore_id, } res = self.rest_client.normal_request(url, "post", data=data) err_msg = f"Failed to clone fs[id:{parent_id}], clone fs name:{clone_fs_name} " @@ -327,10 +360,10 @@ class StorageInf(object): :return: """ data = { - 'NAME': name, - 'PARENTID': fs_id, - 'vstoreId': vstore_id, - 'PARENTTYPE': 40 + "NAME": name, + "PARENTID": fs_id, + "vstoreId": vstore_id, + "PARENTTYPE": 40, } url = Constant.CREATE_FSSNAPSHOT.format(deviceId=self.rest_client.device_id) res = self.rest_client.normal_request(url, "post", data=data) @@ -348,11 +381,7 @@ class StorageInf(object): """ LOG.info("Begin to split clone fs[id:%s]", clone_fs_id) url = Constant.SPLIT_CLONE_FS.format(deviceId=self.rest_client.device_id) - data = { - "ID": clone_fs_id, - "action": action, - "vstoreId": vstore_id - } + data = {"ID": clone_fs_id, "action": action, "vstoreId": vstore_id} res = self.rest_client.normal_request(url, "put", data=data) err_msg = f"Failed to split clone fs[id:{clone_fs_id}]" self.result_parse(err_msg, res) @@ -365,10 +394,7 @@ class StorageInf(object): :param vstore_id: 租户ID :return: """ - data = { - 'ID': snapshot_id, - 'vstoreId': vstore_id - } + data = {"ID": snapshot_id, "vstoreId": vstore_id} url = Constant.ROLLBACK_SNAPSHOT.format(deviceId=self.rest_client.device_id) res = self.rest_client.normal_request(url, "put", data=data) err_msg = f"Failed to rollback snapshot, snapshot_id:[%s]" % snapshot_id @@ -381,8 +407,9 @@ class StorageInf(object): :param snapshot_id: 快照ID :return: """ - url = (Constant.CREATE_FSSNAPSHOT + "/{id}").format(deviceId=self.rest_client.device_id, - id=snapshot_id) + url = (Constant.CREATE_FSSNAPSHOT + "/{id}").format( + deviceId=self.rest_client.device_id, id=snapshot_id + ) res = self.rest_client.normal_request(url, "get") err_msg = f"Failed to query snapshot info, snapshot_id:[%s]" % snapshot_id rsp_data = self.result_parse(err_msg, res) @@ -394,8 +421,9 @@ class StorageInf(object): :param snapshot_id: 快照ID :return: """ - url = (Constant.CREATE_FSSNAPSHOT + "/{id}").format(deviceId=self.rest_client.device_id, - id=snapshot_id) + url = (Constant.CREATE_FSSNAPSHOT + "/{id}").format( + deviceId=self.rest_client.device_id, id=snapshot_id + ) res = self.rest_client.normal_request(url, "delete") err_msg = f"Failed to delete snapshot, snapshot_id:[%s]" % snapshot_id rsp_data = self.result_parse(err_msg, res) @@ -431,10 +459,9 @@ class StorageInf(object): "NAME": 客户端IP或主机名或网络组名。192.168.0.10或192.168.0.0/24或*，“*”表示全部客户端IP地址 :return: nfs client id """ - url = Constant.NFS_SHARE_ADD_CLIENT.\ - format(deviceId=self.rest_client.device_id) + url = Constant.NFS_SHARE_ADD_CLIENT.format(deviceId=self.rest_client.device_id) res = self.rest_client.normal_request(url, "post", data=data) - err_msg = f'Failed to create nfs share, data:{data}' + err_msg = f"Failed to create nfs share, data:{data}" rsp_data = self.result_parse(err_msg, res) LOG.info("Add nfs client success, data:%s", data) return rsp_data.get("ID") @@ -457,11 +484,7 @@ class StorageInf(object): """ LOG.info("Begin to open nfs 4.0 and 4.1 configer of vstore[%s]", vstore_id) url = Constant.NFS_SERVICE.format(deviceId=self.rest_client.device_id) - data = { - "vstoreId": vstore_id, - "SUPPORTV4": True, - "SUPPORTV41": True - } + data = {"vstoreId": vstore_id, "SUPPORTV4": True, "SUPPORTV41": True} res = self.rest_client.normal_request(url, "put", data=data) err_msg = f"Failed to open vstore{vstore_id} nfs service" self.result_parse(err_msg, res) @@ -494,8 +517,9 @@ class StorageInf(object): :return: None """ LOG.info("Begin to del nfs share, id[%s]", nfs_share_id) - del_share_url = Constant.NFS_SHARE_DELETE.\ - format(deviceId=self.rest_client.device_id, id=nfs_share_id) + del_share_url = Constant.NFS_SHARE_DELETE.format( + deviceId=self.rest_client.device_id, id=nfs_share_id + ) url = del_share_url + f"?vstoreId={vstore_id}" res = self.rest_client.normal_request(url, "delete") err_msg = f"Failed to delete {nfs_share_id} nfs share" @@ -509,8 +533,7 @@ class StorageInf(object): :return: """ LOG.info("Begin to del fs by id[%s]", fs_id) - url = Constant.DELETE_FS.\ - format(deviceId=self.rest_client.device_id, id=fs_id) + url = Constant.DELETE_FS.format(deviceId=self.rest_client.device_id, id=fs_id) res = self.rest_client.normal_request(url, "delete") err_msg = f"Failed to delete {fs_id} fs" self.result_parse(err_msg, res) @@ -526,12 +549,14 @@ class StorageInf(object): :return: """ LOG.info("Begin to delete cdp schedule by id[%s]", cdp_id) - url = Constant.DELETE_FS_CDP_SCHEDULE.format(deviceId=self.rest_client.device_id) + url = Constant.DELETE_FS_CDP_SCHEDULE.format( + deviceId=self.rest_client.device_id + ) data = { - "ID": fs_id, - "TIMINGSNAPSHOTSCHEDULEID": cdp_id, - "scheduleName": cdp_name, - "vstoreId": vstore_id + "ID": fs_id, + "TIMINGSNAPSHOTSCHEDULEID": cdp_id, + "scheduleName": cdp_name, + "vstoreId": vstore_id, } res = self.rest_client.normal_request(url, "delete", data=data) err_msg = f"Failed to delete {cdp_id} cdp schedule" diff --git a/pkg/deploy/action/logicrep/logicrep_ctl.py b/pkg/deploy/action/logicrep/logicrep_ctl.py index ad1c5b45..eeed8a40 100644 --- a/pkg/deploy/action/logicrep/logicrep_ctl.py +++ b/pkg/deploy/action/logicrep/logicrep_ctl.py @@ -10,13 +10,24 @@ import glob from get_config_info import get_value from logging import handlers -sys.path.append(os.path.join(os.path.dirname(os.path.abspath(__file__)), "..", - "inspection", "inspection_scripts", "ct_om")) - -sys.path.append(os.path.join(os.path.dirname(os.path.abspath(__file__)), "..", "dbstor")) +sys.path.append( + os.path.join( + os.path.dirname(os.path.abspath(__file__)), + "..", + "inspection", + "inspection_scripts", + "ct_om", + ) +) + +sys.path.append( + os.path.join(os.path.dirname(os.path.abspath(__file__)), "..", "dbstor") +) from kmc_adapter import CApiWrapper -sys.path.append(os.path.join(os.path.dirname(os.path.abspath(__file__)), "..", "cantian")) +sys.path.append( + os.path.join(os.path.dirname(os.path.abspath(__file__)), "..", "cantian") +) from Common import DefaultValue CMD_CREATE_LREP = f"CREATE USER LREP IDENTIFIED BY '%s'; \ @@ -63,7 +74,7 @@ CMD_CHECK_OPEN = "SELECT LREP_MODE FROM SYS.DV_DATABASE;" CMD_CHECK_ACTIVE = "SELECT * FROM DV_LRPL_DETAIL;" DV_LRPL_DETAIL = "select * from DV_LRPL_DETAIL;" CURRENT_PATH = os.path.dirname(os.path.abspath(__file__)) -CTSQL_INI_PATH = r'/mnt/dbdata/local/cantian/tmp/data/cfg/*sql.ini' +CTSQL_INI_PATH = r"/mnt/dbdata/local/cantian/tmp/data/cfg/*sql.ini" PRIMARY_KEYSTORE = r"/opt/cantian/common/config/primary_keystore_bak.ks" STANDBY_KEYSTORE = r"/opt/cantian/common/config/standby_keystore_bak.ks" CANTIAN_CONFIG = os.path.join(CURRENT_PATH, "..", "..", "config", "deploy_param.json") @@ -81,9 +92,7 @@ def setup(): for handler in list(log.handlers): log.removeHandler(handler) - file_log = handlers.RotatingFileHandler( - LOG_FILE, maxBytes=6291456, - backupCount=5) + file_log = handlers.RotatingFileHandler(LOG_FILE, maxBytes=6291456, backupCount=5) log.addHandler(file_log) log.addHandler(console) @@ -91,8 +100,10 @@ def setup(): handler.setFormatter( logging.Formatter( fmt="%(asctime)s %(levelname)s [pid:%(process)d] [%(threadName)s]" - " [tid:%(thread)d] [%(filename)s:%(lineno)d %(funcName)s] %(message)s", - datefmt="%Y-%m-%d %H:%M:%S")) + " [tid:%(thread)d] [%(filename)s:%(lineno)d %(funcName)s] %(message)s", + datefmt="%Y-%m-%d %H:%M:%S", + ) + ) log.setLevel(logging.INFO) return log @@ -101,12 +112,12 @@ LOG = setup() def file_reader(file_path): - with open(file_path, 'r', encoding="utf-8") as file: + with open(file_path, "r", encoding="utf-8") as file: return file.read() def file_lines_reader(file_path): - with open(file_path, 'r', encoding="utf-8") as file: + with open(file_path, "r", encoding="utf-8") as file: return file.readlines() @@ -138,7 +149,9 @@ class Logicrep: self.mode = mode @staticmethod - def kmc_resovle_password(mode, plain_text, key1=PRIMARY_KEYSTORE, key2=STANDBY_KEYSTORE): + def kmc_resovle_password( + mode, plain_text, key1=PRIMARY_KEYSTORE, key2=STANDBY_KEYSTORE + ): kmc_adapter = CApiWrapper(primary_keystore=key1, standby_keystore=key2) kmc_adapter.initialize() try: @@ -147,11 +160,17 @@ class Logicrep: if mode == "decrypted": ret_pwd = kmc_adapter.decrypt(plain_text) except Exception as error: - raise Exception("Failed to %s password. output: %s" % (mode, error)) from error - - split_env = os.environ['LD_LIBRARY_PATH'].split(":") - filtered_env = [single_env for single_env in split_env if "/opt/software/tools/logicrep/lib" not in single_env] - os.environ['LD_LIBRARY_PATH'] = ":".join(filtered_env) + raise Exception( + "Failed to %s password. output: %s" % (mode, error) + ) from error + + split_env = os.environ["LD_LIBRARY_PATH"].split(":") + filtered_env = [ + single_env + for single_env in split_env + if "/opt/software/tools/logicrep/lib" not in single_env + ] + os.environ["LD_LIBRARY_PATH"] = ":".join(filtered_env) kmc_adapter.finalize() return ret_pwd @@ -170,12 +189,16 @@ class Logicrep: try: import cantian_om_logicrep_check except Exception as error: - raise Exception(f"import inspection failed. Error info : {str(error)}") from error + raise Exception( + f"import inspection failed. Error info : {str(error)}" + ) from error far = cantian_om_logicrep_check.LogicrepChecker() result = far.get_format_output() - if result['error']['code'] == -1: - raise Exception(f"check failed.Error info : {result['error']['description']}") + if result["error"]["code"] == -1: + raise Exception( + f"check failed.Error info : {result['error']['description']}" + ) LOG.info("logicrep check success") # 获取用户信息 @@ -189,7 +212,7 @@ class Logicrep: self.deploy_mode = "nas" if info.get("deploy_mode") == "file" else "dbstore" ctsql_file = glob.glob(CTSQL_INI_PATH)[0] ctsql_ini_data = file_reader(ctsql_file) - encrypt_pwd = ctsql_ini_data[ctsql_ini_data.find('=') + 1:].strip() + encrypt_pwd = ctsql_ini_data[ctsql_ini_data.find("=") + 1 :].strip() self.passwd = self.kmc_resovle_password("decrypted", encrypt_pwd) else: info_list = file_lines_reader(self.conf_file) @@ -197,10 +220,12 @@ class Logicrep: if "ds.username=" in line: self.logicrep_user = line[12:].strip() if "ds.passwd=" in line: - self.passwd = self.kmc_resovle_password("decrypted", - line[10:].strip(), - key1=self.key1_file, - key2=self.key2_file) + self.passwd = self.kmc_resovle_password( + "decrypted", + line[10:].strip(), + key1=self.key1_file, + key2=self.key2_file, + ) break if not self.passwd: raise Exception("get password failed") @@ -226,15 +251,17 @@ class Logicrep: if "PRIMARY" in stdout_data: LOG.info("Current mode is primary") return True - LOG.info("Current mode is standby, not allowed to log in to zsql to perform operations.") + LOG.info( + "Current mode is standby, not allowed to log in to zsql to perform operations." + ) return False def execute(self, sql, message): for i in range(RETRY_TIMES): - cmd = "source ~/.bashrc && echo -e '%s' | ctsql sys@127.0.0.1:%s -q -c \"%s\"" % ( - self.passwd, - self.lsnr_port, - sql) + cmd = ( + "source ~/.bashrc && echo -e '%s' | ctsql sys@127.0.0.1:%s -q -c \"%s\"" + % (self.passwd, self.lsnr_port, sql) + ) return_code, stdout_data, stderr_data = DefaultValue.exec_popen(cmd) output = "%s%s" % (str(stdout_data), str(stderr_data)) @@ -243,17 +270,22 @@ class Logicrep: # 数据库尚未启动完全 if re.match(".*CT-00827.*", result) or re.match(".*CT-00601.*", result): time.sleep(30) - LOG.info("Try to reconnect to the database, attempt:%s/%s", i + 1, RETRY_TIMES) + LOG.info( + "Try to reconnect to the database, attempt:%s/%s", + i + 1, + RETRY_TIMES, + ) continue # 创建失败：改用户已经存在 if re.match(".*CT-00753.*", result): - raise USEREXIST("%s already exist,please choose another name" % self.logicrep_user) + raise USEREXIST( + "%s already exist,please choose another name" % self.logicrep_user + ) if self.passwd in output: output = "execute ctsql failed" if return_code: - raise Exception("Failed to %s by sql, output:%s" - % (message, output)) + raise Exception("Failed to %s by sql, output:%s" % (message, output)) # return code is 0, but output has error info, CT-xxx, ZS-xxx if re.match(".*ZS-00001.*", result): @@ -266,13 +298,24 @@ class Logicrep: raise Exception("Execute sql timeout.") def create_db_user(self): - self.execute_sql(CMD_CREATE_LREP.replace("LREP", self.logicrep_user) - % self.passwd, f"create {self.logicrep_user}") - self.execute_sql(CMD_GRANT.replace("LREP", self.logicrep_user), f"create {self.logicrep_user}") - self.execute_sql(CMD_CREATE_PROFILE.replace("LREP", self.logicrep_user), f"create {self.logicrep_user}") + self.execute_sql( + CMD_CREATE_LREP.replace("LREP", self.logicrep_user) % self.passwd, + f"create {self.logicrep_user}", + ) + self.execute_sql( + CMD_GRANT.replace("LREP", self.logicrep_user), + f"create {self.logicrep_user}", + ) + self.execute_sql( + CMD_CREATE_PROFILE.replace("LREP", self.logicrep_user), + f"create {self.logicrep_user}", + ) def set_resource_limit_true(self): - self.execute_sql(CMD_RESOURCE_LIMIT.replace("LREP", self.logicrep_user), "set resource limit >>>") + self.execute_sql( + CMD_RESOURCE_LIMIT.replace("LREP", self.logicrep_user), + "set resource limit >>>", + ) def update_init_properties(self): if self.node_id == "1": @@ -282,24 +325,27 @@ class Logicrep: info_list = file_lines_reader(self.init_conf_file) for i, line in enumerate(info_list): if "binlog.path=" in line: - info_list[i] = f"binlog.path=/mnt/dbdata/remote/archive_{self.storage_archive_fs}/\n" + info_list[i] = ( + f"binlog.path=/mnt/dbdata/remote/archive_{self.storage_archive_fs}/\n" + ) if "archive.path=" in line: - info_list[i] = f"archive.path=/mnt/dbdata/remote/archive_{self.storage_archive_fs}/\n" + info_list[i] = ( + f"archive.path=/mnt/dbdata/remote/archive_{self.storage_archive_fs}/\n" + ) if "deploy.mode" in line: info_list[i] = f"deploy.mode={self.deploy_mode}\n" - with os.fdopen(os.open(self.init_conf_file, flags, modes), 'w') as fs: + with os.fdopen(os.open(self.init_conf_file, flags, modes), "w") as fs: fs.writelines(info_list) # 秘钥等信息写入 def write_key(self): LOG.info("begin to write key") - split_env = os.environ['LD_LIBRARY_PATH'].split(":") + split_env = os.environ["LD_LIBRARY_PATH"].split(":") split_env.append("/opt/software/tools/logicrep/lib") - os.environ['LD_LIBRARY_PATH'] = ":".join(split_env) - new_encrypt_pwd = self.kmc_resovle_password("encrypted", - self.passwd, - key1=self.key1_file, - key2=self.key2_file) + os.environ["LD_LIBRARY_PATH"] = ":".join(split_env) + new_encrypt_pwd = self.kmc_resovle_password( + "encrypted", self.passwd, key1=self.key1_file, key2=self.key2_file + ) url = "" for ip in self.cmsip: url += f"@{ip}:{self.lsnr_port}" @@ -314,7 +360,7 @@ class Logicrep: break flags = os.O_WRONLY | os.O_CREAT modes = stat.S_IWUSR | stat.S_IRUSR - with os.fdopen(os.open(self.conf_file, flags, modes), 'w') as fs: + with os.fdopen(os.open(self.conf_file, flags, modes), "w") as fs: fs.writelines(info_list) self.update_init_properties() @@ -385,16 +431,29 @@ class Logicrep: if self.status_check(): self.execute_sql(CMD_CLOSE, "stop logicrep") else: - LOG.error("turn logicrep switch off failed, because cantiand does not exist.") + LOG.error( + "turn logicrep switch off failed, because cantiand does not exist." + ) self.stop() # 参数解析 def main(): ctl_parse = argparse.ArgumentParser() - ctl_parse.add_argument("--act", type=str, - choices=["install", "init_container", "start", "startup", "stop", "shutdown", "pre_upgrade", - "set_resource_limit"]) + ctl_parse.add_argument( + "--act", + type=str, + choices=[ + "install", + "init_container", + "start", + "startup", + "stop", + "shutdown", + "pre_upgrade", + "set_resource_limit", + ], + ) ctl_parse.add_argument("--mode", required=False, dest="mode") arg = ctl_parse.parse_args() act = arg.act @@ -408,7 +467,7 @@ def main(): "stop": logicrep.stop, "shutdown": logicrep.shutdown, "pre_upgrade": logicrep.pre_upgrade, - "set_resource_limit": logicrep.set_resource_limit + "set_resource_limit": logicrep.set_resource_limit, } func_dict.get(act)() diff --git a/pkg/deploy/action/modify_env.py b/pkg/deploy/action/modify_env.py index 19e89853..f73d72e4 100644 --- a/pkg/deploy/action/modify_env.py +++ b/pkg/deploy/action/modify_env.py @@ -4,27 +4,29 @@ import os import pathlib import re import stat - - + + CUR_PATH = os.path.dirname(os.path.realpath(__file__)) sys.path.append(os.path.join(CUR_PATH, "../")) - + ENV_FILE = str(pathlib.Path(os.path.join(CUR_PATH, "env.sh"))) -CANTIAN_CONFIG = str(pathlib.Path(os.path.join(CUR_PATH, "cantian/install_config.json"))) - - +CANTIAN_CONFIG = str( + pathlib.Path(os.path.join(CUR_PATH, "cantian/install_config.json")) +) + + def read_file(file_path): - with open(file_path, 'r', encoding='utf8') as file_path: + with open(file_path, "r", encoding="utf8") as file_path: return file_path.read() - - + + def write_file(write_data, file_path): modes = stat.S_IRWXU | stat.S_IROTH | stat.S_IRGRP flag = os.O_RDWR | os.O_CREAT | os.O_TRUNC - with os.fdopen(os.open(file_path, flag, modes), 'w') as file_path: + with os.fdopen(os.open(file_path, flag, modes), "w") as file_path: file_path.write(write_data) - - + + def modify_env(): data = read_file(ENV_FILE) new_data = "" @@ -38,16 +40,16 @@ def modify_env(): new_line = re.sub(pattern_logicrep, "", new_line) new_data += new_line + "\n" write_file(new_data, ENV_FILE) - - + + def modify_cantian_config(): data = json.loads(read_file(CANTIAN_CONFIG)) if "USE_DBSTOR" in data.keys(): del data["USE_DBSTOR"] data = json.dumps(data, indent=4) write_file(data, CANTIAN_CONFIG) - - + + if __name__ == "__main__": modify_env() - modify_cantian_config() \ No newline at end of file + modify_cantian_config() diff --git a/pkg/deploy/action/mysql/mysqlctl.py b/pkg/deploy/action/mysql/mysqlctl.py index f9a34a17..c670e055 100644 --- a/pkg/deploy/action/mysql/mysqlctl.py +++ b/pkg/deploy/action/mysql/mysqlctl.py @@ -23,7 +23,9 @@ class MysqlCtl(object): if os.path.exists(INFO_SRC): with open(INFO_SRC, "r") as f: content = f.read() - version = re.findall(r"Cantian patch source ([0-9]+.[0-9]+.[0-9]+)", content) + version = re.findall( + r"Cantian patch source ([0-9]+.[0-9]+.[0-9]+)", content + ) if len(version) > 0: return version[0] return version @@ -37,9 +39,11 @@ class MysqlCtl(object): single_process = install_config.get("M_RUNING_MODE") cantian_in_container = get_value("cantian_in_container") mysql_metadata_in_cantian = get_value("mysql_metadata_in_cantian") - if (cantian_in_container != "0" - or not mysql_metadata_in_cantian - or single_process == "cantiand_in_cluster"): + if ( + cantian_in_container != "0" + or not mysql_metadata_in_cantian + or single_process == "cantiand_in_cluster" + ): return patch_version = self.get_patch_version() if patch_version != TARGET_VERSION: diff --git a/pkg/deploy/action/obtains_lsid.py b/pkg/deploy/action/obtains_lsid.py index c3a25824..367e2236 100644 --- a/pkg/deploy/action/obtains_lsid.py +++ b/pkg/deploy/action/obtains_lsid.py @@ -16,10 +16,7 @@ if os.path.exists(INSTALL_FILE): _tmp = f.read() info = json.loads(_tmp) else: - info = { - "cluster_id": "1", - "random_seed": "1" - } + info = {"cluster_id": "1", "random_seed": "1"} class LSIDGenerate(object): @@ -39,22 +36,31 @@ class LSIDGenerate(object): @staticmethod def generate_random_seed(): cluster_name = info.get("cluster_name", "") - hash_object = int(hashlib.sha256(cluster_name.encode('utf-8')).hexdigest(), 16) + hash_object = int(hashlib.sha256(cluster_name.encode("utf-8")).hexdigest(), 16) random.seed(hash_object) return random.randint(0, 255) def generate_lsid(self): # 返回lsid十六进制 - return int(str(bin(self.n_type))[2:].rjust(2, "0") - + str(bin(3))[2:].rjust(2, "0") - + str(bin(self.cluster_id))[2:].rjust(8, "0") - + str(bin(self.random_seed))[2:].rjust(8, "0") - + str(bin(self.process_id))[2:].rjust(4, "0") - + str(bin(self.node_id))[2:].rjust(8, "0"), 2) + return int( + str(bin(self.n_type))[2:].rjust(2, "0") + + str(bin(3))[2:].rjust(2, "0") + + str(bin(self.cluster_id))[2:].rjust(8, "0") + + str(bin(self.random_seed))[2:].rjust(8, "0") + + str(bin(self.process_id))[2:].rjust(4, "0") + + str(bin(self.node_id))[2:].rjust(8, "0"), + 2, + ) def execute(self): self.random_seed = self.generate_random_seed() - process_uuid = self.generate_uuid(self.n_type, self.cluster_id, self.random_seed, self.process_id, self.node_id) + process_uuid = self.generate_uuid( + self.n_type, + self.cluster_id, + self.random_seed, + self.process_id, + self.node_id, + ) ls_id = self.generate_lsid() return ls_id, process_uuid diff --git a/pkg/deploy/action/om_log.py b/pkg/deploy/action/om_log.py index db54c3e4..08725984 100644 --- a/pkg/deploy/action/om_log.py +++ b/pkg/deploy/action/om_log.py @@ -23,14 +23,30 @@ def _get_log_file_path(project): os.makedirs(logger_dir) return os.path.join(logger_dir, "{}.log".format(project)) - return '' + return "" SENSITIVE_STR = [ - 'Password', 'passWord', 'PASSWORD', 'password', 'Pswd', - 'PSWD', 'pwd', 'signature', 'HmacSHA256', 'newPasswd', - 'private', 'certfile', 'secret', 'token', 'Token', 'pswd', - 'passwd', 'mysql -u', 'session', 'cookie' + "Password", + "passWord", + "PASSWORD", + "password", + "Pswd", + "PSWD", + "pwd", + "signature", + "HmacSHA256", + "newPasswd", + "private", + "certfile", + "secret", + "token", + "Token", + "pswd", + "passwd", + "mysql -u", + "session", + "cookie", ] @@ -52,7 +68,7 @@ def setup(project_name, console_info=True): set_info = logging.INFO if console_info else logging.ERROR console = logging.StreamHandler() console.setLevel(set_info) - console_formatter = logging.Formatter('[%(levelname)s ] %(message)s') + console_formatter = logging.Formatter("[%(levelname)s ] %(message)s") console.setFormatter(console_formatter) log_root = logging.getLogger(project_name) @@ -61,12 +77,16 @@ def setup(project_name, console_info=True): log_path = _get_log_file_path(project_name) file_log = handlers.RotatingFileHandler( - log_path, maxBytes=log_config.get("log_file_max_size"), - backupCount=log_config.get("log_file_backup_count")) + log_path, + maxBytes=log_config.get("log_file_max_size"), + backupCount=log_config.get("log_file_backup_count"), + ) file_log.setFormatter( logging.Formatter( fmt=log_config.get("logging_context_format_string"), - datefmt=log_config.get("log_date_format"))) + datefmt=log_config.get("log_date_format"), + ) + ) log_root.addHandler(file_log) log_root.addHandler(console) log_root.addFilter(DefaultLogFilter()) diff --git a/pkg/deploy/action/om_log_config.py b/pkg/deploy/action/om_log_config.py index 0537fdc5..fe832388 100644 --- a/pkg/deploy/action/om_log_config.py +++ b/pkg/deploy/action/om_log_config.py @@ -7,9 +7,9 @@ CONSOLE_CONF = { "log_file_backup_count": 5, "log_date_format": "%Y-%m-%d %H:%M:%S", "logging_default_format_string": "%(asctime)s %(levelname)s [pid:%(process)d] [%(threadName)s] " - "[tid:%(thread)d] [%(filename)s:%(lineno)d %(funcName)s] %(message)s", + "[tid:%(thread)d] [%(filename)s:%(lineno)d %(funcName)s] %(message)s", "logging_context_format_string": "%(asctime)s %(levelname)s [pid:%(process)d] [%(threadName)s] " - "[tid:%(thread)d] [%(filename)s:%(lineno)d %(funcName)s] %(message)s" + "[tid:%(thread)d] [%(filename)s:%(lineno)d %(funcName)s] %(message)s", } } diff --git a/pkg/deploy/action/pre_install.py b/pkg/deploy/action/pre_install.py index fbf70c92..4207c399 100644 --- a/pkg/deploy/action/pre_install.py +++ b/pkg/deploy/action/pre_install.py @@ -19,44 +19,54 @@ NEEDED_MEM_SIZE = 16 * 1024 # M dir_name, _ = os.path.split(os.path.abspath(__file__)) PKG_DIR = os.path.abspath(os.path.join(dir_name, "..")) -CANTIAN_MEM_SPEC_FILE = os.path.join(dir_name, "config", "container_conf", "init_conf", "mem_spec") +CANTIAN_MEM_SPEC_FILE = os.path.join( + dir_name, "config", "container_conf", "init_conf", "mem_spec" +) CANTIAND_INI_FILE = "/mnt/dbdata/local/cantian/tmp/data/cfg/cantiand.ini" SINGLE_DOUBLE_PROCESS_MAP = { "0": "cantiand_in_cluster", - "1": "cantiand_with_mysql_in_cluster" + "1": "cantiand_with_mysql_in_cluster", } -ip_check_element = { - 'cantian_vlan_ip', - 'storage_vlan_ip', - 'cms_ip' -} +ip_check_element = {"cantian_vlan_ip", "storage_vlan_ip", "cms_ip"} ping_check_element = { - 'cantian_vlan_ip', - 'storage_vlan_ip', - 'cms_ip', - 'share_logic_ip', - 'archive_logic_ip', - 'metadata_logic_ip', - 'storage_logic_ip' + "cantian_vlan_ip", + "storage_vlan_ip", + "cms_ip", + "share_logic_ip", + "archive_logic_ip", + "metadata_logic_ip", + "storage_logic_ip", } kernel_element = { - 'TEMP_BUFFER_SIZE', - 'DATA_BUFFER_SIZE', - 'SHARED_POOL_SIZE', - 'LOG_BUFFER_SIZE', - 'SESSIONS', - 'VARIANT_MEMORY_AREA_SIZE', - '_INDEX_BUFFER_SIZE' + "TEMP_BUFFER_SIZE", + "DATA_BUFFER_SIZE", + "SHARED_POOL_SIZE", + "LOG_BUFFER_SIZE", + "SESSIONS", + "VARIANT_MEMORY_AREA_SIZE", + "_INDEX_BUFFER_SIZE", } use_dbstor = ["dbstor", "combined"] -UnitConversionInfo = collections.namedtuple('UnitConversionInfo', ['tmp_gb', 'tmp_mb', 'tmp_kb', 'key', 'value', - 'sga_buff_size', 'temp_buffer_size', - 'data_buffer_size', 'shared_pool_size', - 'log_buffer_size']) +UnitConversionInfo = collections.namedtuple( + "UnitConversionInfo", + [ + "tmp_gb", + "tmp_mb", + "tmp_kb", + "key", + "value", + "sga_buff_size", + "temp_buffer_size", + "data_buffer_size", + "shared_pool_size", + "log_buffer_size", + ], +) + class ConfigChecker: """ @@ -66,7 +76,7 @@ class ConfigChecker: @staticmethod def node_id(value): - node_id_enum = {'0', '1'} + node_id_enum = {"0", "1"} if value not in node_id_enum: return False @@ -74,7 +84,7 @@ class ConfigChecker: @staticmethod def install_type(value): - install_type_enum = {'override', 'reserve'} + install_type_enum = {"override", "reserve"} if value not in install_type_enum: return False @@ -82,7 +92,7 @@ class ConfigChecker: @staticmethod def link_type(value): - link_type_enum = {'1', '0', '2'} # 1为rdma 0为tcp 2为rdma 1823 + link_type_enum = {"1", "0", "2"} # 1为rdma 0为tcp 2为rdma 1823 if value not in link_type_enum: return False @@ -90,7 +100,7 @@ class ConfigChecker: @staticmethod def db_type(value): - db_type_enum = {'0', '1', '2'} + db_type_enum = {"0", "1", "2"} if value not in db_type_enum: return False @@ -98,7 +108,7 @@ class ConfigChecker: @staticmethod def mysql_in_container(value): - mysql_in_container_enum = {'0', '1'} + mysql_in_container_enum = {"0", "1"} if value not in mysql_in_container_enum: return False @@ -119,13 +129,13 @@ class ConfigChecker: return False return True - + @staticmethod def cantian_in_container(value): - cantian_in_container_enum = {'0', '1', '2'} + cantian_in_container_enum = {"0", "1", "2"} if value not in cantian_in_container_enum: return False - + return True @staticmethod @@ -133,11 +143,11 @@ class ConfigChecker: try: value = int(value) except Exception as error: - LOG.error('cluster id type must be int : %s', str(error)) + LOG.error("cluster id type must be int : %s", str(error)) return False if value < 0 or value > 255: - LOG.error('cluster id cannot be less than 0 or more than 255') + LOG.error("cluster id cannot be less than 0 or more than 255") return False return True @@ -145,7 +155,7 @@ class ConfigChecker: @staticmethod def cluster_name(value): if len(value) > 64 or not value: - LOG.error('cluster name cannot be more than 64 or less than 1 in length') + LOG.error("cluster name cannot be more than 64 or less than 1 in length") return False return True @@ -154,7 +164,7 @@ class ConfigChecker: if value not in ["UC", "TCP", "UC_RDMA"]: return False return True - + @staticmethod def mes_ssl_switch(value): if not isinstance(value, bool): @@ -174,10 +184,10 @@ class ConfigChecker: if int(value) <= 0: return False except Exception as error: - LOG.error('redo_num type must be int : %s', str(error)) + LOG.error("redo_num type must be int : %s", str(error)) return False if int(value) < 3 or int(value) > 256: - LOG.error('redo_num cannot be less than 3 or more than 256') + LOG.error("redo_num cannot be less than 3 or more than 256") return False return True @@ -190,7 +200,7 @@ class ConfigChecker: if int(int_value) <= 0: return False except Exception as error: - LOG.error('redo_size type must be int : %s', str(error)) + LOG.error("redo_size type must be int : %s", str(error)) return False return True @@ -220,7 +230,7 @@ class ConfigChecker: value = int(value) except Exception as error: - LOG.error('dbstor_fs_vstore id type must be int : %s', str(error)) + LOG.error("dbstor_fs_vstore id type must be int : %s", str(error)) return False return True @@ -250,8 +260,10 @@ class CheckBase(metaclass=abc.ABCMeta): class CheckMem(CheckBase): def __init__(self): - super().__init__('memory available size smaller than {}M'.format(NEEDED_MEM_SIZE), - 'current memory size {}M'.format(self.get_mem_available())) + super().__init__( + "memory available size smaller than {}M".format(NEEDED_MEM_SIZE), + "current memory size {}M".format(self.get_mem_available()), + ) @staticmethod def get_mem_available(): @@ -260,15 +272,15 @@ class CheckMem(CheckBase): return:单位M """ res = 0 - with open('/proc/meminfo') as file_path: + with open("/proc/meminfo") as file_path: for line in file_path.readlines(): if "MemFree:" in line: - mem_free = line.split(':')[1].strip() + mem_free = line.split(":")[1].strip() mem_free = mem_free.split(" ")[0] res += int(mem_free) // 1024 if "MemAvailable" in line: - mem_avail = line.split(':')[1].strip() + mem_avail = line.split(":")[1].strip() mem_avail = mem_avail.split(" ")[0] res += int(mem_avail) // 1024 @@ -280,8 +292,10 @@ class CheckMem(CheckBase): class CheckDisk(CheckBase): def __init__(self): - super().__init__('disk capacity available size smaller than {}M'.format(NEEDED_SIZE), - 'current disk capacity {}M'.format(self.get_disk_available())) + super().__init__( + "disk capacity available size smaller than {}M".format(NEEDED_SIZE), + "current disk capacity {}M".format(self.get_disk_available()), + ) @staticmethod def find_dir_path(): @@ -319,26 +333,64 @@ class CheckInstallPath(CheckBase): class CheckInstallConfig(CheckBase): def __init__(self, config_path=None): - super().__init__("check config param", 'please check params in json file {}'.format(config_path)) + super().__init__( + "check config param", + "please check params in json file {}".format(config_path), + ) self.config_path = config_path self.value_checker = ConfigChecker self.config_key = { - 'deploy_user', 'node_id', 'cms_ip', 'storage_dbstor_fs', 'storage_share_fs', 'storage_archive_fs', - 'storage_metadata_fs', 'share_logic_ip', 'archive_logic_ip', 'metadata_logic_ip', 'db_type', - 'MAX_ARCH_FILES_SIZE', 'mysql_in_container', 'mysql_metadata_in_cantian', 'storage_logic_ip', 'deploy_mode', - 'mes_ssl_switch', 'cantian_in_container', 'deploy_policy', 'link_type', 'ca_path', 'crt_path', 'key_path' + "deploy_user", + "node_id", + "cms_ip", + "storage_dbstor_fs", + "storage_share_fs", + "storage_archive_fs", + "storage_metadata_fs", + "share_logic_ip", + "archive_logic_ip", + "metadata_logic_ip", + "db_type", + "MAX_ARCH_FILES_SIZE", + "mysql_in_container", + "mysql_metadata_in_cantian", + "storage_logic_ip", + "deploy_mode", + "mes_ssl_switch", + "cantian_in_container", + "deploy_policy", + "link_type", + "ca_path", + "crt_path", + "key_path", } self.dss_config_key = { - 'deploy_user', 'node_id', 'cms_ip', 'db_type', 'cantian_in_container', - 'MAX_ARCH_FILES_SIZE', 'mysql_in_container', 'mysql_metadata_in_cantian', - 'deploy_mode', 'mes_ssl_switch', "redo_num", "redo_size"} - self.dbstor_config_key = { - 'cluster_name', 'cantian_vlan_ip', 'storage_vlan_ip', 'link_type', 'storage_dbstor_page_fs', - 'kerberos_key', 'cluster_id', 'mes_type', "vstore_id", "dbstor_fs_vstore_id" + "deploy_user", + "node_id", + "cms_ip", + "db_type", + "cantian_in_container", + "MAX_ARCH_FILES_SIZE", + "mysql_in_container", + "mysql_metadata_in_cantian", + "deploy_mode", + "mes_ssl_switch", + "redo_num", + "redo_size", } - self.file_config_key = { - "redo_num", "redo_size" + self.dbstor_config_key = { + "cluster_name", + "cantian_vlan_ip", + "storage_vlan_ip", + "link_type", + "storage_dbstor_page_fs", + "kerberos_key", + "cluster_id", + "mes_type", + "vstore_id", + "dbstor_fs_vstore_id", } + self.file_config_key = {"redo_num", "redo_size"} self.mes_type_key = {"ca_path", "crt_path", "key_path"} self.config_params = {} self.cluster_name = None @@ -356,7 +408,7 @@ class CheckInstallConfig(CheckBase): socket.inet_aton(_ip) except socket.error: return False - return _ip.count('.') == 3 + return _ip.count(".") == 3 except socket.error: return False return True @@ -378,16 +430,30 @@ class CheckInstallConfig(CheckBase): process_list = [] for index, cmd in enumerate(cmd_list): if index == 0: - _p = subprocess.Popen(shlex.split(cmd), stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=False) + _p = subprocess.Popen( + shlex.split(cmd), + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + shell=False, + ) else: - _p = subprocess.Popen(shlex.split(cmd), stdin=process_list[index - 1].stdout, - stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=False) + _p = subprocess.Popen( + shlex.split(cmd), + stdin=process_list[index - 1].stdout, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + shell=False, + ) process_list.append(_p) try: stdout, stderr = process_list[-1].communicate(timeout=30) except Exception as err: return -1, str(err), -1 - return stdout.decode().strip("\n"), stderr.decode().strip("\n"), process_list[-1].returncode + return ( + stdout.decode().strip("\n"), + stderr.decode().strip("\n"), + process_list[-1].returncode, + ) @staticmethod def check_cantian_mem_spec(): @@ -395,8 +461,11 @@ class CheckInstallConfig(CheckBase): with open(CANTIAN_MEM_SPEC_FILE, encoding="utf-8") as f: mem_spec = json.load(f) if mem_spec not in ["0", "1", "2", "3"]: - LOG.error("Check mem spec failed, current value[%s], " \ - "value range [\"0\", \"1\", \"2\", \"3\"]", mem_spec) + LOG.error( + "Check mem spec failed, current value[%s], " + 'value range ["0", "1", "2", "3"]', + mem_spec, + ) return False return True @@ -421,19 +490,23 @@ class CheckInstallConfig(CheckBase): return False scale = 0 for i in range(len(cantian_vlan_ips)): - scale += len(cantian_vlan_ips[i].split(";")) * len(storage_vlan_ips[i].split(";")) + scale += len(cantian_vlan_ips[i].split(";")) * len( + storage_vlan_ips[i].split(";") + ) if scale > 32: - LOG.error("cantian_vlan_ip and storage_vlan_ip scale should be less than 32.") + LOG.error( + "cantian_vlan_ip and storage_vlan_ip scale should be less than 32." + ) return False return True def read_install_config(self): try: - with open(self.config_path, 'r', encoding='utf8') as file_path: + with open(self.config_path, "r", encoding="utf8") as file_path: json_data = json.load(file_path) return json_data except Exception as error: - LOG.error('load %s error, error: %s', self.config_path, str(error)) + LOG.error("load %s error, error: %s", self.config_path, str(error)) return {} @@ -443,15 +516,30 @@ class CheckInstallConfig(CheckBase): # 如果 config_key中存在的关键字install_config.json中没有，报错。 for element in not_in_either: # 去nas忽略部分参数 - dbstor_ignore_params = {"storage_metadata_fs", "share_logic_ip", "archive_logic_ip", "metadata_logic_ip", - "vstore_id", "kerberos_key", "ca_path", "crt_path", "key_path"} + dbstor_ignore_params = { + "storage_metadata_fs", + "share_logic_ip", + "archive_logic_ip", + "metadata_logic_ip", + "vstore_id", + "kerberos_key", + "ca_path", + "crt_path", + "key_path", + } combined_ignore_params = {"share_logic_ip", "vstore_id"} - if element in dbstor_ignore_params and install_config['deploy_mode'] == "dbstor": + if ( + element in dbstor_ignore_params + and install_config["deploy_mode"] == "dbstor" + ): continue - if element in combined_ignore_params and install_config['deploy_mode'] == "combined": + if ( + element in combined_ignore_params + and install_config["deploy_mode"] == "combined" + ): continue if element not in install_config_keys: - LOG.error('config_params.json need param %s', element) + LOG.error("config_params.json need param %s", element) return False return True @@ -491,30 +579,50 @@ class CheckInstallConfig(CheckBase): def write_result_to_json(self): modes = stat.S_IRWXU | stat.S_IROTH | stat.S_IRGRP flag = os.O_RDWR | os.O_CREAT | os.O_TRUNC - with os.fdopen(os.open(str(Path('{}/deploy_param.json'.format(dir_name))), flag, modes), 'w') as file_path: + with os.fdopen( + os.open(str(Path("{}/deploy_param.json".format(dir_name))), flag, modes), + "w", + ) as file_path: config_params = json.dumps(self.config_params, indent=4) file_path.write(config_params) def update_config_params(self): # 使用域名部署场景，share_logic_ip、archive_logic_ip、metadata_logic_ip为空时需要更新字段为cluster_name - if self.config_params.get("share_logic_ip") == "" and \ - self.config_params.get("archive_logic_ip") == "" and \ - self.config_params.get("metadata_logic_ip") == "": - self.config_params["share_logic_ip"] = self.config_params.get("cluster_name") - self.config_params["archive_logic_ip"] = self.config_params.get("cluster_name") - self.config_params["metadata_logic_ip"] = self.config_params.get("cluster_name") + if ( + self.config_params.get("share_logic_ip") == "" + and self.config_params.get("archive_logic_ip") == "" + and self.config_params.get("metadata_logic_ip") == "" + ): + self.config_params["share_logic_ip"] = self.config_params.get( + "cluster_name" + ) + self.config_params["archive_logic_ip"] = self.config_params.get( + "cluster_name" + ) + self.config_params["metadata_logic_ip"] = self.config_params.get( + "cluster_name" + ) modes = stat.S_IRWXU | stat.S_IROTH | stat.S_IRGRP flag = os.O_RDWR | os.O_CREAT | os.O_TRUNC config_params = json.dumps(self.config_params, indent=4) - with os.fdopen(os.open(self.config_path, flag, modes), 'w') as file_path: + with os.fdopen(os.open(self.config_path, flag, modes), "w") as file_path: file_path.write(config_params) def do_unit_conversion(self, get_unit_conversion_info): - tmp_gb, tmp_mb, tmb_kb, key, value,\ - sga_buff_size, temp_buffer_size, data_buffer_size,\ - shared_pool_size, log_buffer_size = get_unit_conversion_info - if value[0: -1].isdigit() and value[-1:] in ["G", "M", "K"]: + ( + tmp_gb, + tmp_mb, + tmb_kb, + key, + value, + sga_buff_size, + temp_buffer_size, + data_buffer_size, + shared_pool_size, + log_buffer_size, + ) = get_unit_conversion_info + if value[0:-1].isdigit() and value[-1:] in ["G", "M", "K"]: unit_map = { "G": tmp_gb, "M": tmp_mb, @@ -522,7 +630,7 @@ class CheckInstallConfig(CheckBase): } size_unit = unit_map.get(value[-1:]) sga_buff_size += int(value[0:-1]) * size_unit - + if key == "TEMP_BUFFER_SIZE": sga_buff_size -= temp_buffer_size if key == "DATA_BUFFER_SIZE": @@ -534,7 +642,7 @@ class CheckInstallConfig(CheckBase): if key == "SESSIONS": buff_size_pre_session = 5.5 * tmp_gb / 1024 sga_buff_size += int(value) * buff_size_pre_session - + return sga_buff_size def check_sga_buff_size(self): @@ -548,37 +656,53 @@ class CheckInstallConfig(CheckBase): shared_pool_size = 128 * tmp_mb data_buffer_size = 128 * tmp_mb temp_buffer_size = 32 * tmp_mb - sga_buff_size = (log_buffer_size + shared_pool_size + data_buffer_size + temp_buffer_size) + sga_buff_size = ( + log_buffer_size + shared_pool_size + data_buffer_size + temp_buffer_size + ) # parse the value of kernel parameters modes = stat.S_IWUSR | stat.S_IRUSR flags = os.O_RDONLY - with os.fdopen(os.open(CANTIAND_INI_FILE, flags, modes), 'r') as fp: + with os.fdopen(os.open(CANTIAND_INI_FILE, flags, modes), "r") as fp: for line in fp: if line == "\n": continue (key, value) = line.split(" = ") if key in kernel_element: # Unit consersion - get_unit_conversion_info = UnitConversionInfo(tmp_gb, tmp_mb, tmp_kb, key, value.strip(), - sga_buff_size, temp_buffer_size, data_buffer_size, - shared_pool_size, log_buffer_size) + get_unit_conversion_info = UnitConversionInfo( + tmp_gb, + tmp_mb, + tmp_kb, + key, + value.strip(), + sga_buff_size, + temp_buffer_size, + data_buffer_size, + shared_pool_size, + log_buffer_size, + ) sga_buff_size = self.do_unit_conversion(get_unit_conversion_info) - + # check sga buff size cmd = "cat /proc/meminfo |grep -wE 'MemFree:|Buffers:|Cached:|SwapCached' |awk '{sum += $2};END {print sum}'" ret_code, cur_avi_memory, stderr = exec_popen(cmd) if ret_code: - LOG.error("cannot get shmmax parameters, command: %s, err: %s" % (cmd, stderr)) + LOG.error( + "cannot get shmmax parameters, command: %s, err: %s" % (cmd, stderr) + ) return False if sga_buff_size < 114 * tmp_mb: LOG.error("sga buffer size should not less than 114MB, please check it!") return False - + # memory for Mysql, share memory, Dbstor, and CMS sga_buff_size += 12.2 * tmp_gb if int(sga_buff_size) > int(cur_avi_memory) * tmp_kb: - LOG.error("sga buffer size(%.2f GB) should less than availble memory(%.2f GB), please check it!" % (int(sga_buff_size) / tmp_gb, int(cur_avi_memory) / tmp_mb)) + LOG.error( + "sga buffer size(%.2f GB) should less than availble memory(%.2f GB), please check it!" + % (int(sga_buff_size) / tmp_gb, int(cur_avi_memory) / tmp_mb) + ) return False cmd = r"cat /proc/1/environ | tr '\0' '\n' | grep MY_MEMORY_SIZE | cut -d= -f2" @@ -587,18 +711,26 @@ class CheckInstallConfig(CheckBase): LOG.error("cannot get memory limit, command: %s, err: %s" % (cmd, stderr)) return False if container_memory_limit and (int(container_memory_limit) / tmp_gb) < 28: - LOG.error("container memory limit(%.2f GB) cannot be less than 28GB, please check it!" % (int(container_memory_limit) / tmp_gb)) + LOG.error( + "container memory limit(%.2f GB) cannot be less than 28GB, please check it!" + % (int(container_memory_limit) / tmp_gb) + ) return False if container_memory_limit and int(sga_buff_size) > int(container_memory_limit): - LOG.error("sga buffer size(%.2f GB) should less than container memory limit(%.2f GB), please check it!" % (int(sga_buff_size) / tmp_gb, int(container_memory_limit) / tmp_gb)) + LOG.error( + "sga buffer size(%.2f GB) should less than container memory limit(%.2f GB), please check it!" + % (int(sga_buff_size) / tmp_gb, int(container_memory_limit) / tmp_gb) + ) return False - + LOG.info("End check sga buffer size") return True def get_result(self, *args, **kwargs): if not self.config_path: - LOG.error('path of config file is not entered, example: sh install.sh xxx/xxx/xxx') + LOG.error( + "path of config file is not entered, example: sh install.sh xxx/xxx/xxx" + ) return False install_config_params = self.read_install_config() @@ -613,30 +745,32 @@ class CheckInstallConfig(CheckBase): self.cluster_name = install_config_params.get("cluster_name") - if install_config_params['deploy_mode'] in use_dbstor: + if install_config_params["deploy_mode"] in use_dbstor: self.config_key.remove("storage_logic_ip") self.config_key.update(self.dbstor_config_key) ping_check_element.remove("storage_logic_ip") - if install_config_params['deploy_mode'] == "dbstor": - if not install_config_params['mysql_metadata_in_cantian']: - LOG.error('Failed to check. deploy_mode is dbstor, mysql_metadata_in_cantian must be true') + if install_config_params["deploy_mode"] == "dbstor": + if not install_config_params["mysql_metadata_in_cantian"]: + LOG.error( + "Failed to check. deploy_mode is dbstor, mysql_metadata_in_cantian must be true" + ) return False ping_check_element.remove("share_logic_ip") - install_config_params['share_logic_ip'] = "127.0.0.1" + install_config_params["share_logic_ip"] = "127.0.0.1" # 去nas防止报错，后续版本删除 - install_config_params['archive_logic_ip'] = "127.0.0.1" - install_config_params['metadata_logic_ip'] = "127.0.0.1" + install_config_params["archive_logic_ip"] = "127.0.0.1" + install_config_params["metadata_logic_ip"] = "127.0.0.1" else: - self.config_params['cluster_id'] = "0" - self.config_params['mes_type'] = "TCP" + self.config_params["cluster_id"] = "0" + self.config_params["mes_type"] = "TCP" self.config_key.update(self.file_config_key) # 不开启归档时不检查归档连通性 if install_config_params.get("storage_archive_fs") == "": ping_check_element.remove("archive_logic_ip") - if install_config_params.get("cantian_in_container", "0") != '0': - ip_check_element.remove('cms_ip') + if install_config_params.get("cantian_in_container", "0") != "0": + ip_check_element.remove("cms_ip") ping_check_element.remove("cms_ip") ip_check_element.remove("cantian_vlan_ip") ping_check_element.remove("cantian_vlan_ip") @@ -650,43 +784,56 @@ class CheckInstallConfig(CheckBase): ping_check_element.remove("metadata_logic_ip") ping_check_element.remove("storage_logic_ip") - if (install_config_params.get("archive_logic_ip", "") == "" - and install_config_params.get('share_logic_ip', "") == "" - and install_config_params.get('metadata_logic_ip', "") == "" - and install_config_params['deploy_mode'] in use_dbstor): - install_config_params['archive_logic_ip'] = self.cluster_name - install_config_params['share_logic_ip'] = self.cluster_name - install_config_params['metadata_logic_ip'] = self.cluster_name - - max_arch_files_size = install_config_params.get('MAX_ARCH_FILES_SIZE', "") + if ( + install_config_params.get("archive_logic_ip", "") == "" + and install_config_params.get("share_logic_ip", "") == "" + and install_config_params.get("metadata_logic_ip", "") == "" + and install_config_params["deploy_mode"] in use_dbstor + ): + install_config_params["archive_logic_ip"] = self.cluster_name + install_config_params["share_logic_ip"] = self.cluster_name + install_config_params["metadata_logic_ip"] = self.cluster_name + + max_arch_files_size = install_config_params.get("MAX_ARCH_FILES_SIZE", "") if not max_arch_files_size: - install_config_params['MAX_ARCH_FILES_SIZE'] = '300G' + install_config_params["MAX_ARCH_FILES_SIZE"] = "300G" if not self.check_install_config_params(install_config_params): return False - if (install_config_params['deploy_mode'] in use_dbstor and - not self.check_storage_cantian_vlan_ip_scale(install_config_params)): + if install_config_params[ + "deploy_mode" + ] in use_dbstor and not self.check_storage_cantian_vlan_ip_scale( + install_config_params + ): return False for key, value in install_config_params.items(): - if not install_config_params.get("mes_ssl_switch") and key in self.mes_type_key: + if ( + not install_config_params.get("mes_ssl_switch") + and key in self.mes_type_key + ): continue if key in self.config_key: checked_result = self.check_install_config_param(key, value) if not checked_result: - LOG.error('check %s with value: %s failed', str(key), str(value)) + LOG.error("check %s with value: %s failed", str(key), str(value)) return False self.config_params[key] = value try: self.update_config_params() except Exception as error: - LOG.error('write config param to config_param.json failed, error: %s', str(error)) + LOG.error( + "write config param to config_param.json failed, error: %s", str(error) + ) return False - if install_config_params.get("'cantian_in_container'", "0") == '0': + if install_config_params.get("'cantian_in_container'", "0") == "0": try: self.write_result_to_json() except Exception as error: - LOG.error('write config param to deploy_param.json failed, error: %s', str(error)) + LOG.error( + "write config param to deploy_param.json failed, error: %s", + str(error), + ) return False return True @@ -702,51 +849,53 @@ class CheckInstallConfig(CheckBase): return True def install_config_params_init(self, install_config_params): - if 'link_type' not in install_config_params.keys(): - install_config_params['link_type'] = '1' - if 'storage_archive_fs' not in install_config_params.keys(): - install_config_params['storage_archive_fs'] = '' - if 'archive_logic_ip' not in install_config_params.keys(): - install_config_params['archive_logic_ip'] = '' - if 'mes_type' not in install_config_params.keys(): - install_config_params['mes_type'] = 'UC' - if 'mes_ssl_switch' not in install_config_params.keys(): - install_config_params['mes_ssl_switch'] = False - if 'deploy_mode' not in install_config_params.keys(): - install_config_params['deploy_mode'] = "combined" - if 'dbstor_fs_vstore_id' not in install_config_params.keys(): - install_config_params['dbstor_fs_vstore_id'] = "0" - if (install_config_params.get("mes_ssl_switch") and - install_config_params.get("cantian_in_container", "-1") == "0"): + if "link_type" not in install_config_params.keys(): + install_config_params["link_type"] = "1" + if "storage_archive_fs" not in install_config_params.keys(): + install_config_params["storage_archive_fs"] = "" + if "archive_logic_ip" not in install_config_params.keys(): + install_config_params["archive_logic_ip"] = "" + if "mes_type" not in install_config_params.keys(): + install_config_params["mes_type"] = "UC" + if "mes_ssl_switch" not in install_config_params.keys(): + install_config_params["mes_ssl_switch"] = False + if "deploy_mode" not in install_config_params.keys(): + install_config_params["deploy_mode"] = "combined" + if "dbstor_fs_vstore_id" not in install_config_params.keys(): + install_config_params["dbstor_fs_vstore_id"] = "0" + if ( + install_config_params.get("mes_ssl_switch") + and install_config_params.get("cantian_in_container", "-1") == "0" + ): self.config_key.update(self.mes_type_key) - if 'db_type' not in install_config_params.keys(): - install_config_params['db_type'] = '0' - if 'mysql_metadata_in_cantian' not in install_config_params.keys(): - install_config_params['mysql_metadata_in_cantian'] = True - if 'cantian_in_container' not in install_config_params.keys(): - install_config_params['cantian_in_container'] = "0" + if "db_type" not in install_config_params.keys(): + install_config_params["db_type"] = "0" + if "mysql_metadata_in_cantian" not in install_config_params.keys(): + install_config_params["mysql_metadata_in_cantian"] = True + if "cantian_in_container" not in install_config_params.keys(): + install_config_params["cantian_in_container"] = "0" def parse_policy_config_file(self): policy_path = os.path.join(dir_name, "deploy_policy_config.json") try: - with open(policy_path, 'r', encoding='utf8') as file_path: + with open(policy_path, "r", encoding="utf8") as file_path: json_data = json.load(file_path) return json_data except Exception as error: - LOG.error('load %s error, error: %s', policy_path, str(error)) + LOG.error("load %s error, error: %s", policy_path, str(error)) return False - + def parse_cantian_config_file(self): cantian_config_path = os.path.join(dir_name, "cantian") cantian_config_path = os.path.join(cantian_config_path, "install_config.json") try: - with open(cantian_config_path, 'r', encoding='utf8') as file_path: + with open(cantian_config_path, "r", encoding="utf8") as file_path: json_data = json.load(file_path) return json_data except Exception as error: - LOG.error('load %s error, error: %s', cantian_config_path, str(error)) + LOG.error("load %s error, error: %s", cantian_config_path, str(error)) return False - + def init_config_by_deploy_policy(self, install_config_params): deploy_policy_json = self.parse_policy_config_file() if deploy_policy_json is False: @@ -765,16 +914,18 @@ class CheckInstallConfig(CheckBase): # 如果未配置套餐参数，初始化套餐参数 install_config_params["deploy_policy"] = "default" return True - LOG.info("deploy policy is %s" % deploy_policy_key) + LOG.info("deploy policy is %s" % deploy_policy_key) # 如果配置方案未配置则返回失败，安装结束 deploy_policy_value = deploy_policy_json.get(deploy_policy_key, {}) if deploy_policy_value == {}: LOG.error("can not find the deploy policy(%s)" % deploy_policy_key) return False - + # 如果配置方案合法则将方案中的参数写入配置文件 is_single_process = deploy_policy_value.get("single_process", "0") - if SINGLE_DOUBLE_PROCESS_MAP.get(is_single_process, "") != cantian_config_json.get("M_RUNING_MODE"): + if SINGLE_DOUBLE_PROCESS_MAP.get( + is_single_process, "" + ) != cantian_config_json.get("M_RUNING_MODE"): LOG.error("The package type does not match the configuration parameters.") return False tmp_config = deploy_policy_value.get("config", {}) @@ -802,7 +953,11 @@ class PreInstall: check_config = CheckInstallConfig() res = check_config.sga_buffer_check() if not res: - LOG.error('failed: %s, suggestion: %s', check_config.check_name, check_config.suggestion) + LOG.error( + "failed: %s, suggestion: %s", + check_config.check_name, + check_config.suggestion, + ) return 1 return 0 @@ -827,19 +982,21 @@ class PreInstall: check_result = False if not check_result: - LOG.error('failed: %s, suggestion: %s', item().check_name, item().suggestion) + LOG.error( + "failed: %s, suggestion: %s", item().check_name, item().suggestion + ) return 1 return 0 -if __name__ == '__main__': +if __name__ == "__main__": config_file = None install_type = sys.argv[1] - if install_type == 'sga_buffer_check': + if install_type == "sga_buffer_check": exit(PreInstall.run_sga_buffer_check()) - elif install_type == 'override': + elif install_type == "override": config_file = sys.argv[2] pre_install = PreInstall(install_type, config_file) - exit(pre_install.check_main()) \ No newline at end of file + exit(pre_install.check_main()) diff --git a/pkg/deploy/action/pre_upgrade.py b/pkg/deploy/action/pre_upgrade.py index d1b2b717..f507ba06 100644 --- a/pkg/deploy/action/pre_upgrade.py +++ b/pkg/deploy/action/pre_upgrade.py @@ -7,26 +7,26 @@ from pre_install import CheckInstallConfig from om_log import LOGGER as LOG CUR_PATH = os.path.dirname(os.path.realpath(__file__)) -NEW_CONFIG_PATH = str(Path(f'{CUR_PATH}/config_params.json')) -SOURCE_CONFIG_PATH = str(Path('/opt/cantian/action/config_params.json')) -NEW_FILE_CONFIG_PATH = str(Path(f'{CUR_PATH}/config_params_file.json')) -SOURCE_FILE_CONFIG_PATH = str(Path('/opt/cantian/action/config_params_file.json')) +NEW_CONFIG_PATH = str(Path(f"{CUR_PATH}/config_params.json")) +SOURCE_CONFIG_PATH = str(Path("/opt/cantian/action/config_params.json")) +NEW_FILE_CONFIG_PATH = str(Path(f"{CUR_PATH}/config_params_file.json")) +SOURCE_FILE_CONFIG_PATH = str(Path("/opt/cantian/action/config_params_file.json")) DEPLOY_CONFIG = str(Path("/opt/cantian/config/deploy_param.json")) NEW_DEPLOY_CONFIG = str(Path(f"{CUR_PATH}/deploy_param.json")) def read_install_config(config_path): try: - with open(config_path, 'r', encoding='utf8') as file_path: + with open(config_path, "r", encoding="utf8") as file_path: json_data = json.load(file_path) return json_data except Exception as error: - LOG.error('load %s error, error: %s', config_path, str(error)) + LOG.error("load %s error, error: %s", config_path, str(error)) return {} -if __name__ == '__main__': +if __name__ == "__main__": # 如果没有指定文件，检查升级包的配置文件中key与源配置文件key是否一致 deploy_config = read_install_config(DEPLOY_CONFIG) deploy_mode = deploy_config.get("deploy_mode", "combined") @@ -35,10 +35,16 @@ if __name__ == '__main__': NEW_CONFIG_PATH = NEW_FILE_CONFIG_PATH if os.path.exists(SOURCE_FILE_CONFIG_PATH): SOURCE_CONFIG_PATH = SOURCE_FILE_CONFIG_PATH - new_config_keys = read_install_config(NEW_CONFIG_PATH).keys() - {"install_type", "cantian_in_container", - "auto_create_fs"} - source_config_keys = read_install_config(SOURCE_CONFIG_PATH).keys() - {"install_type", "cantian_in_container", - "auto_create_fs"} + new_config_keys = read_install_config(NEW_CONFIG_PATH).keys() - { + "install_type", + "cantian_in_container", + "auto_create_fs", + } + source_config_keys = read_install_config(SOURCE_CONFIG_PATH).keys() - { + "install_type", + "cantian_in_container", + "auto_create_fs", + } keys_diff = new_config_keys ^ source_config_keys if keys_diff: LOG.error(f"config keys are different with difference: {keys_diff}") @@ -55,7 +61,9 @@ if __name__ == '__main__': mysql_metadata_in_cantian = deploy_config.get("mysql_metadata_in_cantian") new_mysql_metadata_in_cantian = new_config.get("mysql_metadata_in_cantian") if new_mysql_metadata_in_cantian != mysql_metadata_in_cantian: - LOG.error("mysql_metadata_in_cantian is different from new config file, please check.") + LOG.error( + "mysql_metadata_in_cantian is different from new config file, please check." + ) sys.exit(1) if res: sys.exit(0) diff --git a/pkg/deploy/action/rest_client.py b/pkg/deploy/action/rest_client.py index ca4e51c6..bc9578fe 100644 --- a/pkg/deploy/action/rest_client.py +++ b/pkg/deploy/action/rest_client.py @@ -10,13 +10,17 @@ NORMAL_STATE, ABNORMAL_STATE = 0, 1 def get_cur_timestamp(): utc_now = datetime.utcnow() - return utc_now.replace(tzinfo=timezone.utc).astimezone(tz=None).strftime('%Y%m%d%H%M%S') + return ( + utc_now.replace(tzinfo=timezone.utc) + .astimezone(tz=None) + .strftime("%Y%m%d%H%M%S") + ) class RestElemConstant: - PORT = '8088' - HTTPS = 'https://' - LOGIN = '/deviceManager/rest/xxx/login' + PORT = "8088" + HTTPS = "https://" + LOGIN = "/deviceManager/rest/xxx/login" class ExecutionError(Exception): @@ -35,30 +39,33 @@ class RestClient: self.session = None self.res_login = None - self.handler = {'create': self.create_snapshots, - 'rollback': self.rollback_snapshots} + self.handler = { + "create": self.create_snapshots, + "rollback": self.rollback_snapshots, + } @staticmethod def gen_timestamp(): utc_now = datetime.utcnow() cur_time = utc_now.replace(tzinfo=timezone.utc).astimezone(tz=None) - return str(cur_time.strftime('%Y%m%d%H%M%S')) + return str(cur_time.strftime("%Y%m%d%H%M%S")) @staticmethod def exception_handler(err_msg=None, cur_mode=None): - err_info = '[current_mode] {}, [err_info] {}'.format(cur_mode, err_msg) + err_info = "[current_mode] {}, [err_info] {}".format(cur_mode, err_msg) LOG.error(err_info) raise ExecutionError(err_info) @staticmethod def response_parse(res_data): status_code = res_data.status_code - err_code, err_details = -1, 'failed' + err_code, err_details = -1, "failed" if status_code == 200: exec_res = res_data.json() - err_code, err_details = \ - exec_res.get('error').get('code'), exec_res.get('error').get('description') + err_code, err_details = exec_res.get("error").get("code"), exec_res.get( + "error" + ).get("description") return status_code, int(err_code), err_details @@ -69,42 +76,59 @@ class RestClient: return data def update_cookies(self, res): - res_body, set_cookie = res.json().get('data'), res.headers.get('Set-Cookie') + res_body, set_cookie = res.json().get("data"), res.headers.get("Set-Cookie") - self.token, self.device_id = res_body.get('iBaseToken'), res_body.get('deviceid') + self.token, self.device_id = res_body.get("iBaseToken"), res_body.get( + "deviceid" + ) - match_res = re.findall(r'session=ismsession=\w+;', set_cookie) + match_res = re.findall(r"session=ismsession=\w+;", set_cookie) if match_res: self.ism_session = match_res[0][:-1] - def make_header(self, content_type='application/json'): - header = {'Content-type': content_type} + def make_header(self, content_type="application/json"): + header = {"Content-type": content_type} if self.token: - header['iBaseToken'] = self.token + header["iBaseToken"] = self.token if self.ism_session: - header['Cookie'] = self.ism_session + header["Cookie"] = self.ism_session return header def login(self, fs_name, keep_session=False): - url = '{}{}:{}{}'.format(RestElemConstant.HTTPS, self.ip_addr, RestElemConstant.PORT, RestElemConstant.LOGIN) - user_info = {'username': self.user_name, - 'password': self.passwd, - 'scope': 0, - 'loginMode': 3, - 'timeConversion': 0, - 'isEncrypt': 'false'} - - login_header = {'Content-type': 'application/json', - 'Cookie': '__LANGUAGE_KEY__=zh-CN; __IBASE_LANGUAGE_KEY__=zh-CN'} + url = "{}{}:{}{}".format( + RestElemConstant.HTTPS, + self.ip_addr, + RestElemConstant.PORT, + RestElemConstant.LOGIN, + ) + user_info = { + "username": self.user_name, + "password": self.passwd, + "scope": 0, + "loginMode": 3, + "timeConversion": 0, + "isEncrypt": "false", + } + + login_header = { + "Content-type": "application/json", + "Cookie": "__LANGUAGE_KEY__=zh-CN; __IBASE_LANGUAGE_KEY__=zh-CN", + } requests.packages.urllib3.disable_warnings() with requests.session() as session: - res = session.post(url, data=json.dumps(user_info), headers=login_header, verify=False) + res = session.post( + url, data=json.dumps(user_info), headers=login_header, verify=False + ) status_code, err_code, err_details = self.response_parse(res) if err_code: - err_msg = ('Login {} failed before taking the snapshot of {}, status_code: {}, err_code: {}, ' - 'err_details: {}'.format(fs_name, self.ip_addr, status_code, err_code, err_details)) + err_msg = ( + "Login {} failed before taking the snapshot of {}, status_code: {}, err_code: {}, " + "err_details: {}".format( + fs_name, self.ip_addr, status_code, err_code, err_details + ) + ) return err_code, err_msg self.update_cookies(res) @@ -115,7 +139,7 @@ class RestClient: else: res.close() - return NORMAL_STATE, 'success' + return NORMAL_STATE, "success" def normal_request(self, url, method, data=None, **kwargs): requests.packages.urllib3.disable_warnings() @@ -123,39 +147,51 @@ class RestClient: if keep_session: req = self.session - self.token = self.res_login.get('data').get('ibasetoken') + self.token = self.res_login.get("data").get("ibasetoken") else: req = requests.session() headers = self.make_header() with req as session: - if method == 'put': - res = session.put(url, data=data, headers=headers, verify=False, timeout=timeout) - elif method == 'post': - res = session.post(url, data=data, headers=headers, verify=False, timeout=timeout) - elif method == 'get': - res = session.get(url, data=data, headers=headers, verify=False, timeout=timeout) - elif method == 'delete': - res = session.delete(url, data=data, headers=headers, verify=False, timeout=timeout) + if method == "put": + res = session.put( + url, data=data, headers=headers, verify=False, timeout=timeout + ) + elif method == "post": + res = session.post( + url, data=data, headers=headers, verify=False, timeout=timeout + ) + elif method == "get": + res = session.get( + url, data=data, headers=headers, verify=False, timeout=timeout + ) + elif method == "delete": + res = session.delete( + url, data=data, headers=headers, verify=False, timeout=timeout + ) res.close() return res def get_file_system_id(self, fs_name): - url = '{}{}:{}/deviceManager/rest/{}/filesystem?filter=NAME::{}'.format(RestElemConstant.HTTPS, - self.ip_addr, - RestElemConstant.PORT, - str(self.device_id), - fs_name) - res = self.normal_request(url, 'get') + url = "{}{}:{}/deviceManager/rest/{}/filesystem?filter=NAME::{}".format( + RestElemConstant.HTTPS, + self.ip_addr, + RestElemConstant.PORT, + str(self.device_id), + fs_name, + ) + res = self.normal_request(url, "get") status_code, err_code, err_details = self.response_parse(res) if err_code: - err_msg = 'Get file system id of {} failed, status_code: {}, err_code: {}, ' \ - 'err_details: {}'.format(fs_name, status_code, err_code, err_details) + err_msg = ( + "Get file system id of {} failed, status_code: {}, err_code: {}, " + "err_details: {}".format(fs_name, status_code, err_code, err_details) + ) return ABNORMAL_STATE, err_msg - file_system_id = res.json().get('data')[0].get('ID') + file_system_id = res.json().get("data")[0].get("ID") return NORMAL_STATE, file_system_id def create_snapshots(self, fs_name): @@ -164,25 +200,26 @@ class RestClient: work_state, res_details = self.get_file_system_id(fs_name) if work_state: - return self.exception_handler(err_msg=res_details, cur_mode='create') - - url = '{}{}:{}/deviceManager/rest/{}/fssnapshot'.format(RestElemConstant.HTTPS, - self.ip_addr, - RestElemConstant.PORT, - self.device_id) - reg_version = '{}_{}'.format(fs_name, self.upgrade_version).replace('.', '_') - snapshot_name = '{}_{}'.format(reg_version, self.gen_timestamp()) - data = {'NAME': snapshot_name, - 'PARENTID': int(res_details), - 'PARENTTYPE': 40} - res = self.normal_request(url, 'post', data=json.dumps(data)) + return self.exception_handler(err_msg=res_details, cur_mode="create") + + url = "{}{}:{}/deviceManager/rest/{}/fssnapshot".format( + RestElemConstant.HTTPS, self.ip_addr, RestElemConstant.PORT, self.device_id + ) + reg_version = "{}_{}".format(fs_name, self.upgrade_version).replace(".", "_") + snapshot_name = "{}_{}".format(reg_version, self.gen_timestamp()) + data = {"NAME": snapshot_name, "PARENTID": int(res_details), "PARENTTYPE": 40} + res = self.normal_request(url, "post", data=json.dumps(data)) status_code, err_code, err_details = self.response_parse(res) if err_code: - err_msg = 'Take snapshot of {} failed, status_code: {}, ' \ - 'err_code: {}, err_details: {}'.format(fs_name, status_code, err_code, err_details) - return self.exception_handler(err_msg=err_msg, cur_mode='create') - - snapshot_id = res.json().get('data').get('ID') + err_msg = ( + "Take snapshot of {} failed, status_code: {}, " + "err_code: {}, err_details: {}".format( + fs_name, status_code, err_code, err_details + ) + ) + return self.exception_handler(err_msg=err_msg, cur_mode="create") + + snapshot_id = res.json().get("data").get("ID") self.processed_fs[fs_name] = snapshot_id return NORMAL_STATE @@ -191,17 +228,20 @@ class RestClient: if not self.processed_fs.get(fs_name): return NORMAL_STATE - url = '{}{}:{}/deviceManager/rest/{}/fssnapshot/rollback_fssnapshot'.format(RestElemConstant.HTTPS, - self.ip_addr, - RestElemConstant.PORT, - self.device_id) - data = {'ID': self.processed_fs.get(fs_name)} - res = self.normal_request(url, 'put', data=json.dumps(data)) + url = "{}{}:{}/deviceManager/rest/{}/fssnapshot/rollback_fssnapshot".format( + RestElemConstant.HTTPS, self.ip_addr, RestElemConstant.PORT, self.device_id + ) + data = {"ID": self.processed_fs.get(fs_name)} + res = self.normal_request(url, "put", data=json.dumps(data)) status_code, err_code, err_details = self.response_parse(res) if err_code: - err_msg = 'Rollback snapshot of {} failed, status_code: {}, ' \ - 'err_code: {}, err_details: {}'.format(fs_name, status_code, err_code, err_details) - return self.exception_handler(err_msg=err_msg, cur_mode='rollback') + err_msg = ( + "Rollback snapshot of {} failed, status_code: {}, " + "err_code: {}, err_details: {}".format( + fs_name, status_code, err_code, err_details + ) + ) + return self.exception_handler(err_msg=err_msg, cur_mode="rollback") return NORMAL_STATE @@ -220,23 +260,34 @@ class RestClient: """ if not self.processed_fs.get(fs_name): return NORMAL_STATE, NORMAL_STATE - url = '{}{}:{}/deviceManager/rest/{}/FSSNAPSHOT/query_fs_snapshot_rollback?PARENTNAME={}'\ - .format(RestElemConstant.HTTPS, - self.ip_addr, - RestElemConstant.PORT, - self.device_id, - fs_name) - - res = self.normal_request(url, 'get') + url = "{}{}:{}/deviceManager/rest/{}/FSSNAPSHOT/query_fs_snapshot_rollback?PARENTNAME={}".format( + RestElemConstant.HTTPS, + self.ip_addr, + RestElemConstant.PORT, + self.device_id, + fs_name, + ) + + res = self.normal_request(url, "get") status_code, err_code, err_details = self.response_parse(res) if err_code: - err_msg = 'Query rollback snapshot process of {} failed, status_code: {}, ' \ - 'err_code: {}, err_details: {}'.format(fs_name, status_code, err_code, err_details) - return self.exception_handler(err_msg=err_msg, cur_mode='rollback') + err_msg = ( + "Query rollback snapshot process of {} failed, status_code: {}, " + "err_code: {}, err_details: {}".format( + fs_name, status_code, err_code, err_details + ) + ) + return self.exception_handler(err_msg=err_msg, cur_mode="rollback") data = self.get_data(res) - rollback_rate, rollback_status = data.get("rollbackRate"), data.get("rollbackStatus") - LOG.info("Rollback snapshot process of %s details:rollback_rate[%s] rollback_status[%s]", - fs_name, rollback_rate, rollback_status) + rollback_rate, rollback_status = data.get("rollbackRate"), data.get( + "rollbackStatus" + ) + LOG.info( + "Rollback snapshot process of %s details:rollback_rate[%s] rollback_status[%s]", + fs_name, + rollback_rate, + rollback_status, + ) return rollback_rate, rollback_status def execute(self, fs_name, mode): @@ -246,6 +297,8 @@ class RestClient: if not self.token: work_state, res_details = self.login(fs_name) if work_state: - return self.exception_handler(err_msg=res_details, cur_mode='get login token') + return self.exception_handler( + err_msg=res_details, cur_mode="get login token" + ) return self.handler.get(mode)(fs_name) diff --git a/pkg/deploy/action/storage_operate/create_file_system.py b/pkg/deploy/action/storage_operate/create_file_system.py index aadfca6a..0d117f05 100644 --- a/pkg/deploy/action/storage_operate/create_file_system.py +++ b/pkg/deploy/action/storage_operate/create_file_system.py @@ -23,27 +23,32 @@ DEPLOY_PARAM_PATH = str(pathlib.Path(CUR_PATH, "../../config/deploy_param.json") DEPLOY_PARAM = json.loads(read_helper(DEPLOY_PARAM_PATH)) DEPLOY_MODE = DEPLOY_PARAM.get("deploy_mode") FS_TYPE_LIST = [ - "storage_dbstor_fs", "storage_dbstor_page_fs", - "storage_share_fs", "storage_archive_fs", - "storage_metadata_fs" + "storage_dbstor_fs", + "storage_dbstor_page_fs", + "storage_share_fs", + "storage_archive_fs", + "storage_metadata_fs", ] if DEPLOY_MODE == "dbstor": FS_TYPE_LIST = [ - "storage_dbstor_fs", "storage_dbstor_page_fs", - "storage_share_fs", "storage_archive_fs" + "storage_dbstor_fs", + "storage_dbstor_page_fs", + "storage_share_fs", + "storage_archive_fs", ] -SHARE_FS_TYPE_LIST = [ - "storage_share_fs", "storage_archive_fs", - "storage_metadata_fs" -] +SHARE_FS_TYPE_LIST = ["storage_share_fs", "storage_archive_fs", "storage_metadata_fs"] if DEPLOY_MODE == "file": FS_TYPE_LIST = [ - "storage_dbstor_fs","storage_share_fs", - "storage_archive_fs", "storage_metadata_fs" + "storage_dbstor_fs", + "storage_share_fs", + "storage_archive_fs", + "storage_metadata_fs", ] SHARE_FS_TYPE_LIST = [ - "storage_dbstor_fs", "storage_share_fs", - "storage_archive_fs", "storage_metadata_fs" + "storage_dbstor_fs", + "storage_share_fs", + "storage_archive_fs", + "storage_metadata_fs", ] ID_NAS_DBSTOR = 1038 ID_NAS_DEFAULT = 11 @@ -56,7 +61,7 @@ def is_valid_string(string): :param string: :return: """ - pattern = r'^[\w\u4e00-\u9fa5.-]{1,255}$' + pattern = r"^[\w\u4e00-\u9fa5.-]{1,255}$" match = re.match(pattern, string) return match is not None @@ -106,11 +111,7 @@ class CreateFS(object): """ capacity_digit = re.findall(r"\d+", capacity)[0] capacity_unit = re.findall(r"[A-Z]+", capacity)[0] - convert_dict = { - "GB": 1, - "TB": 1000, - "PB": 1000 ** 2 - } + convert_dict = {"GB": 1, "TB": 1000, "PB": 1000**2} return convert_dict.get(capacity_unit) * int(capacity_digit) * 1024 * 1024 * 2 @staticmethod @@ -125,10 +126,10 @@ class CreateFS(object): def _result_parse(cls, err_msg, res): result = ResponseParse(res) rsp_code, rsp_result, rsp_data = result.get_rsp_data() - error_code = rsp_result.get('code') + error_code = rsp_result.get("code") if rsp_code != 0 or error_code != 0: - error_des = rsp_result.get('description') - error_sgt = rsp_result.get('suggestion') + error_des = rsp_result.get("description") + error_sgt = rsp_result.get("suggestion") err_msg = err_msg % (error_code, error_des, error_sgt) cls.handle_error_msg(err_msg) return rsp_data @@ -143,11 +144,13 @@ class CreateFS(object): 4. check vstore exists :return: """ + def _check_func(): pool_id = self.fs_info.get("PARENTID") self._check_vstore_exists() self._check_storage_pool_exists(pool_id) self._check_fs_exists() + LOG.info("Check file system params start.") self._check_param_type() if self.deploy_info.get("deploy_mode") == "dbstor": @@ -170,6 +173,7 @@ class CreateFS(object): 5. add nfs client :return: """ + def _create_func(): fs_info = {} deploy_mode = self.deploy_info.get("deploy_mode") @@ -180,8 +184,13 @@ class CreateFS(object): _fs_name = _fs_info.get("NAME") LOG.info("Begin to create fs [%s] name: %s", fs_type, _fs_name) vstore_id = self.fs_info.get(fs_type).get("vstoreId") - LOG.info("Begin to create fs [%s] name: %s, vstore id:[%s] in [%s] deploy mode", - fs_type, _fs_name, vstore_id, deploy_mode) + LOG.info( + "Begin to create fs [%s] name: %s, vstore id:[%s] in [%s] deploy mode", + fs_type, + _fs_name, + vstore_id, + deploy_mode, + ) _fs_info = self.storage_opt.query_filesystem_info(_fs_name, vstore_id) if _fs_info: err_msg = "The file system[%s] already exists." % _fs_name @@ -195,9 +204,14 @@ class CreateFS(object): fs_info[fs_type] = { "fs_id": fs_id, "nfs_share_id": nfs_share_id, - "nfs_share_client_id": nfs_share_client_id + "nfs_share_client_id": nfs_share_client_id, } - LOG.info("Create fs [%s] success, detail:name[%s], info:%s", fs_type, _fs_name, fs_info) + LOG.info( + "Create fs [%s] success, detail:name[%s], info:%s", + fs_type, + _fs_name, + fs_info, + ) LOG.info("Create fs start.") self.storage_opt.login() @@ -217,6 +231,7 @@ class CreateFS(object): 4. delete fs info :return: """ + def _delete_func(): for fs_type in FS_TYPE_LIST: _fs_info = self.fs_info.get(fs_type) @@ -233,8 +248,13 @@ class CreateFS(object): nfs_share_id = nfs_share_info[0].get("ID") self.storage_opt.delete_nfs_share(nfs_share_id, vstore_id) else: - LOG.info("The nfs share of fs [%s] name %s is not exist", fs_type, _fs_name) + LOG.info( + "The nfs share of fs [%s] name %s is not exist", + fs_type, + _fs_name, + ) self.storage_opt.delete_file_system(fs_id) + LOG.info("Delete fs info start.") self.storage_opt.login() try: @@ -246,12 +266,16 @@ class CreateFS(object): def _check_vstore_id(self): deploy_info = json.loads(read_helper(DEPLOY_PARAM_PATH)) deploy_info_dbstor_fs_vstore_id = deploy_info.get("dbstor_fs_vstore_id") - fs_info_dbstor_fs_vstore_id = self.fs_info.get("storage_dbstor_fs").get("vstoreId") + fs_info_dbstor_fs_vstore_id = self.fs_info.get("storage_dbstor_fs").get( + "vstoreId" + ) if int(deploy_info_dbstor_fs_vstore_id) != int(fs_info_dbstor_fs_vstore_id): - err_msg = "dbstor_fs_vstore_id of config_params.json is " \ - "different from file_system_info.json,details:" \ - " dbstor_fs_vstore_id:(%s, %s)" % (fs_info_dbstor_fs_vstore_id, - deploy_info_dbstor_fs_vstore_id) + err_msg = ( + "dbstor_fs_vstore_id of config_params.json is " + "different from file_system_info.json,details:" + " dbstor_fs_vstore_id:(%s, %s)" + % (fs_info_dbstor_fs_vstore_id, deploy_info_dbstor_fs_vstore_id) + ) LOG.error(err_msg) raise Exception(err_msg) @@ -269,10 +293,7 @@ class CreateFS(object): name_err = [] digit_err = [] range_err = [] - range_check = { - "SNAPSHOTRESERVEPER": (0, 51), - "CAPACITYTHRESHOLD": (50, 99) - } + range_check = {"SNAPSHOTRESERVEPER": (0, 51), "CAPACITYTHRESHOLD": (50, 99)} digit_check = ["SNAPSHOTRESERVEPER", "CAPACITYTHRESHOLD"] pool_id = self.fs_info.get("PARENTID") if not isinstance(pool_id, int): @@ -291,7 +312,9 @@ class CreateFS(object): for key, value in fs_info.items(): if key in digit_check and not isinstance(value, int): digit_err.append({fs_type: key}) - if key in list(range_check.keys()) and value not in range(*range_check.get(key)): + if key in list(range_check.keys()) and value not in range( + *range_check.get(key) + ): range_err.append({fs_type: key}) if key == "CAPACITY" and not self.check_capacity(value): digit_err.append({fs_type: key}) @@ -363,8 +386,8 @@ class CreateFS(object): def _get_nfs_share_info(self, fs_type, fs_id): data = { "SHAREPATH": f"/{ self.fs_info.get(fs_type).get('NAME')}/", - "vstoreId": self.fs_info.get(fs_type).get("vstoreId"), - "FSID": fs_id + "vstoreId": self.fs_info.get(fs_type).get("vstoreId"), + "FSID": fs_id, } data.update(self.fs_info.get(fs_type)) return data @@ -376,7 +399,7 @@ class CreateFS(object): "ROOTSQUASH": 1, "PARENTID": parent_id, "vstoreId": self.fs_info.get(fs_type).get("vstoreId"), - "NAME": self.fs_info.get("client_ip") + "NAME": self.fs_info.get("client_ip"), } return data @@ -413,10 +436,16 @@ class CreateFS(object): data = self._get_share_client_info(nfs_share_id, fs_type) return self.storage_opt.add_nfs_client(data) + def main(): create_parser = argparse.ArgumentParser() - create_parser.add_argument('--action', choices=["create", "delete", "pre_check"], dest="action", required=True) - create_parser.add_argument('--ip', dest="ip_addr", required=True) + create_parser.add_argument( + "--action", + choices=["create", "delete", "pre_check"], + dest="action", + required=True, + ) + create_parser.add_argument("--ip", dest="ip_addr", required=True) args = create_parser.parse_args() action = args.action ip_addr = args.ip_addr @@ -431,5 +460,9 @@ if __name__ == "__main__": try: main() except Exception as err: - LOG.error("Execute create fs failed, details:%s, traceback:%s", str(err), traceback.format_exc()) + LOG.error( + "Execute create fs failed, details:%s, traceback:%s", + str(err), + traceback.format_exc(), + ) exit(1) diff --git a/pkg/deploy/action/storage_operate/do_snapshot.py b/pkg/deploy/action/storage_operate/do_snapshot.py index 5560d132..2abdf03a 100644 --- a/pkg/deploy/action/storage_operate/do_snapshot.py +++ b/pkg/deploy/action/storage_operate/do_snapshot.py @@ -15,8 +15,8 @@ from storage_operate.dr_deploy_operate.dr_deploy_common import DRDeployCommon from utils.client.rest_client import get_cur_timestamp, read_helper, write_helper from om_log import REST_LOG as LOG -DEPLOY_PARAM_PATH = '/opt/cantian/config/deploy_param.json' -DR_DEPLOY_FLAG = os.path.join(CUR_PATH, '../../config/.dr_deploy_flag') +DEPLOY_PARAM_PATH = "/opt/cantian/config/deploy_param.json" +DR_DEPLOY_FLAG = os.path.join(CUR_PATH, "../../config/.dr_deploy_flag") NORMAL_STATE, ABNORMAL_STATE = 0, 1 @@ -27,14 +27,14 @@ class SnapShotRestClient(object): self.upgrade_version = get_cur_timestamp() self.processed_fs = processed_fs self.handler = { - 'create': self.create_snapshots, - 'rollback': self.rollback_snapshots, - "delete": self.delete_snapshots + "create": self.create_snapshots, + "rollback": self.rollback_snapshots, + "delete": self.delete_snapshots, } @staticmethod def exception_handler(err_msg=None, cur_mode=None): - err_info = '[current_mode] {}, [err_info] {}'.format(cur_mode, err_msg) + err_info = "[current_mode] {}, [err_info] {}".format(cur_mode, err_msg) LOG.error(err_info) raise Exception(err_info) @@ -47,7 +47,9 @@ class SnapShotRestClient(object): self.storage_operate.login() config_params = json.loads(read_helper(DEPLOY_PARAM_PATH)) storage_dbstor_page_fs = config_params.get("storage_dbstor_page_fs") - page_fs_info = self.storage_operate.query_filesystem_info(storage_dbstor_page_fs) + page_fs_info = self.storage_operate.query_filesystem_info( + storage_dbstor_page_fs + ) page_fs_id = page_fs_info.get("ID") dr_deploy_opt = DRDeployCommon(self.storage_operate) page_pair_info = dr_deploy_opt.query_remote_replication_pair_info(page_fs_id) @@ -57,7 +59,9 @@ class SnapShotRestClient(object): if not os.path.exists(DR_DEPLOY_FLAG): modes = stat.S_IWRITE | stat.S_IRUSR flags = os.O_WRONLY | os.O_TRUNC | os.O_CREAT - with os.fdopen(os.open(DR_DEPLOY_FLAG, flags, modes), 'w', encoding='utf-8') as file: + with os.fdopen( + os.open(DR_DEPLOY_FLAG, flags, modes), "w", encoding="utf-8" + ) as file: file.write("") return secondary == "false" LOG.info("Current node is not dr or is primary") @@ -66,15 +70,19 @@ class SnapShotRestClient(object): def create_snapshots(self, fs_name, vstore_id=0): if self.processed_fs.get(fs_name): return NORMAL_STATE - fs_info = self.storage_operate.query_filesystem_info(fs_name, vstore_id=vstore_id) + fs_info = self.storage_operate.query_filesystem_info( + fs_name, vstore_id=vstore_id + ) if not fs_info: err_msg = "file system [%s] is not exist, please check." % fs_name - self.exception_handler(err_msg=err_msg, cur_mode='create') + self.exception_handler(err_msg=err_msg, cur_mode="create") fs_id = fs_info.get("ID") - reg_version = '{}_{}'.format(fs_name, self.upgrade_version).replace('.', '_') - snapshot_name = '{}_{}'.format(reg_version, str(get_cur_timestamp())) - snapshot_info = self.storage_operate.create_file_system_snapshot(snapshot_name, fs_id, vstore_id=vstore_id) - snapshot_id = snapshot_info.get('ID') + reg_version = "{}_{}".format(fs_name, self.upgrade_version).replace(".", "_") + snapshot_name = "{}_{}".format(reg_version, str(get_cur_timestamp())) + snapshot_info = self.storage_operate.create_file_system_snapshot( + snapshot_name, fs_id, vstore_id=vstore_id + ) + snapshot_id = snapshot_info.get("ID") self.processed_fs[fs_name] = snapshot_id return NORMAL_STATE @@ -84,9 +92,14 @@ class SnapShotRestClient(object): return NORMAL_STATE snapshot_id = self.processed_fs.get(fs_name) try: - snapshot_info = self.storage_operate.query_file_system_snapshot_info(snapshot_id) + snapshot_info = self.storage_operate.query_file_system_snapshot_info( + snapshot_id + ) except Exception as e: - if str(e).find("1077937875") != -1 or str(e).find("snapshot does not exist") != -1: + if ( + str(e).find("1077937875") != -1 + or str(e).find("snapshot does not exist") != -1 + ): err_msg = "The snapshot is already not exist, details:%s" % str(e) LOG.info(err_msg) return NORMAL_STATE @@ -105,9 +118,14 @@ class SnapShotRestClient(object): return NORMAL_STATE snapshot_id = self.processed_fs.get(fs_name) try: - snapshot_info = self.storage_operate.query_file_system_snapshot_info(snapshot_id) + snapshot_info = self.storage_operate.query_file_system_snapshot_info( + snapshot_id + ) except Exception as e: - if str(e).find("1077937875") != -1 or str(e).find("snapshot does not exist") != -1: + if ( + str(e).find("1077937875") != -1 + or str(e).find("snapshot does not exist") != -1 + ): err_msg = "The snapshot is already not exist, details:%s" % str(e) LOG.info(err_msg) return NORMAL_STATE @@ -130,8 +148,8 @@ class SnapShotRestClient(object): def get_fs_processed_info(info_path, fs_names): - json_file_path = os.path.join(info_path, 'processed_snapshots.json') - init_fs_info = {name: '' for name in fs_names} + json_file_path = os.path.join(info_path, "processed_snapshots.json") + init_fs_info = {name: "" for name in fs_names} if not os.path.exists(info_path): os.makedirs(info_path) return init_fs_info @@ -160,18 +178,28 @@ def main(mode, ip_address, main_path): dbstor_fs_vstore_id = config_params.get("dbstor_fs_vstore_id", 0) fs_names_type = [] for fs_type, fs_name in config_params.items(): - if fs_type.endswith('_fs') and fs_type.startswith("storage") and fs_type == "storage_share_fs" and fs_name: + if ( + fs_type.endswith("_fs") + and fs_type.startswith("storage") + and fs_type == "storage_share_fs" + and fs_name + ): fs_names_type.append((fs_name, fs_type, vstore_id)) - elif fs_type.endswith('_fs') and fs_type.startswith("storage") and fs_type == "storage_dbstor_fs" and fs_name: + elif ( + fs_type.endswith("_fs") + and fs_type.startswith("storage") + and fs_type == "storage_dbstor_fs" + and fs_name + ): fs_names_type.append((fs_name, fs_type, dbstor_fs_vstore_id)) - elif fs_type.endswith('_fs') and fs_type.startswith("storage") and fs_name: + elif fs_type.endswith("_fs") and fs_type.startswith("storage") and fs_name: fs_names_type.append((fs_name, fs_type, 0)) fs_names = [ fs_val for fs_name, fs_val in config_params.items() - if fs_name.endswith('_fs') and fs_name.startswith("storage") and fs_val + if fs_name.endswith("_fs") and fs_name.startswith("storage") and fs_val ] - process_fs_path = '{}/cantian_upgrade_snapshots'.format(main_path) + process_fs_path = "{}/cantian_upgrade_snapshots".format(main_path) fs_processed_data = get_fs_processed_info(process_fs_path, fs_names) login_data = (ip_address, user_name, passwd) @@ -184,19 +212,23 @@ def main(mode, ip_address, main_path): try: _ = rest_client_obj.execute(fs_name, mode, _vstore_id) except Exception as error: - LOG.error('error happened when try to {} snapshot of {}, err_info: {}, ' - 'err_traceback: {}'.format(mode, fs_name, str(error), traceback.format_exc(limit=-1))) + LOG.error( + "error happened when try to {} snapshot of {}, err_info: {}, " + "err_traceback: {}".format( + mode, fs_name, str(error), traceback.format_exc(limit=-1) + ) + ) return ABNORMAL_STATE - if mode == 'rollback': + if mode == "rollback": query_rollback_process(fs_names_type, rest_client_obj.storage_operate) - recoder_path = os.path.join(process_fs_path, 'processed_snapshots.json') - if mode == 'create': + recoder_path = os.path.join(process_fs_path, "processed_snapshots.json") + if mode == "create": new_processed_info = rest_client_obj.processed_fs write_helper(recoder_path, new_processed_info) - elif mode == 'delete': - write_helper(recoder_path, '') + elif mode == "delete": + write_helper(recoder_path, "") return NORMAL_STATE @@ -213,11 +245,20 @@ def query_rollback_process(fs_names_type, rest_client_obj): while query_list: for item in query_list: fs_name, fs_type, _vstore_id = item - data = rest_client_obj.query_rollback_snapshots_process(fs_name, vstore_id=_vstore_id) - rollback_rate, rollback_status = data.get("rollbackRate"), data.get("rollbackStatus") + data = rest_client_obj.query_rollback_snapshots_process( + fs_name, vstore_id=_vstore_id + ) + rollback_rate, rollback_status = data.get("rollbackRate"), data.get( + "rollbackStatus" + ) if int(rollback_status) == 0: success_list.append(item) - LOG.info("Filesystem[%s] rollback status[%s], process[%s]", fs_name, rollback_status, rollback_rate) + LOG.info( + "Filesystem[%s] rollback status[%s], process[%s]", + fs_name, + rollback_status, + rollback_rate, + ) query_list = list(set(query_list) - set(success_list)) time.sleep(30) @@ -228,8 +269,12 @@ if __name__ == "__main__": try: RET_VAL = main(snapshot_mode, ip, main_backup_file_path) except Exception as err: - LOG.error('{} snapshots failed, err_details: {}, ' - 'err_traceback: {}'.format(snapshot_mode, str(err), traceback.format_exc(limit=-1))) + LOG.error( + "{} snapshots failed, err_details: {}, " + "err_traceback: {}".format( + snapshot_mode, str(err), traceback.format_exc(limit=-1) + ) + ) exit(ABNORMAL_STATE) exit(RET_VAL) diff --git a/pkg/deploy/action/storage_operate/dr_deploy_operate/dr_deploy.py b/pkg/deploy/action/storage_operate/dr_deploy_operate/dr_deploy.py index 8f98ee28..6c2a5559 100644 --- a/pkg/deploy/action/storage_operate/dr_deploy_operate/dr_deploy.py +++ b/pkg/deploy/action/storage_operate/dr_deploy_operate/dr_deploy.py @@ -12,8 +12,18 @@ import traceback from storage_operate.dr_deploy_operate.dr_deploy_common import DRDeployCommon from storage_operate.dr_deploy_operate.dr_deploy_common import KmcResolve -from utils.config.rest_constant import HealthStatus, MetroDomainRunningStatus, SecresAccess, VstorePairRunningStatus, \ - FilesystemPairRunningStatus, ReplicationRunningStatus, CANTIAN_DOMAIN_PREFIX, Constant, SPEED, VstorePairConfigStatus +from utils.config.rest_constant import ( + HealthStatus, + MetroDomainRunningStatus, + SecresAccess, + VstorePairRunningStatus, + FilesystemPairRunningStatus, + ReplicationRunningStatus, + CANTIAN_DOMAIN_PREFIX, + Constant, + SPEED, + VstorePairConfigStatus, +) from logic.storage_operate import StorageInf from logic.common_func import read_json_config from logic.common_func import write_json_config @@ -33,16 +43,22 @@ DR_DEPLOY_CONFIG = os.path.join(CURRENT_PATH, "../../../config/dr_deploy_param.j DEPLOY_PARAM_FILE = "/opt/cantian/config/deploy_param.json" DEFAULT_PARAM_FILE = os.path.join(CURRENT_PATH, "../../config_params.json") EXEC_SQL = os.path.join(CURRENT_PATH, "../../cantian_common/exec_sql.py") -LOCAL_PROCESS_RECORD_FILE = os.path.join(CURRENT_PATH, "../../../config/dr_process_record.json") -FULL_CHECK_POINT_CMD = 'echo -e "alter system checkpoint global;" | '\ - 'su -s /bin/bash - %s -c \'source ~/.bashrc && '\ - 'export LD_LIBRARY_PATH=/opt/cantian/dbstor/lib:${LD_LIBRARY_PATH} && '\ - 'python3 -B %s\'' % (RUN_USER, EXEC_SQL) -CANTIAN_DISASTER_RECOVERY_STATUS_CHECK = 'echo -e "select * from DV_LRPL_DETAIL;" | '\ - 'su -s /bin/bash - %s -c \'source ~/.bashrc && '\ - 'export LD_LIBRARY_PATH=/opt/cantian/dbstor/lib:${LD_LIBRARY_PATH} && '\ - 'python3 -B %s\'' % (RUN_USER, EXEC_SQL) -ZSQL_INI_PATH = '/mnt/dbdata/local/cantian/tmp/data/cfg/ctsql.ini' +LOCAL_PROCESS_RECORD_FILE = os.path.join( + CURRENT_PATH, "../../../config/dr_process_record.json" +) +FULL_CHECK_POINT_CMD = ( + 'echo -e "alter system checkpoint global;" | ' + "su -s /bin/bash - %s -c 'source ~/.bashrc && " + "export LD_LIBRARY_PATH=/opt/cantian/dbstor/lib:${LD_LIBRARY_PATH} && " + "python3 -B %s'" % (RUN_USER, EXEC_SQL) +) +CANTIAN_DISASTER_RECOVERY_STATUS_CHECK = ( + 'echo -e "select * from DV_LRPL_DETAIL;" | ' + "su -s /bin/bash - %s -c 'source ~/.bashrc && " + "export LD_LIBRARY_PATH=/opt/cantian/dbstor/lib:${LD_LIBRARY_PATH} && " + "python3 -B %s'" % (RUN_USER, EXEC_SQL) +) +ZSQL_INI_PATH = "/mnt/dbdata/local/cantian/tmp/data/cfg/ctsql.ini" LOCK_INSTANCE_STEP1 = "set @ctc_ddl_enabled=true" LOCK_INSTANCE_STEP2 = "lock instance for backup" UNLOCK_INSTANCE = "unlock instance" @@ -51,7 +67,7 @@ FLUSH_TABLE = "flush table with read lock;unlock tables;" INSTALL_TIMEOUT = 900 START_TIMEOUT = 3600 FS_CREAT_TIMEOUT = 300 -TOTAL_CHECK_DURATION = 180 # 创建双活pair检查时间 +TOTAL_CHECK_DURATION = 180 # 创建双活pair检查时间 ACTIVE_RECORD_DICT = { @@ -63,7 +79,7 @@ ACTIVE_RECORD_DICT = { "create_metro_fs_pair": "default", "create_rep_page_fs_pair": "default", "sync_metro_fs_pair": "default", - "sync_rep_page_fs_pair": "default" + "sync_rep_page_fs_pair": "default", } STANDBY_RECORD_DICT = { "create_metro_domain": "default", @@ -73,7 +89,7 @@ STANDBY_RECORD_DICT = { "standby_install": "default", "sync_metro_fs_pair": "default", "sync_rep_page_fs_pair": "default", - "standby_start": "default" + "standby_start": "default", } @@ -112,11 +128,15 @@ class DRDeploy(object): 容灾告警需要重启cantian_exporter :return: """ - cmd = "ps -ef | grep \"python3 /opt/cantian/ct_om/service/cantian_exporter/exporter/execute.py\"" \ - " | grep -v grep | awk '{print $2}' | xargs kill -9" + cmd = ( + 'ps -ef | grep "python3 /opt/cantian/ct_om/service/cantian_exporter/exporter/execute.py"' + " | grep -v grep | awk '{print $2}' | xargs kill -9" + ) exec_popen(cmd) - def record_deploy_process(self, exec_step: str, exec_status: str, code=0, description="") -> None: + def record_deploy_process( + self, exec_step: str, exec_status: str, code=0, description="" + ) -> None: """ :param exec_step: 执行步骤 :param exec_status: 执行状态 @@ -133,7 +153,7 @@ class DRDeploy(object): data.update({exec_step: exec_status}) error["code"] = code error["description"] = description - with os.fdopen(os.open(self.record_progress_file, flags, modes), 'w') as fp: + with os.fdopen(os.open(self.record_progress_file, flags, modes), "w") as fp: json.dump(result, fp, indent=4) def record_deploy_process_init(self): @@ -165,38 +185,35 @@ class DRDeploy(object): :return: """ self.metadata_in_cantian = self.dr_deploy_info.get("mysql_metadata_in_cantian") - dr_record_dict = ACTIVE_RECORD_DICT if self.site == "active" else STANDBY_RECORD_DICT + dr_record_dict = ( + ACTIVE_RECORD_DICT if self.site == "active" else STANDBY_RECORD_DICT + ) if not self.metadata_in_cantian: - dr_record_dict.update({ - "create_rep_meta_fs_pair": "default", - "sync_rep_meta_fs_pair": "default" - }) + dr_record_dict.update( + { + "create_rep_meta_fs_pair": "default", + "sync_rep_meta_fs_pair": "default", + } + ) if self.site == "standby": - dr_record_dict.update({ - "cantian_disaster_recovery_status": "default" - }) + dr_record_dict.update({"cantian_disaster_recovery_status": "default"}) else: if not self.metadata_in_cantian: - dr_record_dict.update({ - "cancel_rep_meta_fs_secondary_write_lock": "default" - }) - dr_record_dict.update({ - "cancel_rep_page_fs_secondary_write_lock": "default", - "do_unlock_instance_for_backup": "default" - }) - dr_record_dict.update({"dr_deploy": "default"}) - result = { - "data": dr_record_dict, - "error": + dr_record_dict.update( + {"cancel_rep_meta_fs_secondary_write_lock": "default"} + ) + dr_record_dict.update( { - "code": 0, - "description": "" + "cancel_rep_page_fs_secondary_write_lock": "default", + "do_unlock_instance_for_backup": "default", } - } + ) + dr_record_dict.update({"dr_deploy": "default"}) + result = {"data": dr_record_dict, "error": {"code": 0, "description": ""}} flags = os.O_WRONLY | os.O_CREAT | os.O_TRUNC modes = stat.S_IWUSR | stat.S_IRUSR - with os.fdopen(os.open(self.record_progress_file, flags, modes), 'w') as fp: + with os.fdopen(os.open(self.record_progress_file, flags, modes), "w") as fp: json.dump(result, fp, indent=4) def record_disaster_recovery_info(self, key, value): @@ -219,29 +236,39 @@ class DRDeploy(object): lock_instance_flag = False wait_lock_instance_time = 0 mysql_shell = None - while not lock_instance_flag and wait_lock_instance_time < LOCK_INSTANCE_TIMEOUT: + while ( + not lock_instance_flag and wait_lock_instance_time < LOCK_INSTANCE_TIMEOUT + ): try: # 创建mysql shell会话，并启动会话 - mysql_shell = MysqlShell(self.mysql_cmd, user=self.mysql_user, password=self.mysql_pwd) + mysql_shell = MysqlShell( + self.mysql_cmd, user=self.mysql_user, password=self.mysql_pwd + ) mysql_shell.start_session() self.record_deploy_process("do_lock_instance_for_backup", "start") mysql_shell.execute_command(LOCK_INSTANCE_STEP1, timeout=3) - lock_instance_output = mysql_shell.execute_command(LOCK_INSTANCE_STEP2, timeout=3) + lock_instance_output = mysql_shell.execute_command( + LOCK_INSTANCE_STEP2, timeout=3 + ) if "Query OK" in lock_instance_output: lock_instance_flag = True LOG.info("Success to do lock instance for backup.") self.backup_lock_shell = mysql_shell except Exception as err: mysql_shell.close_session() - LOG.info("Failed to do lock instance for backup, error msg:%s" % (str(err))) + LOG.info( + "Failed to do lock instance for backup, error msg:%s" % (str(err)) + ) wait_lock_instance_time += 10 time.sleep(10) if not lock_instance_flag: err_msg = "Failed to do lock instance for backup, timeout." LOG.error(err_msg) - self.record_deploy_process("do_lock_instance_for_backup", "failed", code=-1, description=err_msg) + self.record_deploy_process( + "do_lock_instance_for_backup", "failed", code=-1, description=err_msg + ) raise Exception("err_msg") self.record_deploy_process("do_lock_instance_for_backup", "success") @@ -279,7 +306,9 @@ class DRDeploy(object): else: err_msg = "Failed to do unlock instance for backup." LOG.error(err_msg) - self.record_deploy_process("do_unlock_instance_for_backup", "failed", code=-1, description=err_msg) + self.record_deploy_process( + "do_unlock_instance_for_backup", "failed", code=-1, description=err_msg + ) raise Exception(err_msg) self.backup_lock_shell = None @@ -290,10 +319,12 @@ class DRDeploy(object): :return: """ LOG.info("Start to do flush table with read lock.") - cmd = "%s -u'%s' -p'%s' -e \"%s;\"" % (self.mysql_cmd, - self.mysql_user, - self.mysql_pwd, - FLUSH_TABLE) + cmd = "%s -u'%s' -p'%s' -e \"%s;\"" % ( + self.mysql_cmd, + self.mysql_user, + self.mysql_pwd, + FLUSH_TABLE, + ) cmd += ";echo last_cmd=$?" self.record_deploy_process("do_flush_table_with_read_lock", "start") attempts = 15 @@ -311,15 +342,19 @@ class DRDeploy(object): time.sleep(20) attempts -= 1 continue - + else: break - err_msg = "Failed to do unlock table with read lock, " \ - "output:%s, stderr:%s" % (output, stderr) + err_msg = ( + "Failed to do unlock table with read lock, " + "output:%s, stderr:%s" % (output, stderr) + ) err_msg.replace(self.mysql_pwd, "***") LOG.error(err_msg) - self.record_deploy_process("do_flush_table_with_read_lock", "failed", code=-1, description=err_msg) + self.record_deploy_process( + "do_flush_table_with_read_lock", "failed", code=-1, description=err_msg + ) raise Exception(err_msg) def do_full_check_point(self): @@ -331,9 +366,14 @@ class DRDeploy(object): self.record_deploy_process("do_full_check_point", "start") return_code, output, stderr = exec_popen(FULL_CHECK_POINT_CMD, timeout=100) if return_code: - err_msg = "Do full checkpoint failed, output: %s, stderr:%s" % (output, stderr) + err_msg = "Do full checkpoint failed, output: %s, stderr:%s" % ( + output, + stderr, + ) LOG.error(err_msg) - self.record_deploy_process("do_full_check_point", "failed", code=-1, description=err_msg) + self.record_deploy_process( + "do_full_check_point", "failed", code=-1, description=err_msg + ) raise Exception(err_msg) self.record_deploy_process("do_full_check_point", "success") LOG.info("Success to do full checkpoint.") @@ -371,17 +411,24 @@ class DRDeploy(object): domain_name = self.dr_deploy_info.get("domain_name") if hyper_domain_id is None: domain_info = self.dr_deploy_opt.create_filesystem_hyper_metro_domain( - remote_dev_name, remote_dev_esn, remote_device_id, domain_name) + remote_dev_name, remote_dev_esn, remote_device_id, domain_name + ) else: - domain_info = self.dr_deploy_opt.query_hyper_metro_domain_info(hyper_domain_id) + domain_info = self.dr_deploy_opt.query_hyper_metro_domain_info( + hyper_domain_id + ) if not domain_info: domain_info = self.dr_deploy_opt.create_filesystem_hyper_metro_domain( - remote_dev_name, remote_dev_esn, remote_device_id, domain_name) + remote_dev_name, remote_dev_esn, remote_device_id, domain_name + ) else: exist_domain_name = domain_info.get("NAME") if exist_domain_name != domain_name: - err_msg = "Hyper metro domain [name:%s] is unmatched with config parmas [name:%s], " \ - "please check, details: %s" % (exist_domain_name, domain_name, domain_info) + err_msg = ( + "Hyper metro domain [name:%s] is unmatched with config parmas [name:%s], " + "please check, details: %s" + % (exist_domain_name, domain_name, domain_info) + ) LOG.error(err_msg) raise Exception(err_msg) @@ -415,20 +462,29 @@ class DRDeploy(object): if vstore_pair_id is None: vstore_pair_info = self.dr_deploy_opt.create_hyper_metro_vstore_pair( - domain_id, local_vstore_id, remote_vstore_id) + domain_id, local_vstore_id, remote_vstore_id + ) else: - vstore_pair_info = self.dr_deploy_opt.query_hyper_metro_vstore_pair_info(vstore_pair_id) + vstore_pair_info = self.dr_deploy_opt.query_hyper_metro_vstore_pair_info( + vstore_pair_id + ) if not vstore_pair_info: vstore_pair_info = self.dr_deploy_opt.create_hyper_metro_vstore_pair( - domain_id, local_vstore_id, remote_vstore_id) + domain_id, local_vstore_id, remote_vstore_id + ) else: exist_remote_vstoreid = vstore_pair_info.get("REMOTEVSTOREID") exist_local_vstoreid = vstore_pair_info.get("LOCALVSTOREID") exist_domain_id = vstore_pair_info.get("DOMAINID") - if exist_local_vstoreid != local_vstore_id or remote_vstore_id != exist_remote_vstoreid or \ - exist_domain_id != domain_id: - err_msg = "The vstore pair [id:%s] is unmatched with config params, please check, details: %s" % \ - (vstore_pair_id, vstore_pair_info) + if ( + exist_local_vstoreid != local_vstore_id + or remote_vstore_id != exist_remote_vstoreid + or exist_domain_id != domain_id + ): + err_msg = ( + "The vstore pair [id:%s] is unmatched with config params, please check, details: %s" + % (vstore_pair_id, vstore_pair_info) + ) LOG.error(err_msg) raise Exception(err_msg) if not vstore_pair_info: @@ -439,24 +495,33 @@ class DRDeploy(object): tmp_time = 0 while tmp_time < TOTAL_CHECK_DURATION: - vstore_pair_info = self.dr_deploy_opt.query_hyper_metro_vstore_pair_info(vstore_pair_id) + vstore_pair_info = self.dr_deploy_opt.query_hyper_metro_vstore_pair_info( + vstore_pair_id + ) health_status = vstore_pair_info.get("HEALTHSTATUS") running_status = vstore_pair_info.get("RUNNINGSTATUS") config_status = vstore_pair_info.get("CONFIGSTATUS") - if (running_status == VstorePairRunningStatus.Normal and health_status == HealthStatus.Normal - and config_status == VstorePairConfigStatus.Normal): + if ( + running_status == VstorePairRunningStatus.Normal + and health_status == HealthStatus.Normal + and config_status == VstorePairConfigStatus.Normal + ): self.record_deploy_process("create_metro_vstore_pair", "success") return vstore_pair_info time.sleep(10) tmp_time += 10 - err_msg = "Hyper metro vstore pair status is not normal, " \ - "health_status[%s], running_status[%s], details: %s" % \ - (get_status(health_status, HealthStatus), - get_status(running_status, VstorePairRunningStatus), - vstore_pair_info) + err_msg = ( + "Hyper metro vstore pair status is not normal, " + "health_status[%s], running_status[%s], details: %s" + % ( + get_status(health_status, HealthStatus), + get_status(running_status, VstorePairRunningStatus), + vstore_pair_info, + ) + ) LOG.error(err_msg) raise Exception(err_msg) @@ -473,38 +538,61 @@ class DRDeploy(object): dbstor_fs_vstore_id = self.dr_deploy_info.get("dbstor_fs_vstore_id") filesystem_pair_id = self.dr_deploy_info.get("ulog_fs_pair_id") storage_dbstor_fs = self.dr_deploy_info.get("storage_dbstor_fs") - dbstor_fs_info = self.dr_deploy_opt.storage_opt.query_filesystem_info(storage_dbstor_fs, - dbstor_fs_vstore_id) + dbstor_fs_info = self.dr_deploy_opt.storage_opt.query_filesystem_info( + storage_dbstor_fs, dbstor_fs_vstore_id + ) dbstor_fs_id = dbstor_fs_info.get("ID") if filesystem_pair_id is None: - filesystem_pair_infos = self.dr_deploy_opt.query_hyper_metro_filesystem_pair_info(dbstor_fs_id) + filesystem_pair_infos = ( + self.dr_deploy_opt.query_hyper_metro_filesystem_pair_info(dbstor_fs_id) + ) if filesystem_pair_infos is None: - filesystem_pair_info = self.dr_deploy_opt.create_hyper_metro_filesystem_pair( - filesystem_id=dbstor_fs_id, pool_id=remote_pool_id, vstore_pair_id=vstore_pair_id) + filesystem_pair_info = ( + self.dr_deploy_opt.create_hyper_metro_filesystem_pair( + filesystem_id=dbstor_fs_id, + pool_id=remote_pool_id, + vstore_pair_id=vstore_pair_id, + ) + ) task_id = filesystem_pair_info.get("taskId") self.record_deploy_process("create_metro_fs_pair", "running") self.dr_deploy_opt.query_omtask_process(task_id, timeout=120) else: - filesystem_pair_info = self.dr_deploy_opt.query_hyper_metro_filesystem_pair_info_by_pair_id( - pair_id=filesystem_pair_id) + filesystem_pair_info = ( + self.dr_deploy_opt.query_hyper_metro_filesystem_pair_info_by_pair_id( + pair_id=filesystem_pair_id + ) + ) if filesystem_pair_info is None: - filesystem_pair_info = self.dr_deploy_opt.create_hyper_metro_filesystem_pair( - filesystem_id=dbstor_fs_id, pool_id=remote_pool_id, vstore_pair_id=vstore_pair_id) + filesystem_pair_info = ( + self.dr_deploy_opt.create_hyper_metro_filesystem_pair( + filesystem_id=dbstor_fs_id, + pool_id=remote_pool_id, + vstore_pair_id=vstore_pair_id, + ) + ) task_id = filesystem_pair_info.get("taskId") self.record_deploy_process("create_metro_fs_pair", "running") self.dr_deploy_opt.query_omtask_process(task_id, timeout=120) else: exist_domain_id = filesystem_pair_info.get("DOMAINID") if exist_domain_id != hyper_domain_id: - err_msg = "The HyperMetro domain [id:%s] of filesystem pair is unmatched with config " \ - "params [id:%s], please check, details: %s" % \ - (exist_domain_id, hyper_domain_id, filesystem_pair_info) + err_msg = ( + "The HyperMetro domain [id:%s] of filesystem pair is unmatched with config " + "params [id:%s], please check, details: %s" + % (exist_domain_id, hyper_domain_id, filesystem_pair_info) + ) LOG.error(err_msg) raise Exception(err_msg) - filesystem_pair_infos = self.dr_deploy_opt.query_hyper_metro_filesystem_pair_info(dbstor_fs_id) + filesystem_pair_infos = ( + self.dr_deploy_opt.query_hyper_metro_filesystem_pair_info(dbstor_fs_id) + ) if len(filesystem_pair_infos) != 1: - err_msg = "The metro filesystem pair create failed, Details: %s" % filesystem_pair_infos + err_msg = ( + "The metro filesystem pair create failed, Details: %s" + % filesystem_pair_infos + ) raise Exception(err_msg) filesystem_pair_id = filesystem_pair_infos[0].get("ID") @@ -513,25 +601,35 @@ class DRDeploy(object): running_status = None config_status = None while tmp_time < TOTAL_CHECK_DURATION: - filesystem_pair_info = self.dr_deploy_opt.query_hyper_metro_filesystem_pair_info_by_pair_id( - pair_id=filesystem_pair_id) + filesystem_pair_info = ( + self.dr_deploy_opt.query_hyper_metro_filesystem_pair_info_by_pair_id( + pair_id=filesystem_pair_id + ) + ) health_status = filesystem_pair_info.get("HEALTHSTATUS") running_status = filesystem_pair_info.get("RUNNINGSTATUS") config_status = filesystem_pair_info.get("CONFIGSTATUS") - if health_status == HealthStatus.Normal and config_status == VstorePairConfigStatus.Normal: + if ( + health_status == HealthStatus.Normal + and config_status == VstorePairConfigStatus.Normal + ): self.record_deploy_process("create_metro_fs_pair", "success") return filesystem_pair_info time.sleep(10) tmp_time += 10 - err_msg = "Hyper metro vstore pair status is not normal, " \ - "health_status[%s], running_status[%s], config_status[%s], details: %s" % \ - (get_status(health_status, HealthStatus), - get_status(running_status, FilesystemPairRunningStatus), - get_status(config_status, VstorePairConfigStatus), - filesystem_pair_info) + err_msg = ( + "Hyper metro vstore pair status is not normal, " + "health_status[%s], running_status[%s], config_status[%s], details: %s" + % ( + get_status(health_status, HealthStatus), + get_status(running_status, FilesystemPairRunningStatus), + get_status(config_status, VstorePairConfigStatus), + filesystem_pair_info, + ) + ) LOG.error(err_msg) raise Exception(err_msg) @@ -541,32 +639,52 @@ class DRDeploy(object): :param pair_id: ulog文件系统pair id :return: """ - filesystem_pair_info = self.dr_deploy_opt.query_hyper_metro_filesystem_pair_info_by_pair_id(pair_id) + filesystem_pair_info = ( + self.dr_deploy_opt.query_hyper_metro_filesystem_pair_info_by_pair_id( + pair_id + ) + ) running_status = filesystem_pair_info.get("RUNNINGSTATUS") sync_progress = filesystem_pair_info.get("SYNCPROGRESS") health_status = filesystem_pair_info.get("HEALTHSTATUS") - if running_status == FilesystemPairRunningStatus.Normal \ - and health_status == HealthStatus.Normal \ - and sync_progress == "100": + if ( + running_status == FilesystemPairRunningStatus.Normal + and health_status == HealthStatus.Normal + and sync_progress == "100" + ): LOG.info("Sync hyper metro filesystem pair success") self.record_deploy_process("sync_metro_fs_pair", "success") return True - if running_status == FilesystemPairRunningStatus.Invalid or health_status != HealthStatus.Normal: - err_msg = "Failed to create hyper metro filesystem pair, " \ - "health status[%s], running status[%s] details:%s" % \ - (get_status(health_status, HealthStatus), - get_status(running_status, FilesystemPairRunningStatus), - filesystem_pair_info) - self.record_deploy_process("sync_metro_fs_pair", "failed", code=-1, description=err_msg) + if ( + running_status == FilesystemPairRunningStatus.Invalid + or health_status != HealthStatus.Normal + ): + err_msg = ( + "Failed to create hyper metro filesystem pair, " + "health status[%s], running status[%s] details:%s" + % ( + get_status(health_status, HealthStatus), + get_status(running_status, FilesystemPairRunningStatus), + filesystem_pair_info, + ) + ) + self.record_deploy_process( + "sync_metro_fs_pair", "failed", code=-1, description=err_msg + ) raise Exception(err_msg) time.sleep(2) - LOG.info("Create hyper metro filesystem pair process[%s%%], " - "running_status[%s], health_status[%s]", - sync_progress, - get_status(running_status, FilesystemPairRunningStatus), - get_status(health_status, HealthStatus)) + LOG.info( + "Create hyper metro filesystem pair process[%s%%], " + "running_status[%s], health_status[%s]", + sync_progress, + get_status(running_status, FilesystemPairRunningStatus), + get_status(health_status, HealthStatus), + ) if running_status == FilesystemPairRunningStatus.Paused: - self.record_deploy_process("sync_metro_fs_pair", get_status(running_status, FilesystemPairRunningStatus)) + self.record_deploy_process( + "sync_metro_fs_pair", + get_status(running_status, FilesystemPairRunningStatus), + ) else: self.record_deploy_process("sync_metro_fs_pair", sync_progress + "%") return False @@ -581,25 +699,42 @@ class DRDeploy(object): remote_pool_id = self.dr_deploy_info.get("remote_pool_id") name_suffix = self.dr_deploy_info.get("name_suffix", "") remote_name_rule = 2 if name_suffix else 1 - remote_replication_pair_info = self.dr_deploy_opt.query_remote_replication_pair_info( - filesystem_id=page_fs_id) + remote_replication_pair_info = ( + self.dr_deploy_opt.query_remote_replication_pair_info( + filesystem_id=page_fs_id + ) + ) if remote_replication_pair_info is None: - rep_filesystem_pair_task_info = self.dr_deploy_opt.create_remote_replication_filesystem_pair( - remote_device_id=remote_device_id, - remote_pool_id=remote_pool_id, - local_fs_id=page_fs_id, - remote_name_rule=remote_name_rule, - name_suffix=name_suffix, - speed=self.sync_speed + rep_filesystem_pair_task_info = ( + self.dr_deploy_opt.create_remote_replication_filesystem_pair( + remote_device_id=remote_device_id, + remote_pool_id=remote_pool_id, + local_fs_id=page_fs_id, + remote_name_rule=remote_name_rule, + name_suffix=name_suffix, + speed=self.sync_speed, + ) ) rep_filesystem_pair_task_id = rep_filesystem_pair_task_info.get("taskId") - self.dr_deploy_opt.query_omtask_process(rep_filesystem_pair_task_id, timeout=120) - remote_replication_pair_info = self.dr_deploy_opt.query_remote_replication_pair_info( - filesystem_id=page_fs_id) + self.dr_deploy_opt.query_omtask_process( + rep_filesystem_pair_task_id, timeout=120 + ) + remote_replication_pair_info = ( + self.dr_deploy_opt.query_remote_replication_pair_info( + filesystem_id=page_fs_id + ) + ) return remote_replication_pair_info - @retry(retry_times=3, wait_times=20, log=LOG, task="do_sync_remote_replication_filesystem_pair") - def do_sync_remote_replication_filesystem_pair(self, pair_id: str, is_page: bool) -> bool: + @retry( + retry_times=3, + wait_times=20, + log=LOG, + task="do_sync_remote_replication_filesystem_pair", + ) + def do_sync_remote_replication_filesystem_pair( + self, pair_id: str, is_page: bool + ) -> bool: """ 同步远程复制pair :param is_page: page文件系统或者是meta文件系统 @@ -607,57 +742,88 @@ class DRDeploy(object): :return: """ exec_step = "sync_rep_meta_fs_pair" if not is_page else "sync_rep_page_fs_pair" - remote_replication_pair_info = self.dr_deploy_opt.query_remote_replication_pair_info_by_pair_id( - pair_id=pair_id) + remote_replication_pair_info = ( + self.dr_deploy_opt.query_remote_replication_pair_info_by_pair_id( + pair_id=pair_id + ) + ) replication_pair_id = remote_replication_pair_info.get("ID") replication_progress = remote_replication_pair_info.get("REPLICATIONPROGRESS") start_time = remote_replication_pair_info.get("STARTTIME") end_time = remote_replication_pair_info.get("ENDTIME") - replication_pair_health_status = remote_replication_pair_info.get("HEALTHSTATUS") - replication_pair_running_status = remote_replication_pair_info.get("RUNNINGSTATUS") + replication_pair_health_status = remote_replication_pair_info.get( + "HEALTHSTATUS" + ) + replication_pair_running_status = remote_replication_pair_info.get( + "RUNNINGSTATUS" + ) # 当已经设置从端可读写状态，且为分裂状态时，直接返回 secres_access = remote_replication_pair_info.get("SECRESACCESS") if not is_page: self.meta_fs_pair_id = replication_pair_id else: self.page_fs_pair_id = replication_pair_id - if secres_access == SecresAccess.ReadAndWrite and \ - replication_pair_running_status == ReplicationRunningStatus.Split: + if ( + secres_access == SecresAccess.ReadAndWrite + and replication_pair_running_status == ReplicationRunningStatus.Split + ): LOG.info("Create remote replication pair success.") self.record_deploy_process(exec_step, "success") return True self.replication_status_check_and_sync(exec_step, remote_replication_pair_info) time.sleep(20) - LOG.info("Sync remote replication filesystem pair[%s], health status:[%s], " - "running status[%s], progress[%s%%], start time[%s]", - replication_pair_id, - get_status(replication_pair_health_status, HealthStatus), - get_status(replication_pair_running_status, ReplicationRunningStatus), - replication_progress, - datetime.datetime.fromtimestamp(int(start_time))) - if replication_progress == "100" and \ - replication_pair_running_status == ReplicationRunningStatus.Normal and \ - replication_pair_health_status == HealthStatus.Normal: - LOG.info("Success to sync remote replication filesystem pair[%s], end time[%s]", - replication_pair_id, - datetime.datetime.fromtimestamp(int(end_time))) + LOG.info( + "Sync remote replication filesystem pair[%s], health status:[%s], " + "running status[%s], progress[%s%%], start time[%s]", + replication_pair_id, + get_status(replication_pair_health_status, HealthStatus), + get_status(replication_pair_running_status, ReplicationRunningStatus), + replication_progress, + datetime.datetime.fromtimestamp(int(start_time)), + ) + if ( + replication_progress == "100" + and replication_pair_running_status == ReplicationRunningStatus.Normal + and replication_pair_health_status == HealthStatus.Normal + ): + LOG.info( + "Success to sync remote replication filesystem pair[%s], end time[%s]", + replication_pair_id, + datetime.datetime.fromtimestamp(int(end_time)), + ) if int(start_time) - int(end_time) > Constant.FULL_SYNC_MAX_TIME: - LOG.info("Do sync remote replication filesystem[%s] pair of full copy." % replication_pair_id) - self.dr_deploy_opt.sync_remote_replication_filesystem_pair(pair_id=replication_pair_id, vstore_id=0, - is_full_copy=False) + LOG.info( + "Do sync remote replication filesystem[%s] pair of full copy." + % replication_pair_id + ) + self.dr_deploy_opt.sync_remote_replication_filesystem_pair( + pair_id=replication_pair_id, vstore_id=0, is_full_copy=False + ) return False self.record_deploy_process(exec_step, "success") return True - if replication_pair_health_status != ReplicationRunningStatus.Normal or \ - replication_pair_running_status not in \ - [ReplicationRunningStatus.Normal, ReplicationRunningStatus.Synchronizing]: - err_msg = "Failed to sync remote replication filesystem[%s] pair." % replication_pair_id - self.record_deploy_process(exec_step, "failed", code=-1, description=err_msg) + if ( + replication_pair_health_status != ReplicationRunningStatus.Normal + or replication_pair_running_status + not in [ + ReplicationRunningStatus.Normal, + ReplicationRunningStatus.Synchronizing, + ] + ): + err_msg = ( + "Failed to sync remote replication filesystem[%s] pair." + % replication_pair_id + ) + self.record_deploy_process( + exec_step, "failed", code=-1, description=err_msg + ) raise Exception(err_msg) self.record_deploy_process(exec_step, replication_progress + "%") return False - def replication_status_check_and_sync(self, exec_step, remote_replication_pair_info): + def replication_status_check_and_sync( + self, exec_step, remote_replication_pair_info + ): """ 检查复制pair对状态，并进行同步 :param exec_step: @@ -666,37 +832,67 @@ class DRDeploy(object): """ replication_pair_id = remote_replication_pair_info.get("ID") start_time = remote_replication_pair_info.get("STARTTIME") - replication_pair_health_status = remote_replication_pair_info.get("HEALTHSTATUS") - replication_pair_running_status = remote_replication_pair_info.get("RUNNINGSTATUS") + replication_pair_health_status = remote_replication_pair_info.get( + "HEALTHSTATUS" + ) + replication_pair_running_status = remote_replication_pair_info.get( + "RUNNINGSTATUS" + ) # 当复制pair对健康状态为非正常状态，并且running状态不为正常、正在同步、待恢复状态，异常退出 - if replication_pair_health_status != HealthStatus.Normal and \ - replication_pair_running_status not in [ReplicationRunningStatus.Normal, - ReplicationRunningStatus.Synchronizing, - ReplicationRunningStatus.TobeRecovered]: - err_msg = "Current replication pair health is not normal, " \ - "current status: %s, running status:%s, filesystem: %s" % \ - (get_status(replication_pair_health_status, HealthStatus), - get_status(replication_pair_running_status, ReplicationRunningStatus), - replication_pair_id) - self.record_deploy_process(exec_step, "failed", code=-1, description=err_msg) + if ( + replication_pair_health_status != HealthStatus.Normal + and replication_pair_running_status + not in [ + ReplicationRunningStatus.Normal, + ReplicationRunningStatus.Synchronizing, + ReplicationRunningStatus.TobeRecovered, + ] + ): + err_msg = ( + "Current replication pair health is not normal, " + "current status: %s, running status:%s, filesystem: %s" + % ( + get_status(replication_pair_health_status, HealthStatus), + get_status( + replication_pair_running_status, ReplicationRunningStatus + ), + replication_pair_id, + ) + ) + self.record_deploy_process( + exec_step, "failed", code=-1, description=err_msg + ) LOG.error(err_msg) raise Exception(err_msg) # 当前远程复制pair对状态为分裂且没有同步开始时间时，表示当前为首次创建还未同步，执行全量同步 - if replication_pair_running_status == ReplicationRunningStatus.Split and start_time is None: - LOG.info("Do sync remote replication filesystem[%s] pair of full copy." % replication_pair_id) - self.dr_deploy_opt.sync_remote_replication_filesystem_pair(pair_id=replication_pair_id, - vstore_id="0", - is_full_copy=True) + if ( + replication_pair_running_status == ReplicationRunningStatus.Split + and start_time is None + ): + LOG.info( + "Do sync remote replication filesystem[%s] pair of full copy." + % replication_pair_id + ) + self.dr_deploy_opt.sync_remote_replication_filesystem_pair( + pair_id=replication_pair_id, vstore_id="0", is_full_copy=True + ) # 当前远程复制pair对状态为分裂且有同步开始时间时，表示当前为首次创建还未同步，执行增量同步 - if replication_pair_running_status in \ - [ReplicationRunningStatus.Split, ReplicationRunningStatus.TobeRecovered] \ - and start_time is not None: - LOG.info("Do sync remote replication filesystem[%s] pair of incremental." % replication_pair_id) - self.dr_deploy_opt.sync_remote_replication_filesystem_pair(pair_id=replication_pair_id, - vstore_id="0", - is_full_copy=False) + if ( + replication_pair_running_status + in [ReplicationRunningStatus.Split, ReplicationRunningStatus.TobeRecovered] + and start_time is not None + ): + LOG.info( + "Do sync remote replication filesystem[%s] pair of incremental." + % replication_pair_id + ) + self.dr_deploy_opt.sync_remote_replication_filesystem_pair( + pair_id=replication_pair_id, vstore_id="0", is_full_copy=False + ) - def do_remote_replication_filesystem_pair_cancel_secondary_write_lock(self, pair_id: str, is_page: bool) -> None: + def do_remote_replication_filesystem_pair_cancel_secondary_write_lock( + self, pair_id: str, is_page: bool + ) -> None: """ 远程复制pair对分裂后取消从端写锁 1、 查询pair对状态 @@ -705,18 +901,31 @@ class DRDeploy(object): :param is_page: 是否是page文件系统，否则为meta文件系统 :param pair_id: 远程复制pair对id """ - exec_step = "cancel_rep_meta_fs_secondary_write_lock" if not is_page \ + exec_step = ( + "cancel_rep_meta_fs_secondary_write_lock" + if not is_page else "cancel_rep_page_fs_secondary_write_lock" + ) self.record_deploy_process(exec_step, "start") - rep_pair_info = self.dr_deploy_opt.query_remote_replication_pair_info_by_pair_id(pair_id) + rep_pair_info = ( + self.dr_deploy_opt.query_remote_replication_pair_info_by_pair_id(pair_id) + ) self.record_deploy_process(exec_step, "running") secres_access = rep_pair_info.get("SECRESACCESS") running_status = rep_pair_info.get("RUNNINGSTATUS") - if secres_access == SecresAccess.ReadAndWrite and running_status == ReplicationRunningStatus.Split: - LOG.info("Current replicantion pair status already is[%s].", get_status(secres_access, SecresAccess)) + if ( + secres_access == SecresAccess.ReadAndWrite + and running_status == ReplicationRunningStatus.Split + ): + LOG.info( + "Current replicantion pair status already is[%s].", + get_status(secres_access, SecresAccess), + ) self.record_deploy_process(exec_step, "success") return - self.dr_deploy_opt.remote_replication_filesystem_pair_cancel_secondary_write_lock(pair_id) + self.dr_deploy_opt.remote_replication_filesystem_pair_cancel_secondary_write_lock( + pair_id + ) self.record_deploy_process(exec_step, "success") def deploy_remote_replication_pair(self, fs_name: str, is_page: bool) -> str: @@ -733,15 +942,24 @@ class DRDeploy(object): :param fs_name: 文件系统名 :return: """ - exec_step = "create_rep_meta_fs_pair" if not is_page else "create_rep_page_fs_pair" + exec_step = ( + "create_rep_meta_fs_pair" if not is_page else "create_rep_page_fs_pair" + ) self.record_deploy_process(exec_step, "start") LOG.info("Start to create [%s]remote replication pair success.", fs_name) - LOG.info("Create remote replication pair step 1: query filesystem[%s] info.", fs_name) + LOG.info( + "Create remote replication pair step 1: query filesystem[%s] info.", fs_name + ) fs_info = self.dr_deploy_opt.storage_opt.query_filesystem_info(fs_name) fs_id = fs_info.get("ID") - LOG.info("Create remote replication pair step 2: create filesystem[%s] pair.", fs_name) + LOG.info( + "Create remote replication pair step 2: create filesystem[%s] pair.", + fs_name, + ) self.record_deploy_process(exec_step, "running") - remote_replication_pair_info = self.do_create_remote_replication_filesystem_pair(fs_id) + remote_replication_pair_info = ( + self.do_create_remote_replication_filesystem_pair(fs_id) + ) replication_pair_id = remote_replication_pair_info[0].get("ID") key = "page_fs_pair_id" if is_page else "meta_fs_pair_id" self.record_disaster_recovery_info(key, replication_pair_id) @@ -762,24 +980,35 @@ class DRDeploy(object): try: domain_info = self.do_create_filesystem_hyper_metro_domain() except Exception as err: - self.record_deploy_process("create_metro_domain", "failed", code=-1, description=str(err)) + self.record_deploy_process( + "create_metro_domain", "failed", code=-1, description=str(err) + ) raise err self.record_disaster_recovery_info("hyper_domain_id", domain_info.get("ID")) try: vstore_pair_info = self.do_create_hyper_metro_vstore_pair(domain_info) except Exception as err: - self.record_deploy_process("create_metro_vstore_pair", "failed", code=-1, description=str(err)) + self.record_deploy_process( + "create_metro_vstore_pair", "failed", code=-1, description=str(err) + ) raise err self.record_disaster_recovery_info("vstore_pair_id", vstore_pair_info.get("ID")) try: - filesystem_pair_info = self.do_create_hyper_metro_filesystem_pair(vstore_pair_info) + filesystem_pair_info = self.do_create_hyper_metro_filesystem_pair( + vstore_pair_info + ) except Exception as err: - self.record_deploy_process("create_metro_fs_pair", "failed", code=-1, description=str(err)) + self.record_deploy_process( + "create_metro_fs_pair", "failed", code=-1, description=str(err) + ) raise err self.ulog_fs_pair_id = filesystem_pair_info.get("ID") - self.dr_deploy_opt.modify_hyper_metro_filesystem_pair_sync_speed(vstore_pair_id=self.ulog_fs_pair_id, - speed=self.sync_speed) - self.record_disaster_recovery_info("ulog_fs_pair_id", filesystem_pair_info.get("ID")) + self.dr_deploy_opt.modify_hyper_metro_filesystem_pair_sync_speed( + vstore_pair_id=self.ulog_fs_pair_id, speed=self.sync_speed + ) + self.record_disaster_recovery_info( + "ulog_fs_pair_id", filesystem_pair_info.get("ID") + ) def query_cantian_disaster_recovery_status(self): """ @@ -789,35 +1018,60 @@ class DRDeploy(object): """ self.record_deploy_process("cantian_disaster_recovery_status", "start") node_id = self.deploy_params.get("node_id") - cms_cmd = "su -s /bin/bash - %s -c 'source ~/.bashrc " \ - "&& cms stat | awk \"{print \$1, \$9}\"'" % self.run_user + cms_cmd = ( + "su -s /bin/bash - %s -c 'source ~/.bashrc " + '&& cms stat | awk "{print \$1, \$9}"\'' % self.run_user + ) return_code, output, stderr = exec_popen(cms_cmd) LOG.info("Check cms reformer node.") if return_code: - err_msg = "Execute cms command[%s] query reform node failed, output:%s, " \ - "stderr:%s" % (cms_cmd, output, stderr) - self.record_deploy_process("cantian_disaster_recovery_status", "failed", - code=-1, description=err_msg) + err_msg = ( + "Execute cms command[%s] query reform node failed, output:%s, " + "stderr:%s" % (cms_cmd, output, stderr) + ) + self.record_deploy_process( + "cantian_disaster_recovery_status", + "failed", + code=-1, + description=err_msg, + ) raise Exception(err_msg) cms_stat = output.split("\n") LOG.info("Cms stat is:\n %s", cms_stat) LOG.info("Check cantian replay status.") self.record_deploy_process("cantian_disaster_recovery_status", "running") for node_stat in cms_stat: - if "REFORMER" in node_stat and node_id == node_stat.split(" ")[0].strip(" "): - return_code, output, stderr = exec_popen(CANTIAN_DISASTER_RECOVERY_STATUS_CHECK, timeout=20) + if "REFORMER" in node_stat and node_id == node_stat.split(" ")[0].strip( + " " + ): + return_code, output, stderr = exec_popen( + CANTIAN_DISASTER_RECOVERY_STATUS_CHECK, timeout=20 + ) if return_code: - err_msg = "Execute check cantian disaster recovery command failed, " \ - "oupout:%s, stderr:%s" % (output, stderr) - self.record_deploy_process("cantian_disaster_recovery_status", "failed", - code=-1, description=err_msg) + err_msg = ( + "Execute check cantian disaster recovery command failed, " + "oupout:%s, stderr:%s" % (output, stderr) + ) + self.record_deploy_process( + "cantian_disaster_recovery_status", + "failed", + code=-1, + description=err_msg, + ) LOG.info("Check cantian replay failed.") raise Exception(err_msg) if "START_REPLAY" not in output: - err_msg = "Cantian lrpl status is abnormal, details: %s" % output.split("SQL>")[1:] - self.record_deploy_process("cantian_disaster_recovery_status", "failed", code=-1, - description=err_msg) + err_msg = ( + "Cantian lrpl status is abnormal, details: %s" + % output.split("SQL>")[1:] + ) + self.record_deploy_process( + "cantian_disaster_recovery_status", + "failed", + code=-1, + description=err_msg, + ) LOG.info("Check cantian replay failed.") raise Exception(err_msg) @@ -841,9 +1095,13 @@ class DRDeploy(object): self.update_install_status(node_id, "start", "failed") err_pattern = re.compile(".*ERROR.*") _err = err_pattern.findall(output + stderr) - err_msg = "Failed to execute start, details:\n%s for details see " \ - "/opt/cantian/log/deploy/deploy.log" % "\n".join(_err) - self.record_deploy_process("standby_start", "failed", code=-1, description=err_msg) + err_msg = ( + "Failed to execute start, details:\n%s for details see " + "/opt/cantian/log/deploy/deploy.log" % "\n".join(_err) + ) + self.record_deploy_process( + "standby_start", "failed", code=-1, description=err_msg + ) raise Exception(err_msg) self.update_install_status(node_id, "start", "success") LOG.info("Start node[%s] cantian success.", node_id) @@ -866,13 +1124,13 @@ class DRDeploy(object): if os.path.exists(install_record_file): with open(install_record_file, "r") as fp: install_status = json.loads(fp.read()) - with os.fdopen(os.open(install_record_file, flags, modes), 'w') as fp: + with os.fdopen(os.open(install_record_file, flags, modes), "w") as fp: install_status.update({exec_step: exec_status}) json.dump(install_status, fp, indent=4) else: data = {"install": "default", "start": "default", "stop": "default"} data.update({exec_step: exec_status}) - with os.fdopen(os.open(install_record_file, flags, modes), 'w') as fp: + with os.fdopen(os.open(install_record_file, flags, modes), "w") as fp: json.dump(data, fp, indent=4) LOG.info("Update %s status[%s] success", exec_step, exec_status) @@ -903,15 +1161,21 @@ class DRDeploy(object): dbstor_fs_name = self.dr_deploy_info.get("storage_dbstor_fs") dbstor_fs_vstore_id = self.dr_deploy_info.get("dbstor_fs_vstore_id") dbstor_fs_info = self.dr_deploy_opt.storage_opt.query_filesystem_info( - dbstor_fs_name, vstore_id=dbstor_fs_vstore_id) + dbstor_fs_name, vstore_id=dbstor_fs_vstore_id + ) ulog_fs_pair_info = None if dbstor_fs_info and not ulog_fs_pair_ready_flag: dbstor_fs_id = dbstor_fs_info.get("ID") try: - ulog_fs_pair_info = self.dr_deploy_opt.query_hyper_metro_filesystem_pair_info(dbstor_fs_id) + ulog_fs_pair_info = ( + self.dr_deploy_opt.query_hyper_metro_filesystem_pair_info( + dbstor_fs_id + ) + ) except Exception as err: - self.record_deploy_process("create_metro_fs_pair", "failed", - code=-1, description=str(err)) + self.record_deploy_process( + "create_metro_fs_pair", "failed", code=-1, description=str(err) + ) raise err self.record_deploy_process("create_metro_domain", "success") self.record_deploy_process("create_metro_vstore_pair", "success") @@ -927,13 +1191,19 @@ class DRDeploy(object): self.record_disaster_recovery_info("hyper_domain_id", hyper_domain_id) self.record_disaster_recovery_info("vstore_pair_id", vstore_pair_id) if running_status == FilesystemPairRunningStatus.Paused: - self.record_deploy_process("sync_metro_fs_pair", - get_status(running_status, FilesystemPairRunningStatus)) + self.record_deploy_process( + "sync_metro_fs_pair", + get_status(running_status, FilesystemPairRunningStatus), + ) else: - self.record_deploy_process("sync_metro_fs_pair", sync_progress + "%") - if running_status == FilesystemPairRunningStatus.Normal \ - and health_status == HealthStatus.Normal \ - and sync_progress == "100": + self.record_deploy_process( + "sync_metro_fs_pair", sync_progress + "%" + ) + if ( + running_status == FilesystemPairRunningStatus.Normal + and health_status == HealthStatus.Normal + and sync_progress == "100" + ): LOG.info("Hyper metro filesystem[%s] pair ready", dbstor_fs_name) self.record_deploy_process("sync_metro_fs_pair", "success") ulog_fs_pair_ready_flag = True @@ -947,23 +1217,36 @@ class DRDeploy(object): """ dbstor_page_fs_name = self.dr_deploy_info.get("storage_dbstor_page_fs") dbstor_page_fs_info = self.dr_deploy_opt.storage_opt.query_filesystem_info( - dbstor_page_fs_name) + dbstor_page_fs_name + ) page_fs_pair_info = None if dbstor_page_fs_info and not page_fs_pair_ready_flag: self.record_deploy_process("create_rep_page_fs_pair", "success") dbstor_page_fs_id = dbstor_page_fs_info.get("ID") - page_fs_pair_info = self.dr_deploy_opt.query_remote_replication_pair_info(dbstor_page_fs_id) + page_fs_pair_info = self.dr_deploy_opt.query_remote_replication_pair_info( + dbstor_page_fs_id + ) if page_fs_pair_info: # 当已经设置从端可读写状态，且为分裂状态时，表示当前同步完成 page_fs_pair_id = page_fs_pair_info[0].get("ID") secres_access = page_fs_pair_info[0].get("SECRESACCESS") running_status = page_fs_pair_info[0].get("RUNNINGSTATUS") self.record_disaster_recovery_info("page_fs_pair_id", page_fs_pair_id) - remote_replication_pair_info = self.dr_deploy_opt.query_remote_replication_pair_info_by_pair_id( - page_fs_pair_id) - replication_progress = remote_replication_pair_info.get("REPLICATIONPROGRESS") - self.record_deploy_process("sync_rep_page_fs_pair", str(replication_progress) + "%") - if secres_access == SecresAccess.ReadAndWrite and running_status == ReplicationRunningStatus.Split: + remote_replication_pair_info = ( + self.dr_deploy_opt.query_remote_replication_pair_info_by_pair_id( + page_fs_pair_id + ) + ) + replication_progress = remote_replication_pair_info.get( + "REPLICATIONPROGRESS" + ) + self.record_deploy_process( + "sync_rep_page_fs_pair", str(replication_progress) + "%" + ) + if ( + secres_access == SecresAccess.ReadAndWrite + and running_status == ReplicationRunningStatus.Split + ): LOG.info("Remote replication pair[%s] ready.", dbstor_page_fs_name) self.record_deploy_process("sync_rep_page_fs_pair", "success") page_fs_pair_ready_flag = True @@ -978,26 +1261,47 @@ class DRDeploy(object): mysql_metadata_in_cantian = self.dr_deploy_info.get("mysql_metadata_in_cantian") metadata_fs_name = self.dr_deploy_info.get("storage_metadata_fs") metadata_fs_info = self.dr_deploy_opt.storage_opt.query_filesystem_info( - metadata_fs_name) - metadata_fs_pair_info = None if not mysql_metadata_in_cantian else metadata_fs_info - if metadata_fs_info and not mysql_metadata_in_cantian and not metadata_fs_ready_flag: + metadata_fs_name + ) + metadata_fs_pair_info = ( + None if not mysql_metadata_in_cantian else metadata_fs_info + ) + if ( + metadata_fs_info + and not mysql_metadata_in_cantian + and not metadata_fs_ready_flag + ): metadata_fs_id = metadata_fs_info.get("ID") - metadata_fs_pair_info = self.dr_deploy_opt.query_remote_replication_pair_info(metadata_fs_id) + metadata_fs_pair_info = ( + self.dr_deploy_opt.query_remote_replication_pair_info(metadata_fs_id) + ) self.record_deploy_process("create_rep_meta_fs_pair", "success") if metadata_fs_pair_info: # 当已经设置从端可读写状态，且为分裂状态时，表示当前同步完成 meta_fs_pair_id = metadata_fs_pair_info[0].get("ID") secres_access = metadata_fs_pair_info[0].get("SECRESACCESS") running_status = metadata_fs_pair_info[0].get("RUNNINGSTATUS") - remote_replication_pair_info = self.dr_deploy_opt. \ - query_remote_replication_pair_info_by_pair_id(meta_fs_pair_id) - replication_progress = remote_replication_pair_info.get("REPLICATIONPROGRESS") - self.record_deploy_process("sync_rep_meta_fs_pair", str(replication_progress) + "%") - if secres_access == SecresAccess.ReadAndWrite and running_status == ReplicationRunningStatus.Split: + remote_replication_pair_info = ( + self.dr_deploy_opt.query_remote_replication_pair_info_by_pair_id( + meta_fs_pair_id + ) + ) + replication_progress = remote_replication_pair_info.get( + "REPLICATIONPROGRESS" + ) + self.record_deploy_process( + "sync_rep_meta_fs_pair", str(replication_progress) + "%" + ) + if ( + secres_access == SecresAccess.ReadAndWrite + and running_status == ReplicationRunningStatus.Split + ): LOG.info("Remote replication pair[%s] ready.", metadata_fs_name) self.record_deploy_process("sync_rep_meta_fs_pair", "success") metadata_fs_ready_flag = True - self.record_disaster_recovery_info("meta_fs_pair_id", meta_fs_pair_id) + self.record_disaster_recovery_info( + "meta_fs_pair_id", meta_fs_pair_id + ) return metadata_fs_pair_info, metadata_fs_ready_flag, metadata_fs_info def create_nfs_share_and_client(self, fs_info: dict) -> None: @@ -1012,11 +1316,7 @@ class DRDeploy(object): return fs_id = fs_info.get("ID") fs_name = fs_info.get("NAME") - share_data = { - "SHAREPATH": f"/{fs_name}/", - "vstoreId": "0", - "FSID": fs_id - } + share_data = {"SHAREPATH": f"/{fs_name}/", "vstoreId": "0", "FSID": fs_id} share_info = self.dr_deploy_opt.storage_opt.query_nfs_info(fs_id) if not share_info: parent_id = self.dr_deploy_opt.storage_opt.create_nfs_share(share_data) @@ -1028,9 +1328,11 @@ class DRDeploy(object): "ROOTSQUASH": 1, "PARENTID": parent_id, "vstoreId": "0", - "NAME": "*" + "NAME": "*", } - client_info = self.dr_deploy_opt.storage_opt.query_nfs_share_auth_client(parent_id) + client_info = self.dr_deploy_opt.storage_opt.query_nfs_share_auth_client( + parent_id + ) if not client_info: self.dr_deploy_opt.storage_opt.add_nfs_client(client_data) @@ -1059,17 +1361,26 @@ class DRDeploy(object): cert_encrypt_pwd = "" if self.dr_deploy_info.get("mes_ssl_switch"): cert_encrypt_pwd = input() - cmd = "echo -e \"%s\\n%s\\n%s\\n%s\\n%s\"|sh %s/install.sh %s" \ - % (dbstor_user, dbstor_pwd, - cantian_pwd, comfirm_cantian_pwd, cert_encrypt_pwd, - ctl_file_path, DEFAULT_PARAM_FILE) + cmd = 'echo -e "%s\\n%s\\n%s\\n%s\\n%s"|sh %s/install.sh %s' % ( + dbstor_user, + dbstor_pwd, + cantian_pwd, + comfirm_cantian_pwd, + cert_encrypt_pwd, + ctl_file_path, + DEFAULT_PARAM_FILE, + ) _, output, stderr = exec_popen(cmd, timeout=600) if "install success" not in output: err_pattern = re.compile(".*ERROR.*") _err = err_pattern.findall(output + stderr) - err_msg = "Failed to execute install, details:\n%s, for details see " \ - "/opt/cantian/log/deploy/deploy.log" % "\n".join(_err) - self.record_deploy_process("standby_install", "failed", code=-1, description=err_msg) + err_msg = ( + "Failed to execute install, details:\n%s, for details see " + "/opt/cantian/log/deploy/deploy.log" % "\n".join(_err) + ) + self.record_deploy_process( + "standby_install", "failed", code=-1, description=err_msg + ) self.update_install_status(node_id, "install", "failed") raise Exception(err_msg) self.update_install_status(node_id, "install", "success") @@ -1080,7 +1391,9 @@ class DRDeploy(object): if self.dr_deploy_info.get("cantian_in_container") == "1": return # 判断是否是单进程，单进程要安装mysql - install_json_path = os.path.join(CURRENT_PATH, "../../cantian/install_config.json") + install_json_path = os.path.join( + CURRENT_PATH, "../../cantian/install_config.json" + ) install_json_data = read_json_config(install_json_path) if install_json_data.get("M_RUNING_MODE") == "cantiand_with_mysql_in_cluster": # 单进程需要在拉起前安装mysql @@ -1106,40 +1419,66 @@ class DRDeploy(object): ulog_ready, page_ready, meta_ready = True, True, True while True: try: - ulog_ready = self.do_sync_hyper_metro_filesystem_pair(self.ulog_fs_pair_id) + ulog_ready = self.do_sync_hyper_metro_filesystem_pair( + self.ulog_fs_pair_id + ) except Exception as err: - self.record_deploy_process("sync_metro_fs_pair", "failed", code=-1, description=str(err)) + self.record_deploy_process( + "sync_metro_fs_pair", "failed", code=-1, description=str(err) + ) raise err try: - page_ready = self.do_sync_remote_replication_filesystem_pair(self.page_fs_pair_id, True) + page_ready = self.do_sync_remote_replication_filesystem_pair( + self.page_fs_pair_id, True + ) except Exception as err: - self.record_deploy_process("sync_rep_page_fs_pair", "failed", code=-1, description=str(err)) + self.record_deploy_process( + "sync_rep_page_fs_pair", "failed", code=-1, description=str(err) + ) raise err if not self.metadata_in_cantian: try: - meta_ready = self.do_sync_remote_replication_filesystem_pair(self.meta_fs_pair_id, False) + meta_ready = self.do_sync_remote_replication_filesystem_pair( + self.meta_fs_pair_id, False + ) except Exception as err: - self.record_deploy_process("sync_rep_meta_fs_pair", "failed", code=-1, description=str(err)) + self.record_deploy_process( + "sync_rep_meta_fs_pair", "failed", code=-1, description=str(err) + ) raise err if ulog_ready and page_ready and meta_ready: break time.sleep(60) try: - self.dr_deploy_opt.split_remote_replication_filesystem_pair(self.page_fs_pair_id) + self.dr_deploy_opt.split_remote_replication_filesystem_pair( + self.page_fs_pair_id + ) self.do_remote_replication_filesystem_pair_cancel_secondary_write_lock( - self.page_fs_pair_id, True) + self.page_fs_pair_id, True + ) except Exception as err: - self.record_deploy_process("cancel_rep_page_fs_secondary_write_lock", - "failed", code=-1, description=str(err)) + self.record_deploy_process( + "cancel_rep_page_fs_secondary_write_lock", + "failed", + code=-1, + description=str(err), + ) raise err if not self.metadata_in_cantian: try: - self.dr_deploy_opt.split_remote_replication_filesystem_pair(self.meta_fs_pair_id) + self.dr_deploy_opt.split_remote_replication_filesystem_pair( + self.meta_fs_pair_id + ) self.do_remote_replication_filesystem_pair_cancel_secondary_write_lock( - self.meta_fs_pair_id, False) + self.meta_fs_pair_id, False + ) except Exception as err: - self.record_deploy_process("cancel_rep_meta_fs_secondary_write_lock", - "failed", code=-1, description=str(err)) + self.record_deploy_process( + "cancel_rep_meta_fs_secondary_write_lock", + "failed", + code=-1, + description=str(err), + ) raise err def copy_param_file_to_metadata(self): @@ -1159,7 +1498,9 @@ class DRDeploy(object): if return_code: err_msg = f"Execution of chown command failed, output: {output}, stderr: {stderr}" LOG.error(err_msg) - self.record_deploy_process("dr_deploy", "failed", code=-1, description=err_msg) + self.record_deploy_process( + "dr_deploy", "failed", code=-1, description=err_msg + ) raise Exception(err_msg) dbstor_del_command = ( f'su -s /bin/bash - "{run_user}" -c \'' @@ -1182,7 +1523,9 @@ class DRDeploy(object): if return_code: err_msg = f"Execution of dbstor command failed, output: {output}, stderr: {stderr}" LOG.error(err_msg) - self.record_deploy_process("dr_deploy", "failed", code=-1, description=err_msg) + self.record_deploy_process( + "dr_deploy", "failed", code=-1, description=err_msg + ) raise Exception(err_msg) LOG.info(f"Successfully executed: {dbstor_command}") @@ -1193,7 +1536,10 @@ class DRDeploy(object): if os.path.exists(config_path): # 删除文件 os.remove(config_path) - shutil.copy(os.path.join(CURRENT_PATH, "../../../config/dr_deploy_param.json"), share_path) + shutil.copy( + os.path.join(CURRENT_PATH, "../../../config/dr_deploy_param.json"), + share_path, + ) except Exception as _err: LOG.info(f"copy dr_deploy_param failed") @@ -1216,7 +1562,9 @@ class DRDeploy(object): if not self.mysql_cmd or not self.mysql_user: err_msg = "Mysql_pwd or mysql_user is None, please check." LOG.error(err_msg) - self.record_deploy_process("dr_deploy", "failed", code=-1, description=err_msg) + self.record_deploy_process( + "dr_deploy", "failed", code=-1, description=err_msg + ) raise Exception(err_msg) if self.mysql_pwd is None: self.mysql_pwd = input() @@ -1227,18 +1575,28 @@ class DRDeploy(object): self.do_flush_table_with_read_lock() dbstor_page_fs_name = self.dr_deploy_info.get("storage_dbstor_page_fs") metadata_fs_name = self.dr_deploy_info.get("storage_metadata_fs") - self.sync_speed = int(SPEED.get(self.dr_deploy_info.get("sync_speed", "medium"))) + self.sync_speed = int( + SPEED.get(self.dr_deploy_info.get("sync_speed", "medium")) + ) self.deploy_hyper_metro_pair() try: - self.page_fs_pair_id = self.deploy_remote_replication_pair(dbstor_page_fs_name, True) + self.page_fs_pair_id = self.deploy_remote_replication_pair( + dbstor_page_fs_name, True + ) except Exception as err: - self.record_deploy_process("create_rep_page_fs_pair", "failed", code=-1, description=str(err)) + self.record_deploy_process( + "create_rep_page_fs_pair", "failed", code=-1, description=str(err) + ) raise err if not self.metadata_in_cantian: try: - self.meta_fs_pair_id = self.deploy_remote_replication_pair(metadata_fs_name, False) + self.meta_fs_pair_id = self.deploy_remote_replication_pair( + metadata_fs_name, False + ) except Exception as err: - self.record_deploy_process("create_rep_meta_fs_pair", "failed", code=-1, description=str(err)) + self.record_deploy_process( + "create_rep_meta_fs_pair", "failed", code=-1, description=str(err) + ) raise err self.active_dr_deploy_and_sync() self.do_unlock_instance_for_backup() @@ -1263,20 +1621,27 @@ class DRDeploy(object): wait_time = 0 metadata_fs_info = None while True: - ulog_fs_pair_info, ulog_fs_pair_ready_flag = \ + ulog_fs_pair_info, ulog_fs_pair_ready_flag = ( self.standby_check_ulog_fs_pair_ready(ulog_fs_pair_ready_flag) - page_fs_pair_info, page_fs_pair_ready_flag = \ + ) + page_fs_pair_info, page_fs_pair_ready_flag = ( self.standby_check_page_fs_pair_ready(page_fs_pair_ready_flag) + ) if self.deploy_mode != "dbstor": - metadata_fs_pair_info, metadata_fs_ready_flag, metadata_fs_info = \ + metadata_fs_pair_info, metadata_fs_ready_flag, metadata_fs_info = ( self.standby_check_metadata_fs_pair_ready(metadata_fs_ready_flag) - fs_ready = ulog_fs_pair_info and page_fs_pair_info and metadata_fs_pair_info + ) + fs_ready = ( + ulog_fs_pair_info and page_fs_pair_info and metadata_fs_pair_info + ) else: fs_ready = ulog_fs_pair_info and page_fs_pair_info if fs_ready and not is_installed_flag: - LOG.info("Filesystem created successfully, start to install Cantian engine.") + LOG.info( + "Filesystem created successfully, start to install Cantian engine." + ) self.record_deploy_process("standby_install", "running") if self.deploy_mode != "dbstor" and metadata_fs_info: @@ -1289,17 +1654,28 @@ class DRDeploy(object): else: if wait_time > FS_CREAT_TIMEOUT and not is_installed_flag: err_msg = "Wait for the filesystem creation timeout, please check." - self.record_deploy_process("standby_install", "failed", code=-1, description=err_msg) + self.record_deploy_process( + "standby_install", "failed", code=-1, description=err_msg + ) LOG.error(err_msg) raise Exception(err_msg) - LOG.info("Waiting until the DR is successfully set up, waited [%s]s", wait_time) - pair_ready = ulog_fs_pair_ready_flag and page_fs_pair_ready_flag and metadata_fs_ready_flag + LOG.info( + "Waiting until the DR is successfully set up, waited [%s]s", + wait_time, + ) + pair_ready = ( + ulog_fs_pair_ready_flag + and page_fs_pair_ready_flag + and metadata_fs_ready_flag + ) if is_installed_flag and pair_ready: self.record_deploy_process("standby_start", "running") try: self.standby_do_start() except Exception as err: - self.record_deploy_process("standby_start", "failed", code=-1, description=str(err)) + self.record_deploy_process( + "standby_start", "failed", code=-1, description=str(err) + ) raise err self.record_deploy_process("standby_start", "success") break @@ -1315,9 +1691,12 @@ class DRDeploy(object): deploy --site=[standby/active] --mysql_cmd='/usr/local/mysql/bin/mysql' --mysql_user=root :return: """ + def _execute(): action_parse = argparse.ArgumentParser() - action_parse.add_argument("--site", dest="site", choices=["standby", "active"], required=True) + action_parse.add_argument( + "--site", dest="site", choices=["standby", "active"], required=True + ) action_parse.add_argument("--mysql_cmd", dest="mysql_cmd", required=False) action_parse.add_argument("--mysql_user", dest="mysql_user", required=False) args = action_parse.parse_args() @@ -1335,23 +1714,34 @@ class DRDeploy(object): else: self.standby_execute() except Exception as err: - self.record_deploy_process("dr_deploy", "failed", code=-1, description=str(err)) + self.record_deploy_process( + "dr_deploy", "failed", code=-1, description=str(err) + ) if self.backup_lock_shell is not None: self.do_unlock_instance_for_backup() - LOG.error("Dr deploy execute failed, traceback:%s", traceback.format_exc()) + LOG.error( + "Dr deploy execute failed, traceback:%s", traceback.format_exc() + ) raise err finally: self.dr_deploy_opt.storage_opt.logout() # 安装部署完成后记录加密密码到配置文件 encrypted_pwd = KmcResolve.kmc_resolve_password("encrypted", self.dm_passwd) self.record_disaster_recovery_info("dm_pwd", encrypted_pwd) - os.chmod(os.path.join(CURRENT_PATH, "../../../config/dr_deploy_param.json"), mode=0o644) + os.chmod( + os.path.join(CURRENT_PATH, "../../../config/dr_deploy_param.json"), + mode=0o644, + ) try: - shutil.copy(os.path.join(CURRENT_PATH, "../../../config/dr_deploy_param.json"), "/opt/cantian/config/") + shutil.copy( + os.path.join(CURRENT_PATH, "../../../config/dr_deploy_param.json"), + "/opt/cantian/config/", + ) except Exception as _err: LOG.info(f"copy dr_deploy_param failed") self.copy_param_file_to_metadata() self.restart_cantian_exporter() + try: _execute() except Exception as err: diff --git a/pkg/deploy/action/storage_operate/dr_deploy_operate/dr_deploy_common.py b/pkg/deploy/action/storage_operate/dr_deploy_operate/dr_deploy_common.py index e3030c51..8ebc68fa 100644 --- a/pkg/deploy/action/storage_operate/dr_deploy_operate/dr_deploy_common.py +++ b/pkg/deploy/action/storage_operate/dr_deploy_operate/dr_deploy_common.py @@ -27,9 +27,7 @@ class RemoteStorageOPT(object): "device_id": self.remote_device_id, "url": Constant.CREATE_VSTORE.replace("{deviceId}", "xxx"), "method": "GET", - "body": { - "ID": f"{vstore_id}" - } + "body": {"ID": f"{vstore_id}"}, } res = self.rest_client.normal_request(url, data=data, method="post") err_msg = "Failed to query remote storage vstore info,vstoreId[%s]" % vstore_id @@ -44,7 +42,7 @@ class RemoteStorageOPT(object): "device_id": self.remote_device_id, "url": Constant.QUERY_SYSTEM_INFO.replace("{deviceId}", "xxx"), "method": "GET", - "body": {} + "body": {}, } res = self.rest_client.normal_request(url, data=data, method="post") err_msg = "Failed to query remote storage system info" @@ -59,35 +57,43 @@ class RemoteStorageOPT(object): "device_id": self.remote_device_id, "url": remote_url.replace("{deviceId}", "xxx"), "method": "GET", - "body": { - "vstoreId": vstore_id - } + "body": {"vstoreId": vstore_id}, } res = self.rest_client.normal_request(url, data=data, method="post") - err_msg = "Failed to query remote storage filesystem num,vstoreId[%s]" % vstore_id + err_msg = ( + "Failed to query remote storage filesystem num,vstoreId[%s]" % vstore_id + ) rsp_data = StorageInf.omstask_result_parse(err_msg, res) - LOG.info("Success to query remote storage filesystem num, vstoreId[%s]", vstore_id) + LOG.info( + "Success to query remote storage filesystem num, vstoreId[%s]", vstore_id + ) return rsp_data def query_remote_filesystem_info(self, fs_name: str, vstore_id: str): url = Constant.REMOTE_EXECUTE data = { "device_id": self.remote_device_id, - "url": Constant.CREATE_FS.replace("{deviceId}", "xxx") + "?filter=NAME::%s" % fs_name, + "url": Constant.CREATE_FS.replace("{deviceId}", "xxx") + + "?filter=NAME::%s" % fs_name, "method": "GET", - "body": { - "vstoreId": vstore_id - } + "body": {"vstoreId": vstore_id}, } res = self.rest_client.normal_request(url, data=data, method="post") - err_msg = "Failed to query remote filesystem[%s] info, vstore_id[%s]" % (fs_name, vstore_id) + err_msg = "Failed to query remote filesystem[%s] info, vstore_id[%s]" % ( + fs_name, + vstore_id, + ) rsp_data = StorageInf.omstask_result_parse(err_msg, res) fs_info = dict() for filesystem_info in rsp_data: if filesystem_info.get("NAME") == fs_name: fs_info = filesystem_info break - LOG.info("Success to query remote filesystem[%s] info, vstore_id[%s]", fs_name, vstore_id) + LOG.info( + "Success to query remote filesystem[%s] info, vstore_id[%s]", + fs_name, + vstore_id, + ) return fs_info def query_remote_storage_pool_info(self, pool_id: str) -> dict: @@ -100,9 +106,7 @@ class RemoteStorageOPT(object): "device_id": self.remote_device_id, "url": remote_url.replace("{deviceId}", "xxx"), "method": "GET", - "body": { - "ID": pool_id - } + "body": {"ID": pool_id}, } res = self.rest_client.normal_request(url, data=data, method="post") err_msg = "Failed to query remote storage pool info,poolId[%s]" % pool_id @@ -121,9 +125,14 @@ class KmcResolve(object): :return: """ run_user = get_env_info("cantian_user") - resolve_file_path = os.path.join(CURRENT_PATH, "../../cantian_common/crypte_adapter.py") - cmd = "su -s /bin/bash - %s -c \"export LD_LIBRARY_PATH=/opt/cantian/dbstor/lib:${LD_LIBRARY_PATH} " \ - "&& echo -e %s | python3 -B %s %s\"" % (run_user, plain_text, resolve_file_path, mode) + resolve_file_path = os.path.join( + CURRENT_PATH, "../../cantian_common/crypte_adapter.py" + ) + cmd = ( + 'su -s /bin/bash - %s -c "export LD_LIBRARY_PATH=/opt/cantian/dbstor/lib:${LD_LIBRARY_PATH} ' + '&& echo -e %s | python3 -B %s %s"' + % (run_user, plain_text, resolve_file_path, mode) + ) return_code, output, stderr = exec_popen(cmd) if return_code == 1: raise Exception("resolve password failed.") @@ -173,9 +182,7 @@ class DRDeployCommon(object): """ LOG.info("Start to query vstore file system num.") url = Constant.QUERY_FILE_SYSTEM_NUM.format(deviceId=self.device_id) - data = { - "vstoreId": vstore_id - } + data = {"vstoreId": vstore_id} res = self.rest_client.normal_request(url, data=data, method="get") err_msg = "Failed to query vstore file system num" rsp_data = StorageInf.result_parse(err_msg, res) @@ -224,28 +231,28 @@ class DRDeployCommon(object): return rsp_data def query_hyper_metro_filesystem_pair_info(self, filesystem_id: str) -> list: - url = (Constant.QUERY_HYPER_METRO_FILE_SYSTEM_PAIR + "?ASSOCIATEOBJTYPE=40&ASSOCIATEOBJID={fs_id}"). \ - format(deviceId=self.device_id, fs_id=filesystem_id) + url = ( + Constant.QUERY_HYPER_METRO_FILE_SYSTEM_PAIR + + "?ASSOCIATEOBJTYPE=40&ASSOCIATEOBJID={fs_id}" + ).format(deviceId=self.device_id, fs_id=filesystem_id) res = self.rest_client.normal_request(url, "get") err_msg = "Failed to query hyper metro file system pair info" rsp_data = StorageInf.result_parse(err_msg, res) return rsp_data - + def query_hyper_metro_filesystem_count_info(self, vstore_id: str) -> list: - url = (Constant.QUERY_HYPER_METRO_FILE_SYSTEM_COUNT).format(deviceId=self.device_id) - data = { - "vstoreId": vstore_id - } + url = (Constant.QUERY_HYPER_METRO_FILE_SYSTEM_COUNT).format( + deviceId=self.device_id + ) + data = {"vstoreId": vstore_id} res = self.rest_client.normal_request(url, data=data, method="get") err_msg = "Failed to query hyper metro file system pair count info" rsp_data = StorageInf.result_parse(err_msg, res) return rsp_data - + def query_ulog_filesystem_info_list(self, vstore_id: str) -> list: url = Constant.HYPER_METRO_FILESYSTEM_PAIR.format(deviceId=self.device_id) - data = { - "vstoreId": vstore_id - } + data = {"vstoreId": vstore_id} res = self.rest_client.normal_request(url, data=data, method="get") err_msg = "Failed to query ulog file system pair info list" rsp_data = StorageInf.result_parse(err_msg, res) @@ -257,9 +264,13 @@ class DRDeployCommon(object): :param pair_id: :return: """ - url = Constant.DELETE_HYPER_METRO_PAIR.format(deviceId=self.device_id, id=pair_id) + url = Constant.DELETE_HYPER_METRO_PAIR.format( + deviceId=self.device_id, id=pair_id + ) res = self.rest_client.normal_request(url, "get") - err_msg = "Failed to query hyper metro filesystem pair info by pair id[%s]" % pair_id + err_msg = ( + "Failed to query hyper metro filesystem pair info by pair id[%s]" % pair_id + ) rsp_data = StorageInf.result_parse(err_msg, res) return rsp_data @@ -269,10 +280,14 @@ class DRDeployCommon(object): :param filesystem_id: 文件系统id :return: list """ - url = (Constant.QUERY_REPLICATION_FILE_SYSTEM_PAIR + "?ASSOCIATEOBJTYPE=40&ASSOCIATEOBJID={fs_id}") \ - .format(deviceId=self.device_id, fs_id=filesystem_id) + url = ( + Constant.QUERY_REPLICATION_FILE_SYSTEM_PAIR + + "?ASSOCIATEOBJTYPE=40&ASSOCIATEOBJID={fs_id}" + ).format(deviceId=self.device_id, fs_id=filesystem_id) res = self.rest_client.normal_request(url, "get") - err_msg = "Failed to query remote replication filesystem[%s] info" % filesystem_id + err_msg = ( + "Failed to query remote replication filesystem[%s] info" % filesystem_id + ) rsp_data = StorageInf.result_parse(err_msg, res) return rsp_data @@ -282,9 +297,14 @@ class DRDeployCommon(object): :param pair_id: 远程复制pair id :return: """ - url = Constant.REMOTE_REPLICATION_FILESYSTEM_PAIR_OPT.format(deviceId=self.device_id, id=pair_id) + url = Constant.REMOTE_REPLICATION_FILESYSTEM_PAIR_OPT.format( + deviceId=self.device_id, id=pair_id + ) res = self.rest_client.normal_request(url, "get") - err_msg = "Failed to query remote replication file system info by pair id[%s]" % pair_id + err_msg = ( + "Failed to query remote replication file system info by pair id[%s]" + % pair_id + ) rsp_data = StorageInf.result_parse(err_msg, res) return rsp_data @@ -295,8 +315,10 @@ class DRDeployCommon(object): :return: """ LOG.info("Start to query filesystem for replication info.") - url = (Constant.QUERY_FILESYSTEM_FOR_REPLICATION + "?rmtDeviceId={remote_device_id}&RSSType=24"). \ - format(deviceId=self.device_id, remote_device_id=remote_device_id) + url = ( + Constant.QUERY_FILESYSTEM_FOR_REPLICATION + + "?rmtDeviceId={remote_device_id}&RSSType=24" + ).format(deviceId=self.device_id, remote_device_id=remote_device_id) res = self.rest_client.normal_request(url, "get") err_msg = "Failed to query filesystem for replication info" rsp_data = StorageInf.result_parse(err_msg, res) @@ -315,8 +337,13 @@ class DRDeployCommon(object): description = rsp_data.get("description") current_step_index = description.get("currentStepIndex") step_count = description.get("stepCount") - LOG.info("Task[%s] status [%s], running process[%s/%s]", - name, task_status, current_step_index, step_count) + LOG.info( + "Task[%s] status [%s], running process[%s/%s]", + name, + task_status, + current_step_index, + step_count, + ) if task_status == "success": break if task_status in ["executing", "wait"]: @@ -328,8 +355,15 @@ class DRDeployCommon(object): LOG.error(err_msg) raise Exception(err_msg) - @retry(retry_times=3, wait_times=20, log=LOG, task="create_filesystem_hyper_metro_domain") - def create_filesystem_hyper_metro_domain(self, dev_name: str, dev_esn: str, dev_id: str, domain_name: str) -> dict: + @retry( + retry_times=3, + wait_times=20, + log=LOG, + task="create_filesystem_hyper_metro_domain", + ) + def create_filesystem_hyper_metro_domain( + self, dev_name: str, dev_esn: str, dev_id: str, domain_name: str + ) -> dict: """ 创建文件系统双活域 :param dev_name: 远端设备名称 @@ -345,12 +379,8 @@ class DRDeployCommon(object): "isShareAuthenticationSync": False, "DESCRIPTION": "", "REMOTEDEVICES": [ - { - "devId": dev_id, - "devESN": dev_esn, - "devName": dev_name - } - ] + {"devId": dev_id, "devESN": dev_esn, "devName": dev_name} + ], } url = Constant.HYPER_METRO_DOMAIN.format(deviceId=self.device_id) res = self.rest_client.normal_request(url, data=data, method="post") @@ -360,7 +390,9 @@ class DRDeployCommon(object): return rsp_data @retry(retry_times=3, wait_times=20, log=LOG, task="create_hyper_metro_vstore_pair") - def create_hyper_metro_vstore_pair(self, domain_id: str, local_vstore_id: str, remote_vstore_id: str) -> dict: + def create_hyper_metro_vstore_pair( + self, domain_id: str, local_vstore_id: str, remote_vstore_id: str + ) -> dict: """ 创建双活租户pair :param domain_id: 文件系统双活域id @@ -375,7 +407,7 @@ class DRDeployCommon(object): "REMOTEVSTOREID": remote_vstore_id, "REPTYPE": "1", "PREFERREDMODE": "0", - "isNetworkSync": False + "isNetworkSync": False, } url = Constant.HYPER_METRO_VSTORE_PAIR.format(deviceId=self.device_id) res = self.rest_client.normal_request(url, data=data, method="post") @@ -384,8 +416,12 @@ class DRDeployCommon(object): LOG.info("Success to create hyper metro vstore pair.") return rsp_data - @retry(retry_times=3, wait_times=20, log=LOG, task="create_hyper_metro_filesystem_pair") - def create_hyper_metro_filesystem_pair(self, filesystem_id: str, pool_id: str, vstore_pair_id: str) -> dict: + @retry( + retry_times=3, wait_times=20, log=LOG, task="create_hyper_metro_filesystem_pair" + ) + def create_hyper_metro_filesystem_pair( + self, filesystem_id: str, pool_id: str, vstore_pair_id: str + ) -> dict: """ 调用omtask接口创建双活文件系统 :param filesystem_id: 本端文件系统id @@ -397,7 +433,7 @@ class DRDeployCommon(object): data = { "vstorePairID": vstore_pair_id, "remoteStoragePoolId": pool_id, - "objs": [filesystem_id] + "objs": [filesystem_id], } url = Constant.CREATE_HYPER_METRO_FILESYSTEM_PAIR res = self.rest_client.normal_request(url, data=data, method="post") @@ -406,8 +442,15 @@ class DRDeployCommon(object): LOG.info("Success to create hyper metro filesystem pair.") return rsp_data - @retry(retry_times=3, wait_times=20, log=LOG, task="modify_hyper_metro_filesystem_pair_sync_speed") - def modify_hyper_metro_filesystem_pair_sync_speed(self, vstore_pair_id: str, speed: int) -> None: + @retry( + retry_times=3, + wait_times=20, + log=LOG, + task="modify_hyper_metro_filesystem_pair_sync_speed", + ) + def modify_hyper_metro_filesystem_pair_sync_speed( + self, vstore_pair_id: str, speed: int + ) -> None: """ 修改同步速度 :param vstore_pair_id: @@ -415,17 +458,19 @@ class DRDeployCommon(object): :return: """ LOG.info("Start to modify hyper metro filesystem pair speed to [%d].", speed) - data = { - "ID": vstore_pair_id, - "SPEED": speed - } + data = {"ID": vstore_pair_id, "SPEED": speed} url = Constant.HYPER_METRO_FILESYSTEM_PAIR.format(deviceId=self.device_id) res = self.rest_client.normal_request(url, data=data, method="put") err_msg = "Failed to modify hyper metro filesystem pair speed" StorageInf.result_parse(err_msg, res) LOG.info("Start to modify hyper metro filesystem pair speed.") - @retry(retry_times=3, wait_times=20, log=LOG, task="create_remote_replication_filesystem_pair") + @retry( + retry_times=3, + wait_times=20, + log=LOG, + task="create_remote_replication_filesystem_pair", + ) def create_remote_replication_filesystem_pair(self, **pair_args) -> dict: """ @@ -459,14 +504,9 @@ class DRDeployCommon(object): "syncPair": True, "syncSnapPolicy": 0, "createType": 0, - "reservedConsistencySnapSwitch": 0 + "reservedConsistencySnapSwitch": 0, }, - "objs": [ - { - "id": local_fs_id, - "vstoreId": 0 - } - ] + "objs": [{"id": local_fs_id, "vstoreId": 0}], } if remote_name_rule == 2: data["replication"]["namePrefix"] = RepFileSystemNameRule.NamePrefix @@ -475,10 +515,17 @@ class DRDeployCommon(object): res = self.rest_client.normal_request(url, data=data, method="post") err_msg = "Failed to create remote replication filesystem pair" rsp_data = StorageInf.omstask_result_parse(err_msg, res) - LOG.info("Success to create remote replication filesystem[%s] pair.", local_fs_id) + LOG.info( + "Success to create remote replication filesystem[%s] pair.", local_fs_id + ) return rsp_data - @retry(retry_times=3, wait_times=20, log=LOG, task="split_remote_replication_filesystem_pair") + @retry( + retry_times=3, + wait_times=20, + log=LOG, + task="split_remote_replication_filesystem_pair", + ) def split_remote_replication_filesystem_pair(self, pair_id: str) -> dict: """ 分裂远程复制pair @@ -486,27 +533,32 @@ class DRDeployCommon(object): :return: {} """ LOG.info("Start to split remote replication filesystem pair[%s].", pair_id) - data = { - "ID": pair_id - } - url = Constant.SPLIT_REMOTE_REPLICATION_FILESYSTEM_PAIR.format(deviceId=self.device_id) + data = {"ID": pair_id} + url = Constant.SPLIT_REMOTE_REPLICATION_FILESYSTEM_PAIR.format( + deviceId=self.device_id + ) res = self.rest_client.normal_request(url, data=data, method="put") err_msg = "Failed to split remote replication filesystem pair" rsp_data = StorageInf.result_parse(err_msg, res) LOG.info("Success to split remote replication filesystem pair") return rsp_data - @retry(retry_times=3, wait_times=20, log=LOG, task="remote_replication_filesystem_pair_cancel_secondary_write_lock") - def remote_replication_filesystem_pair_cancel_secondary_write_lock(self, pair_id: str) -> dict: + @retry( + retry_times=3, + wait_times=20, + log=LOG, + task="remote_replication_filesystem_pair_cancel_secondary_write_lock", + ) + def remote_replication_filesystem_pair_cancel_secondary_write_lock( + self, pair_id: str + ) -> dict: """ 取消远程复制从资源写保护 :param pair_id: 远程复制id :return: """ LOG.info("Start to cancel secondary write lock.") - data = { - "ID": pair_id - } + data = {"ID": pair_id} url = Constant.CANCEL_SECONDARY_WRITE_LOCK.format(deviceId=self.device_id) res = self.rest_client.normal_request(url, data=data, method="put") err_msg = "Failed to to cancel secondary write lock" @@ -514,17 +566,22 @@ class DRDeployCommon(object): LOG.info("Success to cancel secondary write lock.") return rsp_data - @retry(retry_times=3, wait_times=20, log=LOG, task="remote_replication_filesystem_pair_set_secondary_write_lock") - def remote_replication_filesystem_pair_set_secondary_write_lock(self, pair_id: str) -> dict: + @retry( + retry_times=3, + wait_times=20, + log=LOG, + task="remote_replication_filesystem_pair_set_secondary_write_lock", + ) + def remote_replication_filesystem_pair_set_secondary_write_lock( + self, pair_id: str + ) -> dict: """ 设置远程复制从资源写保护 :param pair_id: 远程复制iD :return: """ LOG.info("Start to set secondary write lock.") - data = { - "ID": pair_id - } + data = {"ID": pair_id} url = Constant.SET_SECONDARY_WRITE_LOCK.format(deviceId=self.device_id) res = self.rest_client.normal_request(url, data=data, method="put") err_msg = "Failed to set secondary write lock." @@ -532,8 +589,15 @@ class DRDeployCommon(object): LOG.info("Success to set secondary write lock.") return rsp_data - @retry(retry_times=3, wait_times=20, log=LOG, task="sync_remote_replication_filesystem_pair") - def sync_remote_replication_filesystem_pair(self, pair_id: str, vstore_id: str, is_full_copy: bool) -> None: + @retry( + retry_times=3, + wait_times=20, + log=LOG, + task="sync_remote_replication_filesystem_pair", + ) + def sync_remote_replication_filesystem_pair( + self, pair_id: str, vstore_id: str, is_full_copy: bool + ) -> None: """ 触发远程复制 :param pair_id: 远程复制pair id @@ -542,19 +606,19 @@ class DRDeployCommon(object): :return: """ LOG.info("Start to sync remote replication filesystem pair.") - data = { - "ID": pair_id, - "vstoreId": vstore_id, - "isFullCopy": is_full_copy - } - url = Constant.SYNC_REMOTE_REPLICATION_FILESYSTEM_PAIR.format(deviceId=self.device_id) + data = {"ID": pair_id, "vstoreId": vstore_id, "isFullCopy": is_full_copy} + url = Constant.SYNC_REMOTE_REPLICATION_FILESYSTEM_PAIR.format( + deviceId=self.device_id + ) res = self.rest_client.normal_request(url, data=data, method="put") err_msg = "Failed to sync remote replication filesystem pair" rsp_data = StorageInf.result_parse(err_msg, res) LOG.info("Success to sync remote replication filesystem pair.") return rsp_data - def delete_remote_replication_filesystem_pair(self, pair_id, is_local_del=False) -> dict: + def delete_remote_replication_filesystem_pair( + self, pair_id, is_local_del=False + ) -> dict: """ 删除远程复制 :param pair_id: 远程复制pair id @@ -562,18 +626,19 @@ class DRDeployCommon(object): :return: """ LOG.info("Start to delete remote replication filesystem pair[%s].", pair_id) - data = { - "ISLOCALDELETE": is_local_del, - "TOSYNCSRWHENDELETE": False - } - url = Constant.REMOTE_REPLICATION_FILESYSTEM_PAIR_OPT.format(deviceId=self.device_id, id=pair_id) + data = {"ISLOCALDELETE": is_local_del, "TOSYNCSRWHENDELETE": False} + url = Constant.REMOTE_REPLICATION_FILESYSTEM_PAIR_OPT.format( + deviceId=self.device_id, id=pair_id + ) res = self.rest_client.normal_request(url, data=data, method="delete") err_msg = "Start to delete remote replication filesystem pair[%s]" % pair_id rsp_data = StorageInf.result_parse(err_msg, res) LOG.info("Success to delete remote replication filesystem pair[%s].", pair_id) return rsp_data - def delete_hyper_metro_filesystem_pair(self, pair_id: str, vstore_id: str, is_local_del=False) -> dict: + def delete_hyper_metro_filesystem_pair( + self, pair_id: str, vstore_id: str, is_local_del=False + ) -> dict: """ 删除双活pair :param pair_id: 双活pair id @@ -582,11 +647,10 @@ class DRDeployCommon(object): :return: """ LOG.info("Start to delete hyper metro filesystem pair") - url = Constant.DELETE_HYPER_METRO_PAIR.format(deviceId=self.device_id, id=pair_id) - data = { - "ISLOCALDELETE": is_local_del, - "vstoreId": vstore_id - } + url = Constant.DELETE_HYPER_METRO_PAIR.format( + deviceId=self.device_id, id=pair_id + ) + data = {"ISLOCALDELETE": is_local_del, "vstoreId": vstore_id} res = self.rest_client.normal_request(url, data=data, method="delete") err_mgs = "Failed to to delete hyper metro filesystem pair" rsp_data = StorageInf.result_parse(err_mgs, res) @@ -602,10 +666,10 @@ class DRDeployCommon(object): :return: """ LOG.info("Start to delete hyper metro vstore pair") - url = Constant.DELETE_HYPER_METRO_VSTORE_PAIR.format(deviceId=self.device_id, id=pair_id) - data = { - "isLocalDelete": is_local_del - } + url = Constant.DELETE_HYPER_METRO_VSTORE_PAIR.format( + deviceId=self.device_id, id=pair_id + ) + data = {"isLocalDelete": is_local_del} res = self.rest_client.normal_request(url, data=data, method="delete") err_msg = "Failed to delete hyper metro vstore pair" rsp_data = StorageInf.result_parse(err_msg, res) @@ -618,36 +682,54 @@ class DRDeployCommon(object): :param domain_id: 文件系统双活域id :return: """ - LOG.info("Start to split filesystem hyper metro domain, domain_id[%s]", domain_id) - data = { - "ID": domain_id - } - url = Constant.SPLIT_FILESYSTEM_HYPER_METRO_DOMAIN.format(deviceId=self.device_id) + LOG.info( + "Start to split filesystem hyper metro domain, domain_id[%s]", domain_id + ) + data = {"ID": domain_id} + url = Constant.SPLIT_FILESYSTEM_HYPER_METRO_DOMAIN.format( + deviceId=self.device_id + ) res = self.rest_client.normal_request(url, data=data, method="post") - err_msg = "Failed to split filesystem hyper metro domain, domain_id[%s]" % domain_id + err_msg = ( + "Failed to split filesystem hyper metro domain, domain_id[%s]" % domain_id + ) rsp_data = StorageInf.result_parse(err_msg, res) - LOG.info("Success to split filesystem hyper metro domain, domain_id[%s]", domain_id) + LOG.info( + "Success to split filesystem hyper metro domain, domain_id[%s]", domain_id + ) return rsp_data - def delete_filesystem_hyper_metro_domain(self, domain_id: str, is_local_del=False) -> dict: + def delete_filesystem_hyper_metro_domain( + self, domain_id: str, is_local_del=False + ) -> dict: """ 删除文件系统双活域 :param domain_id: 文件系统双活域id :param is_local_del: 是否执行本端删除，默认False，不执行。当删除失败错误码为1077674506时，使用本地删除 :return: """ - LOG.info("Start to delete filesystem hyper metro domain, " - "domain_id[%s], is_local_del[%s].", domain_id, is_local_del) - data = { - "ISLOCALDELETE": is_local_del - } - url = Constant.DELETE_FILESYSTEM_HYPER_METRO_DOMAIN.format(deviceId=self.device_id, id=domain_id) + LOG.info( + "Start to delete filesystem hyper metro domain, " + "domain_id[%s], is_local_del[%s].", + domain_id, + is_local_del, + ) + data = {"ISLOCALDELETE": is_local_del} + url = Constant.DELETE_FILESYSTEM_HYPER_METRO_DOMAIN.format( + deviceId=self.device_id, id=domain_id + ) res = self.rest_client.normal_request(url, data=data, method="delete") - err_msg = "Failed to delete filesystem hyper metro domain, " \ - "domain_id[%s], is_local_del[%s]." % (domain_id, is_local_del) + err_msg = ( + "Failed to delete filesystem hyper metro domain, " + "domain_id[%s], is_local_del[%s]." % (domain_id, is_local_del) + ) rsp_data = StorageInf.result_parse(err_msg, res) - LOG.info("Success to delete filesystem hyper metro domain, " - "domain_id[%s], is_local_del[%s].", domain_id, is_local_del) + LOG.info( + "Success to delete filesystem hyper metro domain, " + "domain_id[%s], is_local_del[%s].", + domain_id, + is_local_del, + ) return rsp_data def swap_role_replication_pair(self, pair_id: str, vstore_id=0) -> None: @@ -658,10 +740,7 @@ class DRDeployCommon(object): :return: """ LOG.info("Swap replication pair[%s] role start", pair_id) - data = { - "ID": pair_id, - "vstoreId": vstore_id - } + data = {"ID": pair_id, "vstoreId": vstore_id} url = Constant.SWAP_ROLE_REPLICATION_PAIR.format(deviceId=self.device_id) res = self.rest_client.normal_request(url, data=data, method="put") err_msg = "Swap replication pair[%s] role failed" % pair_id @@ -675,32 +754,42 @@ class DRDeployCommon(object): :return: """ LOG.info("Swap role fs hyper metro domain[%s] start.", domain_id) - data = { - "ID": domain_id - } + data = {"ID": domain_id} url = Constant.SWAP_ROLE_FS_HYPER_METRO_DOMAIN.format(deviceId=self.device_id) res = self.rest_client.normal_request(url, data=data, method="post") err_msg = "Swap role fs hyper metro domain[%s] failed" % domain_id StorageInf.result_parse(err_msg, res) LOG.info("Swap role fs hyper metro domain[%s] success.", domain_id) - def change_fs_hyper_metro_domain_second_access(self, domain_id: str, access: str) -> None: + def change_fs_hyper_metro_domain_second_access( + self, domain_id: str, access: str + ) -> None: """ 设置/取消从资源保护 :param access: 1: 禁止访问 2: 读写 :param domain_id: :return: """ - LOG.info("Change fs hyper metro domain[%s] second access[%s] start.", domain_id, access) - data = { - "ID": domain_id, - "access": access - } - url = Constant.CHANGE_FS_HYPER_METRO_DOMAIN_SECOND_ACCESS.format(deviceId=self.device_id) + LOG.info( + "Change fs hyper metro domain[%s] second access[%s] start.", + domain_id, + access, + ) + data = {"ID": domain_id, "access": access} + url = Constant.CHANGE_FS_HYPER_METRO_DOMAIN_SECOND_ACCESS.format( + deviceId=self.device_id + ) res = self.rest_client.normal_request(url, data=data, method="post") - err_msg = "Change fs hyper metro domain[%s] second access[%s] failed." % (domain_id, access) + err_msg = "Change fs hyper metro domain[%s] second access[%s] failed." % ( + domain_id, + access, + ) StorageInf.result_parse(err_msg, res) - LOG.info("Change fs hyper metro domain[%s] second access[%s] success.", domain_id, access) + LOG.info( + "Change fs hyper metro domain[%s] second access[%s] success.", + domain_id, + access, + ) def join_fs_hyper_metro_domain(self, domain_id: str) -> None: """ @@ -709,9 +798,7 @@ class DRDeployCommon(object): :return: """ LOG.info("Join fs hyper metro domain[%s] start", domain_id) - data = { - "ID": domain_id - } + data = {"ID": domain_id} url = Constant.JOIN_FS_HYPER_METRO_DOMAIN.format(deviceId=self.device_id) res = self.rest_client.normal_request(url, data=data, method="post") err_msg = "Failed to join fs hyper metro domain[%s]" % domain_id diff --git a/pkg/deploy/action/storage_operate/dr_deploy_operate/dr_deploy_full_sync.py b/pkg/deploy/action/storage_operate/dr_deploy_operate/dr_deploy_full_sync.py index e8f236f4..53905eef 100644 --- a/pkg/deploy/action/storage_operate/dr_deploy_operate/dr_deploy_full_sync.py +++ b/pkg/deploy/action/storage_operate/dr_deploy_operate/dr_deploy_full_sync.py @@ -7,8 +7,14 @@ import os import stat import time -from utils.config.rest_constant import SecresAccess, HealthStatus, ReplicationRunningStatus, MetroDomainRunningStatus, \ - ConfigRole, Constant +from utils.config.rest_constant import ( + SecresAccess, + HealthStatus, + ReplicationRunningStatus, + MetroDomainRunningStatus, + ConfigRole, + Constant, +) from storage_operate.dr_deploy_operate.dr_deploy import DRDeploy from logic.common_func import get_status from logic.common_func import exec_popen @@ -16,14 +22,18 @@ from om_log import DR_DEPLOY_LOG as LOG from get_config_info import get_env_info CURRENT_PATH = os.path.dirname(os.path.abspath(__file__)) -FULL_SYNC_PROGRESS = os.path.join(CURRENT_PATH, "../../../config/full_sync_progress.json") -ZSQL_INI_PATH = '/mnt/dbdata/local/cantian/tmp/data/cfg/ctsql.ini' +FULL_SYNC_PROGRESS = os.path.join( + CURRENT_PATH, "../../../config/full_sync_progress.json" +) +ZSQL_INI_PATH = "/mnt/dbdata/local/cantian/tmp/data/cfg/ctsql.ini" LOCK_INSTANCE = "lock instance for backup;" UNLOCK_INSTANCE = "unlock instance;" FLUSH_TABLE = "flush table with read lock;" UNLOCK_TABLE = "unlock tables;" -FULL_CHECK_POINT_CMD = 'echo -e %s | su -s /bin/bash - %s -c \'source ~/.bashrc && ' \ - 'ctsql sys@127.0.0.1:1611 -q -c "alter system checkpoint global;"\'' +FULL_CHECK_POINT_CMD = ( + "echo -e %s | su -s /bin/bash - %s -c 'source ~/.bashrc && " + 'ctsql sys@127.0.0.1:1611 -q -c "alter system checkpoint global;"\'' +) class FullSyncRepPair(DRDeploy): @@ -51,8 +61,10 @@ class FullSyncRepPair(DRDeploy): """ check_time = 100 LOG.info("Check cantian status.") - cmd = "su -s /bin/bash - cantian -c \"cms stat | " \ - "grep -v STAT | awk '{print \$3, \$6}'\"" + cmd = ( + 'su -s /bin/bash - cantian -c "cms stat | ' + "grep -v STAT | awk '{print \$3, \$6}'\"" + ) while check_time: time.sleep(10) check_time -= 10 @@ -63,7 +75,10 @@ class FullSyncRepPair(DRDeploy): raise Exception(err_msg) cms_stat = output.split("\n") if len(cms_stat) < 2: - err_msg = "Current cluster status is abnormal, output:%s, stderr:%s" % (output, stderr) + err_msg = "Current cluster status is abnormal, output:%s, stderr:%s" % ( + output, + stderr, + ) LOG.error(err_msg) raise Exception(err_msg) online = True @@ -72,7 +87,11 @@ class FullSyncRepPair(DRDeploy): if online != "ONLINE" or work_stat != "1": online = False if not online: - LOG.info("Current cluster status is abnormal, output:%s, stderr:%s", output, stderr) + LOG.info( + "Current cluster status is abnormal, output:%s, stderr:%s", + output, + stderr, + ) continue else: break @@ -110,7 +129,7 @@ class FullSyncRepPair(DRDeploy): "do_flush_table_with_read_lock": "default", "sync_rep_page_fs_pair": "default", "cancel_rep_page_fs_secondary_write_lock": "default", - "do_unlock_instance_for_backup": "default" + "do_unlock_instance_for_backup": "default", } standby_record_dict = { "sync_rep_page_fs_pair": "default", @@ -118,26 +137,23 @@ class FullSyncRepPair(DRDeploy): "cantian_disaster_recovery_status": "default", } self.metadata_in_cantian = self.dr_deploy_info.get("mysql_metadata_in_cantian") - dr_record_dict = active_record_dict if self.site == "active" else standby_record_dict + dr_record_dict = ( + active_record_dict if self.site == "active" else standby_record_dict + ) if not self.metadata_in_cantian: - dr_record_dict.update({ - "create_rep_meta_fs_pair": "default", - "sync_rep_meta_fs_pair": "default", - "cancel_rep_meta_fs_secondary_write_lock": "default" - }) - dr_record_dict.update({"full_sync": "default"}) - result = { - "data": dr_record_dict, - "error": + dr_record_dict.update( { - "code": 0, - "description": "" + "create_rep_meta_fs_pair": "default", + "sync_rep_meta_fs_pair": "default", + "cancel_rep_meta_fs_secondary_write_lock": "default", } - } + ) + dr_record_dict.update({"full_sync": "default"}) + result = {"data": dr_record_dict, "error": {"code": 0, "description": ""}} flags = os.O_WRONLY | os.O_CREAT | os.O_TRUNC modes = stat.S_IWUSR | stat.S_IRUSR - with os.fdopen(os.open(self.record_progress_file, flags, modes), 'w') as fp: + with os.fdopen(os.open(self.record_progress_file, flags, modes), "w") as fp: json.dump(result, fp, indent=4) def do_full_sync(self, pair_id: str) -> None: @@ -152,14 +168,19 @@ class FullSyncRepPair(DRDeploy): :param pair_id: :return: """ - remote_replication_pair_info = self.dr_deploy_opt.query_remote_replication_pair_info_by_pair_id( - pair_id=pair_id) + remote_replication_pair_info = ( + self.dr_deploy_opt.query_remote_replication_pair_info_by_pair_id( + pair_id=pair_id + ) + ) secres_access = remote_replication_pair_info.get("SECRESACCESS") if secres_access == SecresAccess.ReadAndWrite: - self.dr_deploy_opt.remote_replication_filesystem_pair_set_secondary_write_lock(pair_id=pair_id) - self.dr_deploy_opt.sync_remote_replication_filesystem_pair(pair_id=self.page_fs_pair_id, - vstore_id="0", - is_full_copy=True) + self.dr_deploy_opt.remote_replication_filesystem_pair_set_secondary_write_lock( + pair_id=pair_id + ) + self.dr_deploy_opt.sync_remote_replication_filesystem_pair( + pair_id=self.page_fs_pair_id, vstore_id="0", is_full_copy=True + ) def query_full_sync_status(self, pair_id: str) -> tuple: """ @@ -167,68 +188,98 @@ class FullSyncRepPair(DRDeploy): :param pair_id: :return: """ - remote_replication_pair_info = self.dr_deploy_opt.query_remote_replication_pair_info_by_pair_id( - pair_id=pair_id) + remote_replication_pair_info = ( + self.dr_deploy_opt.query_remote_replication_pair_info_by_pair_id( + pair_id=pair_id + ) + ) replication_progress = remote_replication_pair_info.get("REPLICATIONPROGRESS") start_time = remote_replication_pair_info.get("STARTTIME") end_time = remote_replication_pair_info.get("ENDTIME") - replication_pair_health_status = remote_replication_pair_info.get("HEALTHSTATUS") - replication_pair_running_status = remote_replication_pair_info.get("RUNNINGSTATUS") + replication_pair_health_status = remote_replication_pair_info.get( + "HEALTHSTATUS" + ) + replication_pair_running_status = remote_replication_pair_info.get( + "RUNNINGSTATUS" + ) secres_access = remote_replication_pair_info.get("SECRESACCESS") - LOG.info("Sync remote replication filesystem pair[%s], health status:[%s], " - "running status[%s], progress[%s%%], start time[%s]", - pair_id, - get_status(replication_pair_health_status, HealthStatus), - get_status(replication_pair_running_status, ReplicationRunningStatus), - replication_progress, - datetime.datetime.fromtimestamp(int(start_time))) - if replication_progress == "100" and \ - replication_pair_running_status == ReplicationRunningStatus.Normal and \ - replication_pair_health_status == HealthStatus.Normal: - LOG.info("Success to sync remote replication filesystem pair[%s], end time[%s]", - pair_id, - datetime.datetime.fromtimestamp(int(end_time))) + LOG.info( + "Sync remote replication filesystem pair[%s], health status:[%s], " + "running status[%s], progress[%s%%], start time[%s]", + pair_id, + get_status(replication_pair_health_status, HealthStatus), + get_status(replication_pair_running_status, ReplicationRunningStatus), + replication_progress, + datetime.datetime.fromtimestamp(int(start_time)), + ) + if ( + replication_progress == "100" + and replication_pair_running_status == ReplicationRunningStatus.Normal + and replication_pair_health_status == HealthStatus.Normal + ): + LOG.info( + "Success to sync remote replication filesystem pair[%s], end time[%s]", + pair_id, + datetime.datetime.fromtimestamp(int(end_time)), + ) if int(start_time) - int(end_time) > Constant.FULL_SYNC_MAX_TIME: - LOG.info("Do sync remote replication filesystem[%s] pair of full copy." % pair_id) - self.dr_deploy_opt.sync_remote_replication_filesystem_pair(pair_id=pair_id, vstore_id=0, - is_full_copy=False) + LOG.info( + "Do sync remote replication filesystem[%s] pair of full copy." + % pair_id + ) + self.dr_deploy_opt.sync_remote_replication_filesystem_pair( + pair_id=pair_id, vstore_id=0, is_full_copy=False + ) return False, replication_progress, secres_access return True, replication_progress, secres_access return False, replication_progress, secres_access def standby_check_status(self): LOG.info("Check standby cluster status.") - cmd = "source ~/.bashrc && su -s /bin/bash - cantian -c " \ - "\"cms stat | awk '{print \$3, \$(NF-1), \$NF}'\"" + cmd = ( + "source ~/.bashrc && su -s /bin/bash - cantian -c " + "\"cms stat | awk '{print \$3, \$(NF-1), \$NF}'\"" + ) return_code, output, stderr = exec_popen(cmd) if return_code: err_msg = "Cms stat command execute failed, details:%s" % output + stderr LOG.error(err_msg) - self.record_deploy_process("full_sync", "failed", code=-1, description=err_msg) + self.record_deploy_process( + "full_sync", "failed", code=-1, description=err_msg + ) raise Exception(err_msg) cms_stat = output.split("\n") if len(cms_stat) < 3: err_msg = "Cluster stat is abnormal, details:%s" % output LOG.error(err_msg) - self.record_deploy_process("full_sync", "failed", code=-1, description=err_msg) + self.record_deploy_process( + "full_sync", "failed", code=-1, description=err_msg + ) raise Exception(err_msg) online_flag = True stat_change_time = 0 for status in cms_stat[1:]: node_stat, _date, _time = status.split(" ") - datetime_obj = datetime.datetime.strptime(_date + " " + _time, "%Y-%m-%d %H:%M:%S.%f") + datetime_obj = datetime.datetime.strptime( + _date + " " + _time, "%Y-%m-%d %H:%M:%S.%f" + ) timestamp = int(datetime_obj.timestamp()) - stat_change_time = timestamp if timestamp > stat_change_time else stat_change_time + stat_change_time = ( + timestamp if timestamp > stat_change_time else stat_change_time + ) if node_stat == "UNKNOWN": online_flag = False - LOG.info("Standby cluster status, online[%s], change time[%s]", online_flag, stat_change_time) + LOG.info( + "Standby cluster status, online[%s], change time[%s]", + online_flag, + stat_change_time, + ) return online_flag, stat_change_time def standby_cms_res_opt(self, action="start"): self.record_deploy_process("standby_start", "running") LOG.info("Standby stop by cms command.") - cmd = "su -s /bin/bash - cantian -c " \ - "\"source ~/.bashrc && cms res -stop db\"" + cmd = "su -s /bin/bash - cantian -c " '"source ~/.bashrc && cms res -stop db"' return_code, output, stderr = exec_popen(cmd, timeout=600) if return_code: err_msg = "Cantian stop failed, error:%s." % output + stderr @@ -236,13 +287,17 @@ class FullSyncRepPair(DRDeploy): LOG.info("Stop cantian by cms command success.") time.sleep(60) LOG.info("Standby start by cms command.") - cmd = "su -s /bin/bash - cantian -c " \ - "\"source ~/.bashrc && cms res -%s db\"" % action + cmd = ( + "su -s /bin/bash - cantian -c " + '"source ~/.bashrc && cms res -%s db"' % action + ) return_code, output, stderr = exec_popen(cmd) if return_code: err_msg = "Cantian %s failed, error:%s." % (action, output + stderr) LOG.error(err_msg) - self.record_deploy_process("standby_start", "failed", code=-1, description=err_msg) + self.record_deploy_process( + "standby_start", "failed", code=-1, description=err_msg + ) raise Exception(err_msg) self.check_cluster_status() LOG.info("Standby start by cms command success.") @@ -262,7 +317,9 @@ class FullSyncRepPair(DRDeploy): :return: """ domain_id = self.dr_deploy_info.get("hyper_domain_id") - domain_info = self.dr_deploy_opt.query_hyper_metro_domain_info(domain_id=domain_id) + domain_info = self.dr_deploy_opt.query_hyper_metro_domain_info( + domain_id=domain_id + ) running_status = domain_info.get("RUNNINGSTATUS") if running_status != MetroDomainRunningStatus.Normal: LOG.error("metro domain is not normal, can not exec full sync.") @@ -281,23 +338,37 @@ class FullSyncRepPair(DRDeploy): LOG.info("Full sync replication pair of meta fs") meta_pair_ready = True if self.metadata_in_cantian else False while True: - page_pair_ready, page_pair_progress, secres_access = self.query_full_sync_status(self.page_fs_pair_id) + page_pair_ready, page_pair_progress, secres_access = ( + self.query_full_sync_status(self.page_fs_pair_id) + ) self.record_deploy_process("sync_rep_page_fs_pair", page_pair_progress) if not self.metadata_in_cantian: - meta_pair_ready, meta_pair_progress, secres_access = self.query_full_sync_status(self.meta_fs_pair_id) + meta_pair_ready, meta_pair_progress, secres_access = ( + self.query_full_sync_status(self.meta_fs_pair_id) + ) self.record_deploy_process("sync_rep_meta_fs_pair", meta_pair_progress) if page_pair_ready and meta_pair_ready: break time.sleep(60) - self.dr_deploy_opt.split_remote_replication_filesystem_pair(self.page_fs_pair_id) + self.dr_deploy_opt.split_remote_replication_filesystem_pair( + self.page_fs_pair_id + ) self.record_deploy_process("sync_rep_page_fs_pair", "success") - self.dr_deploy_opt.remote_replication_filesystem_pair_cancel_secondary_write_lock(self.page_fs_pair_id) + self.dr_deploy_opt.remote_replication_filesystem_pair_cancel_secondary_write_lock( + self.page_fs_pair_id + ) self.record_deploy_process("cancel_rep_page_fs_secondary_write_lock", "success") if not self.metadata_in_cantian: - self.dr_deploy_opt.split_remote_replication_filesystem_pair(self.meta_fs_pair_id) + self.dr_deploy_opt.split_remote_replication_filesystem_pair( + self.meta_fs_pair_id + ) self.record_deploy_process("sync_rep_page_fs_pair", "success") - self.dr_deploy_opt.remote_replication_filesystem_pair_cancel_secondary_write_lock(self.meta_fs_pair_id) - self.record_deploy_process("cancel_rep_meta_fs_secondary_write_lock", "success") + self.dr_deploy_opt.remote_replication_filesystem_pair_cancel_secondary_write_lock( + self.meta_fs_pair_id + ) + self.record_deploy_process( + "cancel_rep_meta_fs_secondary_write_lock", "success" + ) self.do_unlock_instance_for_backup() def full_sync_standby(self): @@ -310,7 +381,9 @@ class FullSyncRepPair(DRDeploy): :return: """ domain_id = self.dr_deploy_info.get("hyper_domain_id") - domain_info = self.dr_deploy_opt.query_hyper_metro_domain_info(domain_id=domain_id) + domain_info = self.dr_deploy_opt.query_hyper_metro_domain_info( + domain_id=domain_id + ) running_status = domain_info.get("RUNNINGSTATUS") if running_status != MetroDomainRunningStatus.Normal: LOG.error("metro domain is not normal, can not exec full sync.") @@ -321,22 +394,38 @@ class FullSyncRepPair(DRDeploy): raise Exception("config role is not secondary, can not exec full sync.") ready_flag = False cantian_online, stat_change_time = self.standby_check_status() - meta_access = SecresAccess.ReadAndWrite if self.metadata_in_cantian else SecresAccess.ReadOnly - page_pair_info = self.dr_deploy_opt.query_remote_replication_pair_info_by_pair_id( - pair_id=self.page_fs_pair_id) + meta_access = ( + SecresAccess.ReadAndWrite + if self.metadata_in_cantian + else SecresAccess.ReadOnly + ) + page_pair_info = ( + self.dr_deploy_opt.query_remote_replication_pair_info_by_pair_id( + pair_id=self.page_fs_pair_id + ) + ) page_end_time = page_pair_info.get("ENDTIME") page_access = page_pair_info.get("SECRESACCESS") - if page_access == SecresAccess.ReadAndWrite and \ - page_end_time is not None and int(page_end_time) < stat_change_time: + if ( + page_access == SecresAccess.ReadAndWrite + and page_end_time is not None + and int(page_end_time) < stat_change_time + ): ready_flag = True if not self.metadata_in_cantian: ready_flag = False - meta_pair_info = self.dr_deploy_opt.query_remote_replication_pair_info_by_pair_id( - self.meta_fs_pair_id) + meta_pair_info = ( + self.dr_deploy_opt.query_remote_replication_pair_info_by_pair_id( + self.meta_fs_pair_id + ) + ) meta_end_time = meta_pair_info.get("ENDTIME") meta_access = meta_pair_info.get("SECRESACCESS") - if meta_access == SecresAccess.ReadAndWrite \ - and meta_end_time is not None and int(page_end_time) < stat_change_time: + if ( + meta_access == SecresAccess.ReadAndWrite + and meta_end_time is not None + and int(page_end_time) < stat_change_time + ): ready_flag = True if not ready_flag: self.wait_rep_pair_sync_end(meta_access) @@ -350,13 +439,19 @@ class FullSyncRepPair(DRDeploy): :return: """ while True: - _, page_pair_progress, page_access = self.query_full_sync_status(self.page_fs_pair_id) + _, page_pair_progress, page_access = self.query_full_sync_status( + self.page_fs_pair_id + ) self.record_deploy_process("sync_rep_page_fs_pair", page_pair_progress) if not self.metadata_in_cantian: - _, meta_pair_progress, meta_access = self.query_full_sync_status(self.meta_fs_pair_id) + _, meta_pair_progress, meta_access = self.query_full_sync_status( + self.meta_fs_pair_id + ) self.record_deploy_process("sync_rep_meta_fs_pair", meta_pair_progress) - if page_access == SecresAccess.ReadAndWrite and \ - meta_access == SecresAccess.ReadAndWrite: + if ( + page_access == SecresAccess.ReadAndWrite + and meta_access == SecresAccess.ReadAndWrite + ): self.record_deploy_process("sync_rep_page_fs_pair", "success") if not self.metadata_in_cantian: self.record_deploy_process("sync_rep_meta_fs_pair", "success") @@ -365,7 +460,9 @@ class FullSyncRepPair(DRDeploy): def execute(self): action_parse = argparse.ArgumentParser() - action_parse.add_argument("--site", dest="site", choices=["standby", "active"], required=True) + action_parse.add_argument( + "--site", dest="site", choices=["standby", "active"], required=True + ) action_parse.add_argument("--mysql_cmd", dest="mysql_cmd", required=False) action_parse.add_argument("--mysql_user", dest="mysql_user", required=False) args = action_parse.parse_args() diff --git a/pkg/deploy/action/storage_operate/dr_deploy_operate/dr_deploy_pre_check.py b/pkg/deploy/action/storage_operate/dr_deploy_operate/dr_deploy_pre_check.py index f7129d05..054b20e2 100644 --- a/pkg/deploy/action/storage_operate/dr_deploy_operate/dr_deploy_pre_check.py +++ b/pkg/deploy/action/storage_operate/dr_deploy_operate/dr_deploy_pre_check.py @@ -13,24 +13,42 @@ from pre_install import PreInstall from logic.storage_operate import StorageInf from storage_operate.dr_deploy_operate.dr_deploy_common import DRDeployCommon from storage_operate.dr_deploy_operate.dr_deploy_common import RemoteStorageOPT -from utils.config.rest_constant import SystemRunningStatus, \ - HealthStatus, RemoteDeviceStatus, FilesystemRunningStatus, PoolStatus, PoolHealth, CANTIAN_DOMAIN_PREFIX, \ - RepFileSystemNameRule +from utils.config.rest_constant import ( + SystemRunningStatus, + HealthStatus, + RemoteDeviceStatus, + FilesystemRunningStatus, + PoolStatus, + PoolHealth, + CANTIAN_DOMAIN_PREFIX, + RepFileSystemNameRule, +) from om_log import LOGGER as LOG from get_config_info import get_env_info from obtains_lsid import LSIDGenerate from cantian_common.mysql_shell import MysqlShell -from logic.common_func import exec_popen, read_json_config, write_json_config, get_status +from logic.common_func import ( + exec_popen, + read_json_config, + write_json_config, + get_status, +) CURRENT_PATH = os.path.dirname(os.path.abspath(__file__)) -DR_DEPLOY_PARAM_FILE = os.path.join(CURRENT_PATH, "../../../config/dr_deploy_param.json") +DR_DEPLOY_PARAM_FILE = os.path.join( + CURRENT_PATH, "../../../config/dr_deploy_param.json" +) DEPLOY_POLICY_FILE = os.path.join(CURRENT_PATH, "../../deploy_policy_config.json") DEFAULT_PARAM_FILE = os.path.join(CURRENT_PATH, "../../config_params.json") -DR_PROCESS_RECORD_FILE = os.path.join(CURRENT_PATH, "../../../config/dr_process_record.json") +DR_PROCESS_RECORD_FILE = os.path.join( + CURRENT_PATH, "../../../config/dr_process_record.json" +) CANTIAN_STOP_SUCCESS_FLAG = os.path.join(CURRENT_PATH, "../../../config/.stop_success") DEPLOY_PARAM_FILE = "/opt/cantian/config/deploy_param.json" -DEPLOY_POLICY_CONFIG_FILE = os.path.join(CURRENT_PATH, "../../deploy_policy_config.json") +DEPLOY_POLICY_CONFIG_FILE = os.path.join( + CURRENT_PATH, "../../deploy_policy_config.json" +) DOMAIN_LIMITS = 4 @@ -72,7 +90,11 @@ class DRDeployPreCheck(object): 部署前清理环境 :return: """ - file_list = [DR_PROCESS_RECORD_FILE, DR_DEPLOY_PARAM_FILE, CANTIAN_STOP_SUCCESS_FLAG] + file_list = [ + DR_PROCESS_RECORD_FILE, + DR_DEPLOY_PARAM_FILE, + CANTIAN_STOP_SUCCESS_FLAG, + ] for file in file_list: if os.path.exists(file): os.remove(file) @@ -94,11 +116,17 @@ class DRDeployPreCheck(object): _, sync_proc, _ = exec_popen(check_sync_cmd) err_msg = "" if deploy_proc: - err_msg += "Dr deploy is executing, please check, details:\n%s" % deploy_proc + err_msg += ( + "Dr deploy is executing, please check, details:\n%s" % deploy_proc + ) if undeploy_proc: - err_msg += "Dr undeploy is executing, please check, details:\n%s" % undeploy_proc + err_msg += ( + "Dr undeploy is executing, please check, details:\n%s" % undeploy_proc + ) if sync_proc: - err_msg += "Dr full sync is executing, please check, details:\n%s" % sync_proc + err_msg += ( + "Dr full sync is executing, please check, details:\n%s" % sync_proc + ) if err_msg: raise Exception(err_msg) @@ -109,13 +137,18 @@ class DRDeployPreCheck(object): """ err_msg = [] node_id = self.deploy_params.get("node_id") - cmd = "su -s /bin/bash - %s -c \"cms stat | " \ - "grep -v NODE_ID | awk '{if(\$1==%s){print \$3,\$6,\$9}}'\"" % (self.run_user, node_id) + cmd = ( + 'su -s /bin/bash - %s -c "cms stat | ' + "grep -v NODE_ID | awk '{if(\$1==%s){print \$3,\$6,\$9}}'\"" + % (self.run_user, node_id) + ) return_code, output, stderr = exec_popen(cmd) if return_code == 1: err_msg = ["Execute command[cms stat] failed, details:%s" % stderr] else: - cms_stat = [re.split(r"\s+", item.strip()) for item in output.strip().split("\n")] + cms_stat = [ + re.split(r"\s+", item.strip()) for item in output.strip().split("\n") + ] for index, item in enumerate(cms_stat): if item[0].strip(" ") != "ONLINE": err_msg.append("Node[%s] status is not ONLINE." % index) @@ -146,16 +179,22 @@ class DRDeployPreCheck(object): remote_system_info = remote_opt.query_remote_storage_system_info() remote_mode_string = remote_system_info.get("productModeString") if remote_mode_string != product_mode_string: - err_msg.append("System mode is non-consistent, local mode[%s], remote mode[%s]" % - (product_mode_string, remote_mode_string)) + err_msg.append( + "System mode is non-consistent, local mode[%s], remote mode[%s]" + % (product_mode_string, remote_mode_string) + ) if health_status != HealthStatus.Normal: - err_msg.append("System health status is not normal: health status[%s]." % - get_status(health_status, HealthStatus)) + err_msg.append( + "System health status is not normal: health status[%s]." + % get_status(health_status, HealthStatus) + ) if running_status != SystemRunningStatus.Normal: - err_msg.append("System running status is not normal: running status[%s]." % - get_status(running_status, SystemRunningStatus)) + err_msg.append( + "System running status is not normal: running status[%s]." + % get_status(running_status, SystemRunningStatus) + ) LOG.info("Check storage system info success.") return err_msg @@ -177,17 +216,23 @@ class DRDeployPreCheck(object): if item.get("SN") == remote_esn: remote_device_info = item if not remote_device_info: - err_msg.append("Remote device esn[%s] is not correct, please check" % remote_esn) + err_msg.append( + "Remote device esn[%s] is not correct, please check" % remote_esn + ) return err_msg self.remote_device_id = remote_device_info.get("ID") health_status = remote_device_info.get("HEALTHSTATUS") running_status = remote_device_info.get("RUNNINGSTATUS") if health_status != HealthStatus.Normal: - err_msg.append("Remote device health status is not normal: health status[%s]." % - get_status(health_status, HealthStatus)) + err_msg.append( + "Remote device health status is not normal: health status[%s]." + % get_status(health_status, HealthStatus) + ) if running_status != RemoteDeviceStatus.LinkUp: - err_msg.append("Remote device running status is not normal: running status[%s]." % - get_status(running_status, RemoteDeviceStatus)) + err_msg.append( + "Remote device running status is not normal: running status[%s]." + % get_status(running_status, RemoteDeviceStatus) + ) LOG.info("Check remote device info success.") return err_msg @@ -207,11 +252,15 @@ class DRDeployPreCheck(object): health_status = file_system_info.get("HEALTHSTATUS") running_status = file_system_info.get("RUNNINGSTATUS") if health_status != HealthStatus.Normal: - err_msg.append("Filesystem[%s] health is not normal, status[%s]." % - (fs_name, get_status(health_status, HealthStatus))) + err_msg.append( + "Filesystem[%s] health is not normal, status[%s]." + % (fs_name, get_status(health_status, HealthStatus)) + ) if running_status != FilesystemRunningStatus.Online: - err_msg.append("Filesystem[%s] running status is not normal, status[%s]." % - (fs_name, get_status(running_status, FilesystemRunningStatus))) + err_msg.append( + "Filesystem[%s] running status is not normal, status[%s]." + % (fs_name, get_status(running_status, FilesystemRunningStatus)) + ) LOG.info("Check master filesystem[%s] status success.", fs_name) return err_msg @@ -228,21 +277,30 @@ class DRDeployPreCheck(object): remote_pool_id = self.local_conf_params.get("remote_pool_id") self.remote_operate = RemoteStorageOPT(self.storage_opt, self.remote_device_id) try: - remote_pool_info = self.remote_operate.query_remote_storage_pool_info(pool_id=remote_pool_id) + remote_pool_info = self.remote_operate.query_remote_storage_pool_info( + pool_id=remote_pool_id + ) except Exception as _err: if str(_err).find("1077949965") != -1: err_msg.append("Standby pool[%s] is not exist." % remote_pool_id) else: - err_msg.append("Failed to query remote pool[%s] info, details:%s" % (remote_pool_id, _err)) + err_msg.append( + "Failed to query remote pool[%s] info, details:%s" + % (remote_pool_id, _err) + ) if remote_pool_info: running_status = remote_pool_info.get("RUNNINGSTATUS") health_status = remote_pool_info.get("HEALTHSTATUS") if running_status != PoolStatus.Online: - err_msg.append("Pool running status is not online, current status:[%s]" % - get_status(running_status, PoolStatus)) + err_msg.append( + "Pool running status is not online, current status:[%s]" + % get_status(running_status, PoolStatus) + ) if health_status != PoolHealth.Normal: - err_msg.append("Pool health status is not normal, current status:[%s]" % - get_status(health_status, PoolHealth)) + err_msg.append( + "Pool health status is not normal, current status:[%s]" + % get_status(health_status, PoolHealth) + ) return err_msg def check_standby_filesystem(self) -> list: @@ -265,41 +323,62 @@ class DRDeployPreCheck(object): metadata_in_cantian = self.local_conf_params.get("mysql_metadata_in_cantian") name_suffix = self.local_conf_params.get("name_suffix", "") if name_suffix: - dbstor_page_fs = RepFileSystemNameRule.NamePrefix + dbstor_page_fs + name_suffix + dbstor_page_fs = ( + RepFileSystemNameRule.NamePrefix + dbstor_page_fs + name_suffix + ) if name_suffix and not metadata_in_cantian: - metadata_fs_name = RepFileSystemNameRule.NamePrefix + metadata_fs_name + name_suffix + metadata_fs_name = ( + RepFileSystemNameRule.NamePrefix + metadata_fs_name + name_suffix + ) remote_fs_vstore_id = self.remote_conf_params.get("dbstor_fs_vstore_id") if self.ulog_fs_pair_id is None: dbstor_fs = self.local_conf_params.get("storage_dbstor_fs") - remote_ulog_fs_info = self.remote_operate.\ - query_remote_filesystem_info(fs_name=dbstor_fs, vstore_id=remote_fs_vstore_id) + remote_ulog_fs_info = self.remote_operate.query_remote_filesystem_info( + fs_name=dbstor_fs, vstore_id=remote_fs_vstore_id + ) if remote_ulog_fs_info: - err_msg.append("Standby dbstor filesystem[%s] exist, filesystem id[%s]." % - (dbstor_fs, remote_ulog_fs_info.get("ID"))) + err_msg.append( + "Standby dbstor filesystem[%s] exist, filesystem id[%s]." + % (dbstor_fs, remote_ulog_fs_info.get("ID")) + ) else: if cantian_in_container == "0": - err_msg.append("Standby vstore[%s] exist filesystems." % remote_fs_vstore_id) + err_msg.append( + "Standby vstore[%s] exist filesystems." % remote_fs_vstore_id + ) if self.page_fs_pair_id is None: - remote_dbstor_page_fs_info = self.remote_operate.\ - query_remote_filesystem_info(fs_name=dbstor_page_fs, vstore_id="0") + remote_dbstor_page_fs_info = ( + self.remote_operate.query_remote_filesystem_info( + fs_name=dbstor_page_fs, vstore_id="0" + ) + ) if remote_dbstor_page_fs_info: - err_msg.append("Standby dbstor page filesystem[%s] exist, filesystem id[%s]." % - (dbstor_page_fs, remote_dbstor_page_fs_info.get("ID"))) + err_msg.append( + "Standby dbstor page filesystem[%s] exist, filesystem id[%s]." + % (dbstor_page_fs, remote_dbstor_page_fs_info.get("ID")) + ) else: if cantian_in_container == "0": - err_msg.append("Standby dbstor page filesystem[%s] exist." % dbstor_page_fs) + err_msg.append( + "Standby dbstor page filesystem[%s] exist." % dbstor_page_fs + ) if self.meta_fs_pair_id is None: - remote_metadata_fs_info = self.remote_operate.\ - query_remote_filesystem_info(fs_name=metadata_fs_name, vstore_id="0") + remote_metadata_fs_info = self.remote_operate.query_remote_filesystem_info( + fs_name=metadata_fs_name, vstore_id="0" + ) if remote_metadata_fs_info and not metadata_in_cantian: - err_msg.append("Standby metadata filesystem[%s] exist, filesystem id[%s]." % - (metadata_fs_name, remote_metadata_fs_info.get("ID"))) + err_msg.append( + "Standby metadata filesystem[%s] exist, filesystem id[%s]." + % (metadata_fs_name, remote_metadata_fs_info.get("ID")) + ) else: if cantian_in_container == "0": - err_msg.append("Standby metadata filesystem[%s] exist." % metadata_fs_name) + err_msg.append( + "Standby metadata filesystem[%s] exist." % metadata_fs_name + ) LOG.info("Check standby filesystem nums success.") return err_msg @@ -321,9 +400,15 @@ class DRDeployPreCheck(object): nas_foundation = True err_msg = [] if not nas_foundation: - err_msg.append("NAS Foundation license is not Found or expired, license status: %s" % license_info) + err_msg.append( + "NAS Foundation license is not Found or expired, license status: %s" + % license_info + ) if not hyper_replication: - err_msg.append("HyperReplication license is not Found or expired, license status: %s" % license_info) + err_msg.append( + "HyperReplication license is not Found or expired, license status: %s" + % license_info + ) LOG.info("Check license effectivity success.") return err_msg @@ -338,11 +423,15 @@ class DRDeployPreCheck(object): LOG.info("Check disaster status start.") dbstor_fs = self.local_conf_params.get("storage_dbstor_fs") dbstor_fs_vstore_id = self.local_conf_params.get("dbstor_fs_vstore_id") - remote_dbstor_fs_vstore_id = self.local_conf_params.get("remote_dbstor_fs_vstore_id") + remote_dbstor_fs_vstore_id = self.local_conf_params.get( + "remote_dbstor_fs_vstore_id" + ) dbstor_page_fs = self.local_conf_params.get("storage_dbstor_page_fs") metadata_fs = self.local_conf_params.get("storage_metadata_fs") metadata_in_cantian = self.local_conf_params.get("mysql_metadata_in_cantian") - dbstor_fs_info = self.storage_opt.query_filesystem_info(dbstor_fs, dbstor_fs_vstore_id) + dbstor_fs_info = self.storage_opt.query_filesystem_info( + dbstor_fs, dbstor_fs_vstore_id + ) dbstor_fs_id = dbstor_fs_info.get("ID") dbstor_page_fs_info = self.storage_opt.query_filesystem_info(dbstor_page_fs) metadata_fs_info = self.storage_opt.query_filesystem_info(metadata_fs) @@ -357,7 +446,9 @@ class DRDeployPreCheck(object): else: domain_name = self.local_conf_params.get("domain_name", "") if domain_name == "": - err_msg.append("The 'domain_name' parameter of the 'deploy_param.json' file is empty.") + err_msg.append( + "The 'domain_name' parameter of the 'deploy_param.json' file is empty." + ) domain_exist = False for domain_info in domain_infos: if domain_info.get("NAME") == domain_name: @@ -374,52 +465,88 @@ class DRDeployPreCheck(object): if domain_exist: break else: - _err_msg = "Domain name[%s] is exist, " \ - "but remote esn[%s] matching failed." % (domain_name, remote_esn) + _err_msg = ( + "Domain name[%s] is exist, " + "but remote esn[%s] matching failed." + % (domain_name, remote_esn) + ) err_msg.append(_err_msg) else: LOG.info("Domain name[%s] is not exist." % domain_name) if len(domain_infos) >= DOMAIN_LIMITS and not domain_exist: - err_msg.append("The number of HyperMetro domains has reached the upper limit %s." % DOMAIN_LIMITS) - page_pair_info = self.deploy_operate.query_remote_replication_pair_info(page_fs_id) + err_msg.append( + "The number of HyperMetro domains has reached the upper limit %s." + % DOMAIN_LIMITS + ) + page_pair_info = self.deploy_operate.query_remote_replication_pair_info( + page_fs_id + ) if page_pair_info: if cantian_in_container == "0": - err_msg.append("Filesystem[%s] replication pair is exist." % dbstor_page_fs) + err_msg.append( + "Filesystem[%s] replication pair is exist." % dbstor_page_fs + ) else: - if len(page_pair_info) == 1 and page_pair_info[0].get("REMOTEDEVICEID") == self.remote_device_id: + if ( + len(page_pair_info) == 1 + and page_pair_info[0].get("REMOTEDEVICEID") == self.remote_device_id + ): self.page_fs_pair_id = page_pair_info[0].get("ID") else: - _err_msg = ("Filesystem[%s] replication pair is exist, but match failed, " - "details: %s") % (dbstor_page_fs, page_pair_info) + _err_msg = ( + "Filesystem[%s] replication pair is exist, but match failed, " + "details: %s" + ) % (dbstor_page_fs, page_pair_info) err_msg.append(_err_msg) vstore_pair_infos = self.deploy_operate.query_hyper_metro_vstore_pair_info() for vstore_pair_info in vstore_pair_infos: exist_remote_vstoreid = vstore_pair_info.get("REMOTEVSTOREID") exist_local_vstoreid = vstore_pair_info.get("LOCALVSTOREID") - if exist_local_vstoreid == dbstor_fs_vstore_id and remote_dbstor_fs_vstore_id == exist_remote_vstoreid: + if ( + exist_local_vstoreid == dbstor_fs_vstore_id + and remote_dbstor_fs_vstore_id == exist_remote_vstoreid + ): if cantian_in_container == "0": - err_msg.append("Vstore[%s] metro pair is exist." % dbstor_fs_vstore_id) + err_msg.append( + "Vstore[%s] metro pair is exist." % dbstor_fs_vstore_id + ) else: if vstore_pair_info.get("DOMAINNAME") == domain_name: domain_id = vstore_pair_info.get("DOMAINID") if domain_id != self.hyper_domain_id: - _err_msg = "Vstore[%s] metro pair is exist, " \ - "but domain id[%s] matching failed." % (dbstor_fs_vstore_id, domain_id) + _err_msg = ( + "Vstore[%s] metro pair is exist, " + "but domain id[%s] matching failed." + % (dbstor_fs_vstore_id, domain_id) + ) err_msg.append(_err_msg) self.vstore_pair_id = vstore_pair_info.get("ID") - LOG.info("Vstore[%s] metro pair is exist." % dbstor_fs_vstore_id) + LOG.info( + "Vstore[%s] metro pair is exist." % dbstor_fs_vstore_id + ) else: - _err_msg = "Vstore[%s] metro pair is exist, " \ - "but domain name[%s] matching failed." % (dbstor_fs_vstore_id, domain_name) + _err_msg = ( + "Vstore[%s] metro pair is exist, " + "but domain name[%s] matching failed." + % (dbstor_fs_vstore_id, domain_name) + ) err_msg.append(_err_msg) break else: - system_count = self.remote_operate.query_remote_storage_vstore_filesystem_num(remote_dbstor_fs_vstore_id) + system_count = ( + self.remote_operate.query_remote_storage_vstore_filesystem_num( + remote_dbstor_fs_vstore_id + ) + ) if system_count and system_count.get("COUNT") != "0": - err_msg.append("Standby vstore[%s] exist filesystems, count[%s]" - % (remote_dbstor_fs_vstore_id, system_count.get("COUNT"))) - - ulog_pair_info = self.deploy_operate.query_hyper_metro_filesystem_pair_info(dbstor_fs_id) + err_msg.append( + "Standby vstore[%s] exist filesystems, count[%s]" + % (remote_dbstor_fs_vstore_id, system_count.get("COUNT")) + ) + + ulog_pair_info = self.deploy_operate.query_hyper_metro_filesystem_pair_info( + dbstor_fs_id + ) if ulog_pair_info: if cantian_in_container == "0": err_msg.append("Filesystem[%s] metro pair is exist." % dbstor_fs) @@ -429,21 +556,34 @@ class DRDeployPreCheck(object): self.ulog_fs_pair_id = pair_info.get("ID") LOG.info("Filesystem[%s] metro pair is exist." % dbstor_fs) else: - _err_msg = "Filesystem[%s] metro pair is exist, " \ - "but domain name[%s] matching failed." % (dbstor_fs, domain_name) + _err_msg = ( + "Filesystem[%s] metro pair is exist, " + "but domain name[%s] matching failed." + % (dbstor_fs, domain_name) + ) err_msg.append(_err_msg) deploy_mode = self.local_conf_params.get("deploy_mode") if metadata_in_cantian and deploy_mode != "dbstor": - meta_pair_info = self.deploy_operate.query_remote_replication_pair_info(metadata_fs_id) + meta_pair_info = self.deploy_operate.query_remote_replication_pair_info( + metadata_fs_id + ) if meta_pair_info: if cantian_in_container == "0": - err_msg.append("Filesystem[%s] replication pair is exist." % metadata_fs) + err_msg.append( + "Filesystem[%s] replication pair is exist." % metadata_fs + ) else: - if len(meta_pair_info) == 1 and meta_pair_info[0].get("REMOTEDEVICEID") == self.remote_device_id: + if ( + len(meta_pair_info) == 1 + and meta_pair_info[0].get("REMOTEDEVICEID") + == self.remote_device_id + ): self.meta_fs_pair_id = meta_pair_info[0].get("ID") else: - err_msg.append("Filesystem[%s] replication pair is exist." % metadata_fs) + err_msg.append( + "Filesystem[%s] replication pair is exist." % metadata_fs + ) LOG.info("Check disaster status success.") return err_msg @@ -464,8 +604,10 @@ class DRDeployPreCheck(object): "deploy_mode", ] if not os.path.exists(DEPLOY_PARAM_FILE): - _err_msg = "Deploy param file[%s] is not exists, " \ - "please check cantian is deployed." % DEPLOY_PARAM_FILE + _err_msg = ( + "Deploy param file[%s] is not exists, " + "please check cantian is deployed." % DEPLOY_PARAM_FILE + ) LOG.error(_err_msg) raise Exception(_err_msg) deploy_mode = self.deploy_params.get("deploy_mode") @@ -473,7 +615,9 @@ class DRDeployPreCheck(object): check_list.append("storage_metadata_fs") diff_list = [] for check_item in check_list: - if self.deploy_params.get(check_item) != self.local_conf_params.get(check_item): + if self.deploy_params.get(check_item) != self.local_conf_params.get( + check_item + ): diff_list.append(check_item) if diff_list: err_msg.append("Param check failed, different items[%s]" % diff_list) @@ -495,21 +639,29 @@ class DRDeployPreCheck(object): local_site = self.site remote_site = ({"standby", "active"} - {self.site}).pop() local_dr_deploy_param = conf_params.get("dr_deploy").get(local_site) - local_dr_deploy_param["domain_name"] = conf_params.get("dr_deploy").get("domain_name", "") + local_dr_deploy_param["domain_name"] = conf_params.get("dr_deploy").get( + "domain_name", "" + ) remote_dr_deploy_param = conf_params.get("dr_deploy").get(remote_site) - remote_dr_deploy_param["domain_name"] = conf_params.get("dr_deploy").get("domain_name", "") + remote_dr_deploy_param["domain_name"] = conf_params.get("dr_deploy").get( + "domain_name", "" + ) remote_pool_id = conf_params.get("dr_deploy").get("standby").get("pool_id") - remote_dbstor_fs_vstore_id = conf_params.get("dr_deploy").get("standby").get("dbstor_fs_vstore_id") + remote_dbstor_fs_vstore_id = ( + conf_params.get("dr_deploy").get("standby").get("dbstor_fs_vstore_id") + ) name_suffix = conf_params.get("dr_deploy").get("standby").get("name_suffix", "") del conf_params["dr_deploy"] self.local_conf_params = copy.deepcopy(conf_params) self.local_conf_params.update(local_dr_deploy_param) if self.site == "active": - self.local_conf_params.update({ - "remote_pool_id": remote_pool_id, - "remote_dbstor_fs_vstore_id": remote_dbstor_fs_vstore_id, - "name_suffix": name_suffix - }) + self.local_conf_params.update( + { + "remote_pool_id": remote_pool_id, + "remote_dbstor_fs_vstore_id": remote_dbstor_fs_vstore_id, + "name_suffix": name_suffix, + } + ) self.remote_conf_params = copy.deepcopy(conf_params) self.remote_conf_params.update(remote_dr_deploy_param) deploy_policy = self.local_conf_params.get("deploy_policy", "default") @@ -530,21 +682,31 @@ class DRDeployPreCheck(object): "remote_pool_id": self.local_conf_params.get("remote_pool_id"), "remote_cluster_name": self.local_conf_params.get("remote_cluster_name"), "remote_device_id": self.remote_device_id, - "remote_dbstor_fs_vstore_id": self.local_conf_params.get("remote_dbstor_fs_vstore_id"), + "remote_dbstor_fs_vstore_id": self.local_conf_params.get( + "remote_dbstor_fs_vstore_id" + ), "domain_name": self.local_conf_params.get("domain_name"), "hyper_domain_id": self.hyper_domain_id, "vstore_pair_id": self.vstore_pair_id, - "ulog_fs_pair_id": self.ulog_fs_pair_id + "ulog_fs_pair_id": self.ulog_fs_pair_id, } name_suffix = self.local_conf_params.get("name_suffix") if name_suffix and self.site == "standby": - self.local_conf_params["storage_dbstor_page_fs"] = RepFileSystemNameRule.NamePrefix + \ - self.local_conf_params[ - "storage_dbstor_page_fs"] + name_suffix - if name_suffix and self.site == "standby" and not self.local_conf_params.get("mysql_metadata_in_cantian"): - self.local_conf_params["mysql_metadata_in_cantian"] = RepFileSystemNameRule.NamePrefix + \ - self.local_conf_params[ - "mysql_metadata_in_cantian"] + name_suffix + self.local_conf_params["storage_dbstor_page_fs"] = ( + RepFileSystemNameRule.NamePrefix + + self.local_conf_params["storage_dbstor_page_fs"] + + name_suffix + ) + if ( + name_suffix + and self.site == "standby" + and not self.local_conf_params.get("mysql_metadata_in_cantian") + ): + self.local_conf_params["mysql_metadata_in_cantian"] = ( + RepFileSystemNameRule.NamePrefix + + self.local_conf_params["mysql_metadata_in_cantian"] + + name_suffix + ) self.local_conf_params.update(dr_params) del self.local_conf_params["node_id"] write_json_config(DR_DEPLOY_PARAM_FILE, self.local_conf_params) @@ -552,14 +714,18 @@ class DRDeployPreCheck(object): def init_opt(self): local_login_ip = self.local_conf_params.get("dm_ip") local_login_user = self.local_conf_params.get("dm_user") - storage_operate = StorageInf((local_login_ip, local_login_user, self.dm_login_passwd)) + storage_operate = StorageInf( + (local_login_ip, local_login_user, self.dm_login_passwd) + ) storage_operate.login() self.storage_opt = storage_operate self.deploy_operate = DRDeployCommon(storage_operate) def parse_input_params(self): parse_params = argparse.ArgumentParser() - parse_params.add_argument("-s", "--site", dest="site", choices=['active', "standby"], required=True) + parse_params.add_argument( + "-s", "--site", dest="site", choices=["active", "standby"], required=True + ) parse_params.add_argument("-l", "--conf", dest="conf", required=True) args = parse_params.parse_args() self.site = args.site @@ -572,25 +738,43 @@ class DRDeployPreCheck(object): check_result = [] if self.site == "standby": return check_result - if not os.path.exists(os.path.join(CURRENT_PATH, "../../../config/deploy_param.json")): - shutil.copy("/opt/cantian/config/deploy_param.json", os.path.join(CURRENT_PATH, "../../../config")) + if not os.path.exists( + os.path.join(CURRENT_PATH, "../../../config/deploy_param.json") + ): + shutil.copy( + "/opt/cantian/config/deploy_param.json", + os.path.join(CURRENT_PATH, "../../../config"), + ) return check_result self.deploy_params = read_json_config(DEPLOY_PARAM_FILE) check_result.extend(self.check_master_cantian_status()) - check_result.extend(self.check_file_system_status( - fs_name=self.local_conf_params.get("storage_dbstor_page_fs"), - vstore_id="0")) - check_result.extend(self.check_file_system_status( - fs_name=self.local_conf_params.get("storage_dbstor_fs"), - vstore_id=self.local_conf_params.get("dbstor_fs_vstore_id"))) + check_result.extend( + self.check_file_system_status( + fs_name=self.local_conf_params.get("storage_dbstor_page_fs"), + vstore_id="0", + ) + ) + check_result.extend( + self.check_file_system_status( + fs_name=self.local_conf_params.get("storage_dbstor_fs"), + vstore_id=self.local_conf_params.get("dbstor_fs_vstore_id"), + ) + ) check_result.extend(self.check_active_exist_params()) if not self.local_conf_params.get("mysql_metadata_in_cantian"): - check_result.extend(self.check_file_system_status( - fs_name=self.local_conf_params.get("storage_metadata_fs"), vstore_id="0")) + check_result.extend( + self.check_file_system_status( + fs_name=self.local_conf_params.get("storage_metadata_fs"), + vstore_id="0", + ) + ) speed_val_list = ["low", "medium", "high", "highest"] sync_speed = self.local_conf_params.get("sync_speed", "medium") if sync_speed not in speed_val_list: - check_result.append("sync_speed[%s] is invalid, the option is %s" % (sync_speed, speed_val_list)) + check_result.append( + "sync_speed[%s] is invalid, the option is %s" + % (sync_speed, speed_val_list) + ) return check_result def check_nfs_lif_info(self): @@ -604,29 +788,45 @@ class DRDeployPreCheck(object): archive_logic_ip = self.local_conf_params.get("archive_logic_ip") metadata_logic_ip = self.local_conf_params.get("metadata_logic_ip") deploy_mode = self.local_conf_params.get("deploy_mode") - mysql_metadata_in_cantian = self.local_conf_params.get("mysql_metadata_in_cantian") - share_fs_info = self.storage_opt.query_filesystem_info(storage_share_fs, vstore_id="0") + mysql_metadata_in_cantian = self.local_conf_params.get( + "mysql_metadata_in_cantian" + ) + share_fs_info = self.storage_opt.query_filesystem_info( + storage_share_fs, vstore_id="0" + ) if not share_fs_info: check_result.append(err_msg % ("storage_share_fs", storage_share_fs)) if deploy_mode == "combined": - meta_lif_info = self.storage_opt.query_logical_port_info(metadata_logic_ip, vstore_id="0") + meta_lif_info = self.storage_opt.query_logical_port_info( + metadata_logic_ip, vstore_id="0" + ) if not meta_lif_info: check_result.append(err_msg % ("metadata_logic_ip", metadata_logic_ip)) - meta_fs_info = self.storage_opt.query_filesystem_info(storage_metadata_fs, vstore_id="0") + meta_fs_info = self.storage_opt.query_filesystem_info( + storage_metadata_fs, vstore_id="0" + ) if mysql_metadata_in_cantian and not meta_fs_info: - check_result.append(err_msg % ("storage_metadata_fs", storage_metadata_fs)) + check_result.append( + err_msg % ("storage_metadata_fs", storage_metadata_fs) + ) system_nfs_service = self.storage_opt.query_nfs_service(vstore_id="0") support_v41 = system_nfs_service.get("SUPPORTV41") if support_v41 == "false": check_result.append("System vstore nfs service[v4.1] is not support.") if db_type == "1" and deploy_mode == "combined": - archive_lif_info = self.storage_opt.query_logical_port_info(archive_logic_ip, vstore_id="0") + archive_lif_info = self.storage_opt.query_logical_port_info( + archive_logic_ip, vstore_id="0" + ) if not archive_lif_info: check_result.append(err_msg % ("archive_logic_ip", archive_logic_ip)) if db_type == "1": - archive_fs_info = self.storage_opt.query_filesystem_info(storage_archive_fs, vstore_id="0") + archive_fs_info = self.storage_opt.query_filesystem_info( + storage_archive_fs, vstore_id="0" + ) if not archive_fs_info: - check_result.append(err_msg % ("storage_archive_fs", storage_archive_fs)) + check_result.append( + err_msg % ("storage_archive_fs", storage_archive_fs) + ) return check_result def check_standby_params(self): @@ -648,15 +848,21 @@ class DRDeployPreCheck(object): check_result.append("Params check failed") conf_params = read_json_config(self.conf) dbstor_fs_vstore_id = conf_params.get("dbstor_fs_vstore_id") - remote_dbstor_fs_vstore_id = conf_params.get("dr_deploy").get("standby").get("dbstor_fs_vstore_id") + remote_dbstor_fs_vstore_id = ( + conf_params.get("dr_deploy").get("standby").get("dbstor_fs_vstore_id") + ) if dbstor_fs_vstore_id != remote_dbstor_fs_vstore_id: - check_result.append("Inconsistent dbstor fs vstore id, %s and %s" % (dbstor_fs_vstore_id, - remote_dbstor_fs_vstore_id)) + check_result.append( + "Inconsistent dbstor fs vstore id, %s and %s" + % (dbstor_fs_vstore_id, remote_dbstor_fs_vstore_id) + ) return check_result try: self.deploy_operate.storage_opt.query_vstore_info(dbstor_fs_vstore_id) except Exception as err: - check_result.append("Vstore[%s] is not exist, details: %s" % (dbstor_fs_vstore_id, str(err))) + check_result.append( + "Vstore[%s] is not exist, details: %s" % (dbstor_fs_vstore_id, str(err)) + ) return check_result check_result.extend(self.check_nfs_lif_info()) LOG.info("Param check success") @@ -667,8 +873,10 @@ class DRDeployPreCheck(object): remote_cluster_name = self.local_conf_params.get("remote_cluster_name") cluster_name = self.local_conf_params.get("cluster_name") if cluster_name != remote_cluster_name: - check_result.append("Inconsistent cluster names, remote[%s], local[%s]." - % (remote_cluster_name, cluster_name)) + check_result.append( + "Inconsistent cluster names, remote[%s], local[%s]." + % (remote_cluster_name, cluster_name) + ) return check_result def check_standby_install(self): @@ -686,7 +894,9 @@ class DRDeployPreCheck(object): ctom_result_code, _, _ = exec_popen(check_ctom_cmd) if not cantain_result_code or not ctom_result_code: check_result.append("Cantian standby has been installed, please check!") - install_json_path = os.path.join(CURRENT_PATH, "../../cantian/install_config.json") + install_json_path = os.path.join( + CURRENT_PATH, "../../cantian/install_config.json" + ) install_json_data = read_json_config(install_json_path) root_dir = os.path.join(CURRENT_PATH, "../../../../") if install_json_data.get("M_RUNING_MODE") == "cantiand_with_mysql_in_cluster": @@ -719,7 +929,12 @@ class DRDeployPreCheck(object): check_result.extend(self.check_disaster_exist()) check_result.extend(self.check_standby_filesystem()) if check_result: - _err = "\n".join([" " * 8 + str(index + 1) + "." + err for index, err in enumerate(check_result)]) + _err = "\n".join( + [ + " " * 8 + str(index + 1) + "." + err + for index, err in enumerate(check_result) + ] + ) _err = "DR deploy pre_check failed, details:\n" + _err raise Exception(str(_err)) finally: @@ -760,7 +975,9 @@ class ParamCheck(object): for i in range(3): try: time.sleep(10) - mysql_shell = MysqlShell(self.mysql_cmd, user=self.mysql_user, password=mysql_pwd) + mysql_shell = MysqlShell( + self.mysql_cmd, user=self.mysql_user, password=mysql_pwd + ) mysql_shell.start_session() mysql_shell.close_session() break @@ -775,9 +992,15 @@ class ParamCheck(object): parse_params = argparse.ArgumentParser() parse_params.add_argument("--action", dest="action", required=True) parse_params.add_argument("--site", dest="site", required=False, default="") - parse_params.add_argument("--mysql_cmd", dest="mysql_cmd", required=False, default="") - parse_params.add_argument("--mysql_user", dest="mysql_user", required=False, default="") - parse_params.add_argument("--display", dest="display", required=False, default="") + parse_params.add_argument( + "--mysql_cmd", dest="mysql_cmd", required=False, default="" + ) + parse_params.add_argument( + "--mysql_user", dest="mysql_user", required=False, default="" + ) + parse_params.add_argument( + "--display", dest="display", required=False, default="" + ) args = parse_params.parse_args() self.action = args.action self.site = args.site @@ -785,6 +1008,8 @@ class ParamCheck(object): self.mysql_user = args.mysql_user dm_pwd = input() self.check_dm_pwd(dm_pwd) - if self.site == "active" and (self.action == "deploy" or self.action == "full_sync"): + if self.site == "active" and ( + self.action == "deploy" or self.action == "full_sync" + ): my_pwd = input() self.check_mysql_pwd(my_pwd) diff --git a/pkg/deploy/action/storage_operate/dr_deploy_operate/dr_deploy_progress_query.py b/pkg/deploy/action/storage_operate/dr_deploy_operate/dr_deploy_progress_query.py index 3d1b8547..d0c6987c 100644 --- a/pkg/deploy/action/storage_operate/dr_deploy_operate/dr_deploy_progress_query.py +++ b/pkg/deploy/action/storage_operate/dr_deploy_operate/dr_deploy_progress_query.py @@ -10,12 +10,21 @@ from storage_operate.dr_deploy_operate.dr_deploy_common import DRDeployCommon from logic.common_func import read_json_config from logic.common_func import exec_popen from logic.storage_operate import StorageInf -from utils.config.rest_constant import HealthStatus, VstorePairConfigStatus, VstorePairRunningStatus, \ - MetroDomainRunningStatus, ReplicationRunningStatus +from utils.config.rest_constant import ( + HealthStatus, + VstorePairConfigStatus, + VstorePairRunningStatus, + MetroDomainRunningStatus, + ReplicationRunningStatus, +) CURRENT_PATH = os.path.dirname(os.path.abspath(__file__)) -LOCAL_PROCESS_RECORD_FILE = os.path.join(CURRENT_PATH, "../../../config/dr_process_record.json") -FULL_SYNC_PROGRESS = os.path.join(CURRENT_PATH, "../../../config/full_sync_progress.json") +LOCAL_PROCESS_RECORD_FILE = os.path.join( + CURRENT_PATH, "../../../config/dr_process_record.json" +) +FULL_SYNC_PROGRESS = os.path.join( + CURRENT_PATH, "../../../config/full_sync_progress.json" +) DR_DEPLOY_CONFIG = os.path.join(CURRENT_PATH, "../../../config/dr_deploy_param.json") DR_STATUS = os.path.join(CURRENT_PATH, "../../../config/dr_status.json") @@ -29,10 +38,14 @@ class DrDeployQuery(object): data = process_data.get("data") table = "" table += "-" * 68 + "\n" - table += "|" + "task".center(50, " ") + "|" + "status".center(15, " ") + "|" + "\n" + table += ( + "|" + "task".center(50, " ") + "|" + "status".center(15, " ") + "|" + "\n" + ) table += "-" * 68 + "\n" for key, value in data.items(): - table += "|" + key.center(50, " ") + "|" + value.center(15, " ") + "|" + "\n" + table += ( + "|" + key.center(50, " ") + "|" + value.center(15, " ") + "|" + "\n" + ) error = process_data.get("error") code = error.get("code") if code != 0: @@ -43,7 +56,7 @@ class DrDeployQuery(object): err_list = [] for _err in err_msg_list: for i in range(0, len(_err), 62): - err_list.append(_err[i:i + 62]) + err_list.append(_err[i : i + 62]) for _err in err_list: table += "| " + _err.ljust(62, " ") + " |" + "\n" table += "-" * 68 + "\n" @@ -70,8 +83,10 @@ class DrDeployQuery(object): data["dr_deploy"] = "failed" error["code"] = -1 if description == "": - error["description"] = "The process exits abnormally," \ - "see /opt/cantian/log/deploy/om_deploy/dr_deploy.log for more details." + error["description"] = ( + "The process exits abnormally," + "see /opt/cantian/log/deploy/om_deploy/dr_deploy.log for more details." + ) table_data = self.table_format(process_data) json_data = json.dumps(process_data, indent=4) return json_data if is_json_display else table_data @@ -89,16 +104,30 @@ class DrStatusCheck(object): def table_format(statuses: dict) -> str: table = "" table += "-" * 68 + "\n" - table += "|" + "Component".center(50, " ") + "|" + "Status".center(15, " ") + "|\n" + table += ( + "|" + "Component".center(50, " ") + "|" + "Status".center(15, " ") + "|\n" + ) table += "-" * 68 + "\n" for key, value in statuses.items(): if key == "dr_status": continue - table += "|" + key.replace("_", " ").capitalize().center(50, " ") + "|" + value.center(15, " ") + "|\n" + table += ( + "|" + + key.replace("_", " ").capitalize().center(50, " ") + + "|" + + value.center(15, " ") + + "|\n" + ) table += "-" * 68 + "\n" - table += "|" + "DR Status".center(50, " ") + "|" + statuses["dr_status"].center(15, " ") + "|\n" + table += ( + "|" + + "DR Status".center(50, " ") + + "|" + + statuses["dr_status"].center(15, " ") + + "|\n" + ) table += "-" * 68 + "\n" return table @@ -115,7 +144,9 @@ class DrStatusCheck(object): def query_domain_status(self) -> str: hyper_domain_id = self.dr_deploy_info.get("hyper_domain_id") try: - domain_info = self.dr_deploy_opt.query_hyper_metro_domain_info(hyper_domain_id) + domain_info = self.dr_deploy_opt.query_hyper_metro_domain_info( + hyper_domain_id + ) if domain_info: domain_running_status = domain_info.get("RUNNINGSTATUS") if domain_running_status == MetroDomainRunningStatus.Normal: @@ -129,14 +160,24 @@ class DrStatusCheck(object): def query_vstore_pair_status(self) -> str: vstore_pair_id = self.dr_deploy_info.get("vstore_pair_id") try: - vstore_pair_info = self.dr_deploy_opt.query_hyper_metro_vstore_pair_info(vstore_pair_id) + vstore_pair_info = self.dr_deploy_opt.query_hyper_metro_vstore_pair_info( + vstore_pair_id + ) if vstore_pair_info: vstore_running_status = vstore_pair_info.get("RUNNINGSTATUS") - if (vstore_running_status == VstorePairRunningStatus.Normal and - vstore_pair_info.get("HEALTHSTATUS") == HealthStatus.Normal): - if vstore_pair_info.get("CONFIGSTATUS") == VstorePairConfigStatus.Synchronizing: + if ( + vstore_running_status == VstorePairRunningStatus.Normal + and vstore_pair_info.get("HEALTHSTATUS") == HealthStatus.Normal + ): + if ( + vstore_pair_info.get("CONFIGSTATUS") + == VstorePairConfigStatus.Synchronizing + ): return "Running" - if vstore_pair_info.get("CONFIGSTATUS") == VstorePairConfigStatus.Normal: + if ( + vstore_pair_info.get("CONFIGSTATUS") + == VstorePairConfigStatus.Normal + ): return "Normal" else: return "Abnormal" @@ -147,15 +188,26 @@ class DrStatusCheck(object): def query_ulog_fs_pair_status(self) -> str: filesystem_pair_id = self.dr_deploy_info.get("ulog_fs_pair_id") try: - filesystem_pair_info = self.dr_deploy_opt.query_hyper_metro_filesystem_pair_info_by_pair_id( - pair_id=filesystem_pair_id) + filesystem_pair_info = ( + self.dr_deploy_opt.query_hyper_metro_filesystem_pair_info_by_pair_id( + pair_id=filesystem_pair_id + ) + ) if filesystem_pair_info: ulog_fs_running_status = filesystem_pair_info.get("RUNNINGSTATUS") - if (ulog_fs_running_status == VstorePairRunningStatus.Normal and - filesystem_pair_info.get("HEALTHSTATUS") == HealthStatus.Normal): - if filesystem_pair_info.get("CONFIGSTATUS") == VstorePairConfigStatus.Synchronizing: + if ( + ulog_fs_running_status == VstorePairRunningStatus.Normal + and filesystem_pair_info.get("HEALTHSTATUS") == HealthStatus.Normal + ): + if ( + filesystem_pair_info.get("CONFIGSTATUS") + == VstorePairConfigStatus.Synchronizing + ): return "Runing" - if filesystem_pair_info.get("CONFIGSTATUS") == VstorePairConfigStatus.Normal: + if ( + filesystem_pair_info.get("CONFIGSTATUS") + == VstorePairConfigStatus.Normal + ): return "Normal" else: return "Abnormal" @@ -166,9 +218,13 @@ class DrStatusCheck(object): def query_page_fs_pair_status(self) -> str: dbstor_page_fs_name = self.dr_deploy_info.get("storage_dbstor_page_fs") try: - dbstor_page_fs_info = self.dr_deploy_opt.storage_opt.query_filesystem_info(dbstor_page_fs_name) + dbstor_page_fs_info = self.dr_deploy_opt.storage_opt.query_filesystem_info( + dbstor_page_fs_name + ) dbstor_page_fs_id = dbstor_page_fs_info.get("ID") - page_fs_pair_info = self.dr_deploy_opt.query_remote_replication_pair_info(dbstor_page_fs_id) + page_fs_pair_info = self.dr_deploy_opt.query_remote_replication_pair_info( + dbstor_page_fs_id + ) if page_fs_pair_info: if page_fs_pair_info[0].get("HEALTHSTATUS") == HealthStatus.Normal: return "Normal" @@ -188,11 +244,15 @@ class DrStatusCheck(object): deploy_user = self.dr_deploy_info.get("deploy_user").strip().split(":")[0] if deploy_mode == "dbstor": storage_fs = self.dr_deploy_info.get("storage_share_fs") - cmd = (f"su -s /bin/bash - {deploy_user} -c 'dbstor --query-file " - f"--fs-name={storage_fs} --file-dir=/' | grep 'dr_deploy_param.json' | wc -l") + cmd = ( + f"su -s /bin/bash - {deploy_user} -c 'dbstor --query-file " + f"--fs-name={storage_fs} --file-dir=/' | grep 'dr_deploy_param.json' | wc -l" + ) else: storage_fs = self.dr_deploy_info.get("storage_metadata_fs") - if os.path.exists(f"/mnt/dbdata/remote/metadata_{storage_fs}/dr_deploy_param.json"): + if os.path.exists( + f"/mnt/dbdata/remote/metadata_{storage_fs}/dr_deploy_param.json" + ): return "Normal" return "Abnormal" code, count, err = exec_popen(cmd, timeout=180) @@ -210,7 +270,9 @@ class DrStatusCheck(object): os.makedirs(os.path.dirname(DR_STATUS), exist_ok=True) flags = os.O_WRONLY | os.O_CREAT | os.O_TRUNC mode = stat.S_IWUSR | stat.S_IRUSR - with os.fdopen(os.open(DR_STATUS, flags, mode), "w", encoding='utf-8') as file: + with os.fdopen( + os.open(DR_STATUS, flags, mode), "w", encoding="utf-8" + ) as file: json.dump(statuses, file, indent=4) os.chmod(DR_STATUS, 0o640) except Exception as e: @@ -226,33 +288,53 @@ class DrStatusCheck(object): "ulog_fs_pair_status": "Unknown", "page_fs_pair_status": "Unknown", "dr_status_file": "Unknown", - "dr_status": "Abnormal" + "dr_status": "Abnormal", } statuses["domain_status"] = self.query_domain_status() if statuses["domain_status"] in ["Unknown", "Abnormal"]: self.update_dr_status_file(statuses) - return json.dumps(statuses, indent=4) if is_json_display else self.table_format(statuses) + return ( + json.dumps(statuses, indent=4) + if is_json_display + else self.table_format(statuses) + ) statuses["vstore_pair_status"] = self.query_vstore_pair_status() if statuses["vstore_pair_status"] in ["Unknown", "Abnormal", "Running"]: self.update_dr_status_file(statuses) - return json.dumps(statuses, indent=4) if is_json_display else self.table_format(statuses) + return ( + json.dumps(statuses, indent=4) + if is_json_display + else self.table_format(statuses) + ) statuses["ulog_fs_pair_status"] = self.query_ulog_fs_pair_status() if statuses["ulog_fs_pair_status"] in ["Unknown", "Abnormal", "Running"]: self.update_dr_status_file(statuses) - return json.dumps(statuses, indent=4) if is_json_display else self.table_format(statuses) + return ( + json.dumps(statuses, indent=4) + if is_json_display + else self.table_format(statuses) + ) statuses["page_fs_pair_status"] = self.query_page_fs_pair_status() if statuses["page_fs_pair_status"] in ["Unknown", "Abnormal"]: self.update_dr_status_file(statuses) - return json.dumps(statuses, indent=4) if is_json_display else self.table_format(statuses) + return ( + json.dumps(statuses, indent=4) + if is_json_display + else self.table_format(statuses) + ) statuses["dr_status_file"] = self.query_dr_status_file() if statuses["dr_status_file"] in ["Abnormal"]: self.update_dr_status_file(statuses) - return json.dumps(statuses, indent=4) if is_json_display else self.table_format(statuses) + return ( + json.dumps(statuses, indent=4) + if is_json_display + else self.table_format(statuses) + ) if all(status == "Normal" for status in list(statuses.values())[:-1]): statuses["dr_status"] = "Normal" @@ -261,7 +343,11 @@ class DrStatusCheck(object): self.update_dr_status_file(statuses) - return json.dumps(statuses, indent=4) if is_json_display else self.table_format(statuses) + return ( + json.dumps(statuses, indent=4) + if is_json_display + else self.table_format(statuses) + ) class FullSyncProgress(DrDeployQuery): @@ -290,8 +376,10 @@ class FullSyncProgress(DrDeployQuery): data["full_sync"] = "failed" error["code"] = -1 if description == "": - error["description"] = "The process exits abnormally," \ - "see /opt/cantian/log/deploy/om_deploy/dr_deploy.log for more details." + error["description"] = ( + "The process exits abnormally," + "see /opt/cantian/log/deploy/om_deploy/dr_deploy.log for more details." + ) table_data = self.table_format(process_data) json_data = json.dumps(process_data, indent=4) return json_data if is_json_display else table_data @@ -303,8 +391,12 @@ class ProgressQuery(object): @staticmethod def execute(action=None, display=None): parse_params = argparse.ArgumentParser() - parse_params.add_argument("--action", dest="action", required=False, default="deploy") - parse_params.add_argument("--display", dest="display", required=False, default="json") + parse_params.add_argument( + "--action", dest="action", required=False, default="deploy" + ) + parse_params.add_argument( + "--display", dest="display", required=False, default="json" + ) args = parse_params.parse_args() if action is None: action = args.action @@ -321,4 +413,4 @@ class ProgressQuery(object): result = full_sync_progress.execute(display) else: result = "Invalid input." - print(result) \ No newline at end of file + print(result) diff --git a/pkg/deploy/action/storage_operate/dr_deploy_operate/dr_deploy_switchover.py b/pkg/deploy/action/storage_operate/dr_deploy_operate/dr_deploy_switchover.py index 20ddcda6..771b4d6f 100644 --- a/pkg/deploy/action/storage_operate/dr_deploy_operate/dr_deploy_switchover.py +++ b/pkg/deploy/action/storage_operate/dr_deploy_operate/dr_deploy_switchover.py @@ -9,8 +9,15 @@ from logic.common_func import read_json_config, get_status, exec_popen from logic.storage_operate import StorageInf from storage_operate.dr_deploy_operate.dr_deploy_common import DRDeployCommon from om_log import LOGGER as LOG -from utils.config.rest_constant import DomainAccess, MetroDomainRunningStatus, VstorePairRunningStatus, HealthStatus, \ - ConfigRole, DataIntegrityStatus, ReplicationRunningStatus +from utils.config.rest_constant import ( + DomainAccess, + MetroDomainRunningStatus, + VstorePairRunningStatus, + HealthStatus, + ConfigRole, + DataIntegrityStatus, + ReplicationRunningStatus, +) from get_config_info import get_env_info RUN_USER = get_env_info("cantian_user") @@ -19,15 +26,17 @@ DR_DEPLOY_CONFIG = os.path.join(CURRENT_PATH, "../../../config/dr_deploy_param.j DEPLOY_PARAMS_CONFIG = os.path.join(CURRENT_PATH, "../../../config/deploy_param.json") LOGICREP_APPCTL_FILE = os.path.join(CURRENT_PATH, "../../logicrep/appctl.sh") EXEC_SQL = os.path.join(CURRENT_PATH, "../../cantian_common/exec_sql.py") -CANTIAN_DISASTER_RECOVERY_STATUS_CHECK = 'echo -e "select DATABASE_ROLE from DV_LRPL_DETAIL;" | '\ - 'su -s /bin/bash - %s -c \'source ~/.bashrc && '\ - 'export LD_LIBRARY_PATH=/opt/cantian/dbstor/lib:${LD_LIBRARY_PATH} && '\ - 'python3 -B %s\'' % (RUN_USER, EXEC_SQL) +CANTIAN_DISASTER_RECOVERY_STATUS_CHECK = ( + 'echo -e "select DATABASE_ROLE from DV_LRPL_DETAIL;" | ' + "su -s /bin/bash - %s -c 'source ~/.bashrc && " + "export LD_LIBRARY_PATH=/opt/cantian/dbstor/lib:${LD_LIBRARY_PATH} && " + "python3 -B %s'" % (RUN_USER, EXEC_SQL) +) DBSTOR_CHECK_VERSION_FILE = "/opt/cantian/dbstor/tools/cs_baseline.sh" def load_json_file(file_path): - with open(file_path, 'r') as f: + with open(file_path, "r") as f: return json.load(f) @@ -55,8 +64,10 @@ class SwitchOver(object): if check_time < 20: check_count = 1 LOG.info("Check cantian status.") - cmd = "su -s /bin/bash - %s -c \"cms stat | " \ - "grep -v STAT | awk '{print \$1, \$3, \$6}'\"" % self.run_user + cmd = ( + 'su -s /bin/bash - %s -c "cms stat | ' + "grep -v STAT | awk '{print \$1, \$3, \$6}'\"" % self.run_user + ) check_time_step = check_time // check_count while check_time: time.sleep(check_time_step) @@ -68,7 +79,10 @@ class SwitchOver(object): raise Exception(err_msg) cms_stat = output.split("\n") if len(cms_stat) < 2: - err_msg = "Current cluster status is abnormal, output:%s, stderr:%s" % (output, stderr) + err_msg = "Current cluster status is abnormal, output:%s, stderr:%s" % ( + output, + stderr, + ) LOG.error(err_msg) raise Exception(err_msg) online_flag = True @@ -80,7 +94,11 @@ class SwitchOver(object): if (online != "ONLINE" or work_stat != "1") and node_id == target_node: online_flag = False if not online_flag: - LOG.info("Current cluster status is abnormal, output:%s, stderr:%s", output, stderr) + LOG.info( + "Current cluster status is abnormal, output:%s, stderr:%s", + output, + stderr, + ) continue else: break @@ -94,19 +112,33 @@ class SwitchOver(object): def query_sync_status(self): while True: - vstore_pair_info = self.dr_deploy_opt.query_hyper_metro_vstore_pair_info(self.vstore_pair_id) + vstore_pair_info = self.dr_deploy_opt.query_hyper_metro_vstore_pair_info( + self.vstore_pair_id + ) health_status = vstore_pair_info.get("HEALTHSTATUS") running_status = vstore_pair_info.get("RUNNINGSTATUS") - LOG.info("Vstore pair sync running, running status[%s]", - get_status(running_status, VstorePairRunningStatus)) - if running_status == VstorePairRunningStatus.Invalid or health_status == HealthStatus.Faulty: - err_msg = "Hyper metro vstore pair status is not normal, " \ - "health_status[%s], running_status[%s], details: %s" % \ - (get_status(health_status, HealthStatus), - get_status(running_status, VstorePairRunningStatus), - vstore_pair_info) + LOG.info( + "Vstore pair sync running, running status[%s]", + get_status(running_status, VstorePairRunningStatus), + ) + if ( + running_status == VstorePairRunningStatus.Invalid + or health_status == HealthStatus.Faulty + ): + err_msg = ( + "Hyper metro vstore pair status is not normal, " + "health_status[%s], running_status[%s], details: %s" + % ( + get_status(health_status, HealthStatus), + get_status(running_status, VstorePairRunningStatus), + vstore_pair_info, + ) + ) LOG.error(err_msg) - if running_status == VstorePairRunningStatus.Normal and health_status == HealthStatus.Normal: + if ( + running_status == VstorePairRunningStatus.Normal + and health_status == HealthStatus.Normal + ): LOG.info("Vstore pair sync complete.") break time.sleep(60) @@ -122,8 +154,10 @@ class SwitchOver(object): def standby_cms_res_stop(self): LOG.info("Standby stop by cms command.") - cmd = "source ~/.bashrc && su -s /bin/bash - %s -c " \ - "\"cms res -stop db\"" % self.run_user + cmd = ( + "source ~/.bashrc && su -s /bin/bash - %s -c " + '"cms res -stop db"' % self.run_user + ) return_code, output, stderr = exec_popen(cmd, timeout=600) if return_code: err_msg = "Cantian stop failed, error:%s." % output + stderr @@ -132,8 +166,10 @@ class SwitchOver(object): def standby_cms_res_start(self): LOG.info("Standby start by cms command.") - cmd = "source ~/.bashrc && su -s /bin/bash - %s -c " \ - "\"cms res -start db\"" % self.run_user + cmd = ( + "source ~/.bashrc && su -s /bin/bash - %s -c " + '"cms res -start db"' % self.run_user + ) return_code, output, stderr = exec_popen(cmd, timeout=600) if return_code: err_msg = "Cantian start failed, error:%s." % output + stderr @@ -151,8 +187,10 @@ class SwitchOver(object): return outputs def wait_res_stop(self): - cmd = "su -s /bin/bash - %s -c \"cms stat | " \ - "grep -v STAT | awk '{print \$1, \$3}'\"" % self.run_user + cmd = ( + 'su -s /bin/bash - %s -c "cms stat | ' + "grep -v STAT | awk '{print \$1, \$3}'\"" % self.run_user + ) wait_time = 30 wait_time_step = 2 while wait_time: @@ -196,8 +234,10 @@ class SwitchOver(object): :return: """ LOG.info(f"Standby set iof[{iof}].") - cmd = "su -s /bin/bash - %s -c \"source ~/.bashrc && "\ - "dbstor --io-forbidden %s\"" % (self.run_user, iof) + cmd = ( + 'su -s /bin/bash - %s -c "source ~/.bashrc && ' + 'dbstor --io-forbidden %s"' % (self.run_user, iof) + ) return_code, output, stderr = exec_popen(cmd, timeout=60) if return_code: err_msg = "set iof failed, error:%s." % output + stderr @@ -224,7 +264,9 @@ class SwitchOver(object): """ LOG.info("Start querying the replay.") while True: - return_code, output, stderr = exec_popen(CANTIAN_DISASTER_RECOVERY_STATUS_CHECK, timeout=20) + return_code, output, stderr = exec_popen( + CANTIAN_DISASTER_RECOVERY_STATUS_CHECK, timeout=20 + ) if return_code: err_msg = "Query database role failed, error:%s." % output + stderr LOG.error(err_msg) @@ -258,41 +300,70 @@ class SwitchOver(object): LOG.info("Active/standby switch start.") self.check_cluster_status(target_node=self.node_id) self.init_storage_opt() - pair_info = self.dr_deploy_opt.query_hyper_metro_filesystem_pair_info_by_pair_id(self.ulog_fs_pair_id) + pair_info = ( + self.dr_deploy_opt.query_hyper_metro_filesystem_pair_info_by_pair_id( + self.ulog_fs_pair_id + ) + ) local_data_status = pair_info.get("LOCALDATASTATE") remote_data_status = pair_info.get("REMOTEDATASTATE") - if local_data_status == DataIntegrityStatus.inconsistent or remote_data_status == DataIntegrityStatus.inconsistent: + if ( + local_data_status == DataIntegrityStatus.inconsistent + or remote_data_status == DataIntegrityStatus.inconsistent + ): err_msg = "Data is inconsistent, please check." LOG.error(err_msg) raise Exception(err_msg) - domain_info = self.dr_deploy_opt.query_hyper_metro_domain_info(self.hyper_domain_id) + domain_info = self.dr_deploy_opt.query_hyper_metro_domain_info( + self.hyper_domain_id + ) running_status = domain_info.get("RUNNINGSTATUS") config_role = domain_info.get("CONFIGROLE") - if running_status != MetroDomainRunningStatus.Normal and running_status != MetroDomainRunningStatus.Split: - err_msg = "DR recover operation is not allowed in %s status." % \ - get_status(running_status, MetroDomainRunningStatus) + if ( + running_status != MetroDomainRunningStatus.Normal + and running_status != MetroDomainRunningStatus.Split + ): + err_msg = "DR recover operation is not allowed in %s status." % get_status( + running_status, MetroDomainRunningStatus + ) LOG.error(err_msg) raise Exception(err_msg) - if config_role == ConfigRole.Primary and running_status == MetroDomainRunningStatus.Normal: + if ( + config_role == ConfigRole.Primary + and running_status == MetroDomainRunningStatus.Normal + ): self.standby_logicrep_stop() self.standby_cms_res_stop() self.wait_res_stop() self.dr_deploy_opt.split_filesystem_hyper_metro_domain(self.hyper_domain_id) self.dr_deploy_opt.change_fs_hyper_metro_domain_second_access( - self.hyper_domain_id, DomainAccess.ReadAndWrite) + self.hyper_domain_id, DomainAccess.ReadAndWrite + ) self.dr_deploy_opt.swap_role_fs_hyper_metro_domain(self.hyper_domain_id) - self.dr_deploy_opt.change_fs_hyper_metro_domain_second_access(self.hyper_domain_id, DomainAccess.ReadOnly) + self.dr_deploy_opt.change_fs_hyper_metro_domain_second_access( + self.hyper_domain_id, DomainAccess.ReadOnly + ) self.dr_deploy_opt.join_fs_hyper_metro_domain(self.hyper_domain_id) self.query_sync_status() - pair_info = self.dr_deploy_opt.query_remote_replication_pair_info_by_pair_id(self.page_fs_pair_id) + pair_info = ( + self.dr_deploy_opt.query_remote_replication_pair_info_by_pair_id( + self.page_fs_pair_id + ) + ) page_role = pair_info.get("ISPRIMARY") if page_role == "true": self.dr_deploy_opt.swap_role_replication_pair(self.page_fs_pair_id) else: - LOG.info("Page fs rep pair is already standby site, pair_id[%s].", self.page_fs_pair_id) + LOG.info( + "Page fs rep pair is already standby site, pair_id[%s].", + self.page_fs_pair_id, + ) if not self.metadata_in_cantian: - meta_info = self.dr_deploy_opt.query_remote_replication_pair_info_by_pair_id( - self.meta_fs_pair_id) + meta_info = ( + self.dr_deploy_opt.query_remote_replication_pair_info_by_pair_id( + self.meta_fs_pair_id + ) + ) meta_role = meta_info.get("ISPRIMARY") if meta_role == "true": self.dr_deploy_opt.swap_role_replication_pair(self.meta_fs_pair_id) @@ -319,36 +390,46 @@ class DRRecover(SwitchOver): """ check_time_step = 2 LOG.info("Check cluster status.") - cmd_srv = "su -s /bin/bash - %s -c \"cms stat -server | " \ - "grep -v SRV_READY | awk '{print \$1, \$2}'\"" % self.run_user - cmd_voting = "su -s /bin/bash - %s -c \"cms node -connected | " \ - "grep -v VOTING | awk '{print \$1, \$NF}'\"" % self.run_user - + cmd_srv = ( + 'su -s /bin/bash - %s -c "cms stat -server | ' + "grep -v SRV_READY | awk '{print \$1, \$2}'\"" % self.run_user + ) + cmd_voting = ( + 'su -s /bin/bash - %s -c "cms node -connected | ' + "grep -v VOTING | awk '{print \$1, \$NF}'\"" % self.run_user + ) + while check_time: check_time -= check_time_step # 检查所有节点cms正常 - srv_stat= self.query_cluster_status(cmd_srv) + srv_stat = self.query_cluster_status(cmd_srv) ready_flag = False - if len(srv_stat)>1: + if len(srv_stat) > 1: ready_flag = True for node_stat in srv_stat: _, ready_stat = node_stat.split(" ") if ready_stat == "FALSE": ready_flag = False if not ready_flag: - LOG.info("Current cms server status is NOT ready, details (node_id, SRV_READY): %s", ';'.join(srv_stat)) + LOG.info( + "Current cms server status is NOT ready, details (node_id, SRV_READY): %s", + ";".join(srv_stat), + ) time.sleep(check_time_step) continue cms_voting_stat = self.query_cluster_status(cmd_voting) voting_flag = True - if len(cms_voting_stat)>1: + if len(cms_voting_stat) > 1: voting_flag = False for node_stat in cms_voting_stat: _, voting_stat = node_stat.split(" ") if voting_stat == "TRUE": voting_flag = True if voting_flag: - LOG.info("Current cms is voting, details (node_id, VOTING): %s", ';'.join(cms_voting_stat)) + LOG.info( + "Current cms is voting, details (node_id, VOTING): %s", + ";".join(cms_voting_stat), + ) time.sleep(check_time_step) continue break @@ -362,30 +443,42 @@ class DRRecover(SwitchOver): if self.single_write == "1" or self.cantian_recover_type == "full_sync": LOG.info("cantian_recover_type is %s", self.cantian_recover_type) if running_status != ReplicationRunningStatus.Synchronizing: - self.dr_deploy_opt.sync_remote_replication_filesystem_pair(pair_id=pair_id, - vstore_id="0", - is_full_copy=False) + self.dr_deploy_opt.sync_remote_replication_filesystem_pair( + pair_id=pair_id, vstore_id="0", is_full_copy=False + ) time.sleep(10) - pair_info = self.dr_deploy_opt.query_remote_replication_pair_info_by_pair_id( - pair_id) + pair_info = ( + self.dr_deploy_opt.query_remote_replication_pair_info_by_pair_id( + pair_id + ) + ) running_status = pair_info.get("RUNNINGSTATUS") while running_status == ReplicationRunningStatus.Synchronizing: - pair_info = self.dr_deploy_opt.query_remote_replication_pair_info_by_pair_id( - pair_id) + pair_info = ( + self.dr_deploy_opt.query_remote_replication_pair_info_by_pair_id( + pair_id + ) + ) running_status = pair_info.get("RUNNINGSTATUS") replication_progress = pair_info.get("REPLICATIONPROGRESS") - LOG.info(f"Page fs rep pair is synchronizing, current progress: {replication_progress}%, please wait...") + LOG.info( + f"Page fs rep pair is synchronizing, current progress: {replication_progress}%, please wait..." + ) time.sleep(10) else: LOG.info("Single write is disabled, no need to execute replication steps.") self.repl_success_flag = True self.dr_deploy_opt.split_remote_replication_filesystem_pair(pair_id) - self.dr_deploy_opt.remote_replication_filesystem_pair_cancel_secondary_write_lock(pair_id) + self.dr_deploy_opt.remote_replication_filesystem_pair_cancel_secondary_write_lock( + pair_id + ) def standby_cms_purge_backup(self): LOG.info("Standby purge backup by cms command.") - cmd = "source ~/.bashrc && su -s /bin/bash - %s -c " \ - "\"ctbackup --purge-logs\"" % self.run_user + cmd = ( + "source ~/.bashrc && su -s /bin/bash - %s -c " + '"ctbackup --purge-logs"' % self.run_user + ) return_code, output, stderr = exec_popen(cmd, timeout=600) if return_code: err_msg = "Execute command[ctbackup --purge-logs] failed." @@ -406,12 +499,15 @@ class DRRecover(SwitchOver): def rep_pair_recover(self, pair_id: str) -> None: pair_info = self.dr_deploy_opt.query_remote_replication_pair_info_by_pair_id( - pair_id) + pair_id + ) page_role = pair_info.get("ISPRIMARY") running_status = pair_info.get("RUNNINGSTATUS") if page_role == "true": self.dr_deploy_opt.swap_role_replication_pair(pair_id) - self.dr_deploy_opt.remote_replication_filesystem_pair_set_secondary_write_lock(pair_id) + self.dr_deploy_opt.remote_replication_filesystem_pair_set_secondary_write_lock( + pair_id + ) self.execute_replication_steps(running_status, pair_id) else: LOG.info("Page fs rep pair is already standby site.") @@ -419,12 +515,16 @@ class DRRecover(SwitchOver): try: self.check_cluster_status(log_type="info") except Exception as _er: - self.dr_deploy_opt.remote_replication_filesystem_pair_set_secondary_write_lock(pair_id) + self.dr_deploy_opt.remote_replication_filesystem_pair_set_secondary_write_lock( + pair_id + ) self.execute_replication_steps(running_status, pair_id) else: return - elif running_status == ReplicationRunningStatus.Normal or \ - running_status == ReplicationRunningStatus.Synchronizing: + elif ( + running_status == ReplicationRunningStatus.Normal + or running_status == ReplicationRunningStatus.Synchronizing + ): self.execute_replication_steps(running_status, pair_id) else: err_msg = "Remote replication filesystem pair is not in normal status." @@ -432,18 +532,26 @@ class DRRecover(SwitchOver): raise Exception(err_msg) def hyper_metro_status_check(self, running_status, config_role): - if running_status != MetroDomainRunningStatus.Normal and running_status != MetroDomainRunningStatus.Split: - err_msg = "DR recover operation is not allowed in %s status." % \ - get_status(running_status, MetroDomainRunningStatus) + if ( + running_status != MetroDomainRunningStatus.Normal + and running_status != MetroDomainRunningStatus.Split + ): + err_msg = "DR recover operation is not allowed in %s status." % get_status( + running_status, MetroDomainRunningStatus + ) LOG.error(err_msg) raise Exception(err_msg) - if running_status == MetroDomainRunningStatus.Normal and config_role == ConfigRole.Primary: - err_msg = "DR recover operation is not allowed in %s status." % \ - get_status(running_status, MetroDomainRunningStatus) + if ( + running_status == MetroDomainRunningStatus.Normal + and config_role == ConfigRole.Primary + ): + err_msg = "DR recover operation is not allowed in %s status." % get_status( + running_status, MetroDomainRunningStatus + ) LOG.error(err_msg) raise Exception(err_msg) - def execute(self, cantian_recover_type = None): + def execute(self, cantian_recover_type=None): """ step: 1、检查当前双活域状态： @@ -473,14 +581,17 @@ class DRRecover(SwitchOver): LOG.info("DR recover start.") self.check_cluster_status_for_recover() self.init_storage_opt() - domain_info = self.dr_deploy_opt.query_hyper_metro_domain_info(self.hyper_domain_id) + domain_info = self.dr_deploy_opt.query_hyper_metro_domain_info( + self.hyper_domain_id + ) running_status = domain_info.get("RUNNINGSTATUS") config_role = domain_info.get("CONFIGROLE") self.hyper_metro_status_check(running_status, config_role) if running_status == MetroDomainRunningStatus.Split: if config_role == ConfigRole.Primary: self.dr_deploy_opt.change_fs_hyper_metro_domain_second_access( - self.hyper_domain_id, DomainAccess.ReadAndWrite) + self.hyper_domain_id, DomainAccess.ReadAndWrite + ) self.dr_deploy_opt.swap_role_fs_hyper_metro_domain(self.hyper_domain_id) try: self.standby_cms_res_stop() @@ -496,7 +607,8 @@ class DRRecover(SwitchOver): raise Exception(err_msg) self.single_write = self.do_dbstor_baseline() self.dr_deploy_opt.change_fs_hyper_metro_domain_second_access( - self.hyper_domain_id, DomainAccess.ReadOnly) + self.hyper_domain_id, DomainAccess.ReadOnly + ) try: self.dr_deploy_opt.join_fs_hyper_metro_domain(self.hyper_domain_id) except Exception as _er: @@ -535,7 +647,9 @@ class FailOver(SwitchOver): """ LOG.info("Cancel secondary resource protection start.") self.init_storage_opt() - domain_info = self.dr_deploy_opt.query_hyper_metro_domain_info(self.hyper_domain_id) + domain_info = self.dr_deploy_opt.query_hyper_metro_domain_info( + self.hyper_domain_id + ) config_role = domain_info.get("CONFIGROLE") if config_role == ConfigRole.Primary: err_msg = "Fail over operation is not allowed in primary node." @@ -545,17 +659,20 @@ class FailOver(SwitchOver): if running_status == MetroDomainRunningStatus.Normal: self.dr_deploy_opt.split_filesystem_hyper_metro_domain(self.hyper_domain_id) self.dr_deploy_opt.change_fs_hyper_metro_domain_second_access( - self.hyper_domain_id, DomainAccess.ReadAndWrite) + self.hyper_domain_id, DomainAccess.ReadAndWrite + ) try: self.standby_cms_res_start() except Exception as _er: - err_msg ="Standby cms res start failed, error: {}".format(_er) + err_msg = "Standby cms res start failed, error: {}".format(_er) LOG.error(err_msg) try: - self.check_cluster_status(target_node=self.node_id, log_type="info",check_time=300) + self.check_cluster_status( + target_node=self.node_id, log_type="info", check_time=300 + ) except Exception as _er: err_msg = "Check cluster status failed, error: {}".format(_er) LOG.error(err_msg) raise Exception(err_msg) self.query_database_role() - LOG.info("Cancel secondary resource protection success.") \ No newline at end of file + LOG.info("Cancel secondary resource protection success.") diff --git a/pkg/deploy/action/storage_operate/dr_deploy_operate/dr_undeploy.py b/pkg/deploy/action/storage_operate/dr_deploy_operate/dr_undeploy.py index 1705c1ad..e284924f 100644 --- a/pkg/deploy/action/storage_operate/dr_deploy_operate/dr_undeploy.py +++ b/pkg/deploy/action/storage_operate/dr_deploy_operate/dr_undeploy.py @@ -7,7 +7,9 @@ import time import shutil from storage_operate.dr_deploy_operate.dr_deploy_common import DRDeployCommon -from storage_operate.dr_deploy_operate.dr_deploy_pre_check import CANTIAN_STOP_SUCCESS_FLAG +from storage_operate.dr_deploy_operate.dr_deploy_pre_check import ( + CANTIAN_STOP_SUCCESS_FLAG, +) from storage_operate.dr_deploy_operate.dr_deploy import DRDeploy from logic.storage_operate import StorageInf from logic.common_func import read_json_config @@ -18,7 +20,9 @@ from get_config_info import get_env_info CURRENT_PATH = os.path.dirname(os.path.abspath(__file__)) CANTIAN_DEPLOY_CONFIG = os.path.join(CURRENT_PATH, "../../../config/deploy_param.json") DR_DEPLOY_CONFIG = os.path.join(CURRENT_PATH, "../../../config/dr_deploy_param.json") -DR_DEPLOY_REMOTE_CONFIG = os.path.join(CURRENT_PATH, "../../../config/remote/dr_deploy_param.json") +DR_DEPLOY_REMOTE_CONFIG = os.path.join( + CURRENT_PATH, "../../../config/remote/dr_deploy_param.json" +) UNINSTALL_TIMEOUT = 900 @@ -31,7 +35,7 @@ class UNDeploy(object): self.dr_deploy_info = read_json_config(DR_DEPLOY_CONFIG) self.deploy_params = read_json_config(CANTIAN_DEPLOY_CONFIG) self.run_user = get_env_info("cantian_user") - + def init_storage_opt(self): dm_ip = self.dr_deploy_info.get("dm_ip") dm_user = self.dr_deploy_info.get("dm_user") @@ -54,31 +58,40 @@ class UNDeploy(object): try: self.dr_deploy_opt.delete_remote_replication_filesystem_pair(page_id) except Exception as err: - self.dr_deploy_opt.delete_remote_replication_filesystem_pair(page_id, is_local_del=True) + self.dr_deploy_opt.delete_remote_replication_filesystem_pair( + page_id, is_local_del=True + ) LOG.info("Delete Replication pair[id:%s] success", page_id) def delete_filesystem(self, vstore_id, fs_name): if self.site == "active": return else: - fs_info = self.dr_deploy_opt.storage_opt.query_filesystem_info(fs_name, - vstore_id) + fs_info = self.dr_deploy_opt.storage_opt.query_filesystem_info( + fs_name, vstore_id + ) if not fs_info: LOG.info("Filesystem[%s] is not exist.", fs_name) return if fs_info.get("scheduleName") != "--": # schedule cdp 存在，先删除 LOG.info("Delete schedule[%s]", fs_info.get("scheduleName")) - self.dr_deploy_opt.storage_opt.delete_fs_cdp_schedule(fs_info.get("ID"), - fs_info.get("TIMINGSNAPSHOTSCHEDULEID"), - fs_info.get("scheduleName"), - fs_info.get("vstoreId")) + self.dr_deploy_opt.storage_opt.delete_fs_cdp_schedule( + fs_info.get("ID"), + fs_info.get("TIMINGSNAPSHOTSCHEDULEID"), + fs_info.get("scheduleName"), + fs_info.get("vstoreId"), + ) LOG.info("Delete schedule[%s] success!", fs_info.get("scheduleName")) fs_id = fs_info.get("ID") - nfs_share_info = self.storage_opt.query_nfs_info(fs_id=fs_id, vstore_id=vstore_id) + nfs_share_info = self.storage_opt.query_nfs_info( + fs_id=fs_id, vstore_id=vstore_id + ) if nfs_share_info: nfs_share_id = nfs_share_info[0].get("ID") - self.storage_opt.delete_nfs_share(nfs_share_id=nfs_share_id, vstore_id=vstore_id) + self.storage_opt.delete_nfs_share( + nfs_share_id=nfs_share_id, vstore_id=vstore_id + ) LOG.info("Delete file system %s nfs share success!", fs_name) self.storage_opt.delete_file_system(fs_id) LOG.info("Delete file system %s success!", fs_name) @@ -95,23 +108,32 @@ class UNDeploy(object): storage_dbstor_fs = self.dr_deploy_info.get("storage_dbstor_fs") if not storage_dbstor_fs: return - dbstor_fs_info = self.dr_deploy_opt.storage_opt.query_filesystem_info(storage_dbstor_fs, - dbstor_fs_vstore_id) + dbstor_fs_info = self.dr_deploy_opt.storage_opt.query_filesystem_info( + storage_dbstor_fs, dbstor_fs_vstore_id + ) if not dbstor_fs_info: LOG.info("Filesystem[%s] is not exist.", storage_dbstor_fs) return # 双活文件系统id dbstor_fs_id = dbstor_fs_info.get("ID") # 通过双活文件系统id查询双活文件系统pair id - hyper_filesystem_pair_info = self.dr_deploy_opt.query_hyper_metro_filesystem_pair_info(dbstor_fs_id) + hyper_filesystem_pair_info = ( + self.dr_deploy_opt.query_hyper_metro_filesystem_pair_info(dbstor_fs_id) + ) if hyper_filesystem_pair_info: hyper_filesystem_pair_id = hyper_filesystem_pair_info[0].get("ID") try: - self.dr_deploy_opt.delete_hyper_metro_filesystem_pair(hyper_filesystem_pair_id, dbstor_fs_vstore_id) + self.dr_deploy_opt.delete_hyper_metro_filesystem_pair( + hyper_filesystem_pair_id, dbstor_fs_vstore_id + ) except Exception as err: - self.dr_deploy_opt.delete_hyper_metro_filesystem_pair(hyper_filesystem_pair_id, dbstor_fs_vstore_id, - is_local_del=True) - LOG.info("Delete Hyper Metro filesystem pair id[%s] success", hyper_filesystem_pair_id) + self.dr_deploy_opt.delete_hyper_metro_filesystem_pair( + hyper_filesystem_pair_id, dbstor_fs_vstore_id, is_local_del=True + ) + LOG.info( + "Delete Hyper Metro filesystem pair id[%s] success", + hyper_filesystem_pair_id, + ) LOG.info("Delete Hyper Metro filesystem pair id success") def delete_hyper_metro_filesystem_vstore_id(self): @@ -122,25 +144,39 @@ class UNDeploy(object): if not hyper_metro_vstore_pair_id: return False try: - self.dr_deploy_opt.query_hyper_metro_vstore_pair_info(hyper_metro_vstore_pair_id) + self.dr_deploy_opt.query_hyper_metro_vstore_pair_info( + hyper_metro_vstore_pair_id + ) except Exception as err: if "1073781761" in str(err): - LOG.info("Hyper Metro pair id[%s] is not exist.", hyper_metro_vstore_pair_id) + LOG.info( + "Hyper Metro pair id[%s] is not exist.", hyper_metro_vstore_pair_id + ) return False else: raise err dbstor_fs_vstore_id = self.dr_deploy_info.get("dbstor_fs_vstore_id") - file_system_count = self.dr_deploy_opt.query_hyper_metro_filesystem_count_info(dbstor_fs_vstore_id) + file_system_count = self.dr_deploy_opt.query_hyper_metro_filesystem_count_info( + dbstor_fs_vstore_id + ) if file_system_count and file_system_count.get("COUNT") != "0": - msg = "Delete Hyper Metro pair id[id:%s], " \ - "but there are also other pair file systems" % dbstor_fs_vstore_id + msg = ( + "Delete Hyper Metro pair id[id:%s], " + "but there are also other pair file systems" % dbstor_fs_vstore_id + ) LOG.info(msg) return False try: - self.dr_deploy_opt.delete_hyper_metro_vstore_pair(hyper_metro_vstore_pair_id) + self.dr_deploy_opt.delete_hyper_metro_vstore_pair( + hyper_metro_vstore_pair_id + ) except Exception as err: - self.dr_deploy_opt.delete_hyper_metro_vstore_pair(hyper_metro_vstore_pair_id, is_local_del=True) - LOG.info("Delete Hyper Metro pair id[id:%s] success", hyper_metro_vstore_pair_id) + self.dr_deploy_opt.delete_hyper_metro_vstore_pair( + hyper_metro_vstore_pair_id, is_local_del=True + ) + LOG.info( + "Delete Hyper Metro pair id[id:%s] success", hyper_metro_vstore_pair_id + ) return True def delete_hyper_metro_domain(self): @@ -162,9 +198,13 @@ class UNDeploy(object): try: self.dr_deploy_opt.delete_filesystem_hyper_metro_domain(hyper_domain_id) except Exception as err: - self.dr_deploy_opt.delete_filesystem_hyper_metro_domain(hyper_domain_id, is_local_del=True) + self.dr_deploy_opt.delete_filesystem_hyper_metro_domain( + hyper_domain_id, is_local_del=True + ) metadata_dr_deploy_path = f"/mnt/dbdata/remote/metadata_{self.dr_deploy_info.get('storage_metadata_fs')}" - metadata_dr_deploy_config = os.path.join(metadata_dr_deploy_path, "dr_deploy_param.json") + metadata_dr_deploy_config = os.path.join( + metadata_dr_deploy_path, "dr_deploy_param.json" + ) dr_deploy_config = f"/opt/cantian/config/dr_deploy_param.json" if os.path.exists(dr_deploy_config): os.remove(dr_deploy_config) @@ -174,7 +214,9 @@ class UNDeploy(object): def delete_replication(self): action_parse = argparse.ArgumentParser() - action_parse.add_argument("--site", dest="site", choices=["standby", "active"], required=True) + action_parse.add_argument( + "--site", dest="site", choices=["standby", "active"], required=True + ) args = action_parse.parse_args() self.site = args.site page_fs_pair_id = self.dr_deploy_info.get("page_fs_pair_id") @@ -189,7 +231,9 @@ class UNDeploy(object): def delete_hyper(self): action_parse = argparse.ArgumentParser() - action_parse.add_argument("--site", dest="site", choices=["standby", "active"], required=True) + action_parse.add_argument( + "--site", dest="site", choices=["standby", "active"], required=True + ) args = action_parse.parse_args() self.site = args.site # 删除双活文件系统pair id @@ -209,7 +253,10 @@ class UNDeploy(object): cmd = "sh %s/stop.sh;last_cmd=$?" % ctl_file_path _, output, stderr = exec_popen(cmd, timeout=3600) if "Stop Cantian Engine success." not in output: - err_msg = "Failed to execute stop, stderr:%s, output:%s" % (stderr, output) + err_msg = "Failed to execute stop, stderr:%s, output:%s" % ( + stderr, + output, + ) raise Exception(err_msg) if not os.path.exists(install_record_file): return @@ -222,15 +269,20 @@ class UNDeploy(object): cmd = "sh %s/uninstall.sh override" % ctl_file_path _, output, stderr = exec_popen(cmd, timeout=180) if "uninstall finished" not in output: - err_msg = "Failed to execute uninstall, stderr:%s, output:%s" % (stderr, output) + err_msg = "Failed to execute uninstall, stderr:%s, output:%s" % ( + stderr, + output, + ) raise Exception(err_msg) if os.path.exists("/opt/cantian"): shutil.rmtree("/opt/cantian") def wait_remote_node_exec(self, node_id, timeout): wait_time = 0 - cmd = "su -s /bin/bash - %s -c \"cms stat | " \ - "grep -v STAT | awk '{print \$1, \$3, \$6}'\"" % self.run_user + cmd = ( + 'su -s /bin/bash - %s -c "cms stat | ' + "grep -v STAT | awk '{print \$1, \$3, \$6}'\"" % self.run_user + ) while timeout: return_code, output, stderr = exec_popen(cmd, timeout=100) cms_stat = output.split("\n") @@ -268,26 +320,38 @@ class UNDeploy(object): if self.deploy_params.get("deploy_mode") == "dbstor": fs_name = self.deploy_params.get("storage_share_fs") - clean_cmd = f"su -s /bin/bash - {self.run_user} -c \"dbstor --delete-file --fs-name={fs_name} " \ - "--file-name=/dr_deploy_param.json\"" + clean_cmd = ( + f'su -s /bin/bash - {self.run_user} -c "dbstor --delete-file --fs-name={fs_name} ' + '--file-name=/dr_deploy_param.json"' + ) else: fs_name = self.deploy_params.get("storage_metadata_fs") - clean_cmd = f"rm -rf /mnt/dbdata/remote/metadata_{fs_name}/dr_deploy_param.json" + clean_cmd = ( + f"rm -rf /mnt/dbdata/remote/metadata_{fs_name}/dr_deploy_param.json" + ) try: ret_code, output, stderr = exec_popen(clean_cmd) if ret_code: LOG.info(f"Failed to execute command '{clean_cmd}', error: {stderr}") except Exception as e: - LOG.info(f"Exception occurred while executing command '{clean_cmd}': {str(e)}") + LOG.info( + f"Exception occurred while executing command '{clean_cmd}': {str(e)}" + ) def standby_uninstall(self, node_id, uninstall_cantian_flag): - if self.site == "standby" and os.path.exists(CANTIAN_DEPLOY_CONFIG) and uninstall_cantian_flag: + if ( + self.site == "standby" + and os.path.exists(CANTIAN_DEPLOY_CONFIG) + and uninstall_cantian_flag + ): self.do_stop() LOG.info("Stop Cantian engine success.") if node_id == "0": LOG.info("Start to delete dr deploy!") rep_fs_name = self.dr_deploy_info.get("storage_dbstor_page_fs") - mysql_metadata_in_cantian = self.dr_deploy_info.get("mysql_metadata_in_cantian") + mysql_metadata_in_cantian = self.dr_deploy_info.get( + "mysql_metadata_in_cantian" + ) metadata_fs = self.dr_deploy_info.get("storage_metadata_fs") self.delete_replication() self.delete_filesystem(vstore_id="0", fs_name=rep_fs_name) @@ -301,7 +365,11 @@ class UNDeploy(object): except Exception as err: LOG.info("Standby site delete hyper system failed: %s", str(err)) self.clean_dr_config_file() - if self.site == "standby" and os.path.exists(CANTIAN_DEPLOY_CONFIG) and uninstall_cantian_flag: + if ( + self.site == "standby" + and os.path.exists(CANTIAN_DEPLOY_CONFIG) + and uninstall_cantian_flag + ): if node_id == "0": self.wait_remote_node_exec("1", UNINSTALL_TIMEOUT) self.do_uninstall() @@ -320,7 +388,9 @@ class UNDeploy(object): self.init_storage_opt() node_id = self.deploy_params.get("node_id") action_parse = argparse.ArgumentParser() - action_parse.add_argument("--site", dest="site", choices=["standby", "active"], required=True) + action_parse.add_argument( + "--site", dest="site", choices=["standby", "active"], required=True + ) args = action_parse.parse_args() self.site = args.site # 告警提示，是否确认卸载；是否卸载Cantian diff --git a/pkg/deploy/action/storage_operate/dr_deploy_operate/install_mysql.py b/pkg/deploy/action/storage_operate/dr_deploy_operate/install_mysql.py index 29f4e7ed..da1300ab 100644 --- a/pkg/deploy/action/storage_operate/dr_deploy_operate/install_mysql.py +++ b/pkg/deploy/action/storage_operate/dr_deploy_operate/install_mysql.py @@ -12,7 +12,8 @@ from om_log import DR_DEPLOY_LOG as LOG CURRENT_PATH = os.path.dirname(os.path.abspath(__file__)) ROOT_DIR = os.path.abspath(os.path.join(CURRENT_PATH, "../../../../")) MYSQL_INSTALL_PATH = "/opt/cantian/mysql/install" -WORKDIR = '/opt/cantian/image/cantian_connector/cantian-connector-mysql/mysql_bin/' +WORKDIR = "/opt/cantian/image/cantian_connector/cantian-connector-mysql/mysql_bin/" + def get_file_name(file_patten): files = os.listdir(ROOT_DIR) @@ -43,7 +44,7 @@ def execute_meta(): except Exception as e: LOG.error("Obtain u_id or g_id failed, details: %s", str(e)) raise e - file_name = get_file_name(r'Mysql_server_single.*.tgz') + file_name = get_file_name(r"Mysql_server_single.*.tgz") source_path = os.path.join(ROOT_DIR, file_name) try: with tarfile.open(source_path) as tar_ref: @@ -52,27 +53,39 @@ def execute_meta(): LOG.error("Extractall failed, details: %s", str(e)) raise e LOG.info("Begin to copy mysql files, deploy mode meta.") - cmd = "rm -rf %s/lib/plugin/ha_ctc.so" % os.path.join(WORKDIR, 'Mysql_server/mysql') + cmd = "rm -rf %s/lib/plugin/ha_ctc.so" % os.path.join(WORKDIR, "Mysql_server/mysql") return_code, stdout, stderr = exec_popen(cmd, timeout=30) - LOG.info("Execute cmd[%s], return_code[%s], stdout[%s], stderr[%s]" % (cmd, return_code, stdout, stderr)) - cmd = "cp -arf %s %s" % (os.path.join(WORKDIR, 'Mysql_server/mysql'), WORKDIR) + LOG.info( + "Execute cmd[%s], return_code[%s], stdout[%s], stderr[%s]" + % (cmd, return_code, stdout, stderr) + ) + cmd = "cp -arf %s %s" % (os.path.join(WORKDIR, "Mysql_server/mysql"), WORKDIR) return_code, stdout, stderr = exec_popen(cmd, timeout=180) - LOG.info("Execute cmd[%s], return_code[%s], stdout[%s], stderr[%s]" % (cmd, return_code, stdout, stderr)) - cmd = "cp -arf %s %s" % (os.path.join(WORKDIR, 'mysql'), MYSQL_INSTALL_PATH) + LOG.info( + "Execute cmd[%s], return_code[%s], stdout[%s], stderr[%s]" + % (cmd, return_code, stdout, stderr) + ) + cmd = "cp -arf %s %s" % (os.path.join(WORKDIR, "mysql"), MYSQL_INSTALL_PATH) return_code, stdout, stderr = exec_popen(cmd, timeout=180) - LOG.info("Execute cmd[%s], return_code[%s], stdout[%s], stderr[%s]" % (cmd, return_code, stdout, stderr)) + LOG.info( + "Execute cmd[%s], return_code[%s], stdout[%s], stderr[%s]" + % (cmd, return_code, stdout, stderr) + ) cmd = f"cp -pf {MYSQL_INSTALL_PATH}/mysql/bin/mysql /usr/bin/" return_code, stdout, stderr = exec_popen(cmd, timeout=30) - LOG.info("Execute cmd[%s], return_code[%s], stdout[%s], stderr[%s]" % (cmd, return_code, stdout, stderr)) - os.chown('/opt/cantian/mysql/install/mysql', u_id, g_id) - for root, dirs, files in os.walk('/opt/cantian/mysql/install/mysql'): + LOG.info( + "Execute cmd[%s], return_code[%s], stdout[%s], stderr[%s]" + % (cmd, return_code, stdout, stderr) + ) + os.chown("/opt/cantian/mysql/install/mysql", u_id, g_id) + for root, dirs, files in os.walk("/opt/cantian/mysql/install/mysql"): for d in dirs: os.chown(os.path.join(root, d), u_id, g_id) for f in files: os.chown(os.path.join(root, f), u_id, g_id) - if os.path.exists('/usr/local/mysql'): - shutil.rmtree('/usr/local/mysql') - shutil.copytree('/opt/cantian/mysql/install/mysql', '/usr/local/mysql') + if os.path.exists("/usr/local/mysql"): + shutil.rmtree("/usr/local/mysql") + shutil.copytree("/opt/cantian/mysql/install/mysql", "/usr/local/mysql") LOG.info("Success to install mysql, deploy mode meta.") @@ -92,19 +105,19 @@ def execute_nometa(): except Exception as e: LOG.error("Obtain u_id or g_id failed, details: %s", str(e)) raise e - file_name = get_file_name(r'mysql_release_.*.tar.gz') + file_name = get_file_name(r"mysql_release_.*.tar.gz") source_path = os.path.join(ROOT_DIR, file_name) with tarfile.open(source_path) as tar_ref: tar_ref.extractall(WORKDIR) - tar_ref.extractall('/opt/cantian/mysql/install/') - os.chown('/opt/cantian/mysql/install/mysql', u_id, g_id) - for root, dirs, files in os.walk('/opt/cantian/mysql/install/mysql'): + tar_ref.extractall("/opt/cantian/mysql/install/") + os.chown("/opt/cantian/mysql/install/mysql", u_id, g_id) + for root, dirs, files in os.walk("/opt/cantian/mysql/install/mysql"): for d in dirs: os.chown(os.path.join(root, d), u_id, g_id) for f in files: os.chown(os.path.join(root, f), u_id, g_id) - shutil.copyfile('/opt/cantian/mysql/install/mysql/bin/mysql', '/usr/bin/mysql') - if os.path.exists('/usr/local/mysql'): - shutil.rmtree('/usr/local/mysql') - shutil.copytree('/opt/cantian/mysql/install/mysql', '/usr/local/mysql') + shutil.copyfile("/opt/cantian/mysql/install/mysql/bin/mysql", "/usr/bin/mysql") + if os.path.exists("/usr/local/mysql"): + shutil.rmtree("/usr/local/mysql") + shutil.copytree("/opt/cantian/mysql/install/mysql", "/usr/local/mysql") LOG.info("Success to install mysql, deploy mode nometa.") diff --git a/pkg/deploy/action/storage_operate/dr_deploy_operate/update_dr_params.py b/pkg/deploy/action/storage_operate/dr_deploy_operate/update_dr_params.py index e4a9473f..7b967f64 100644 --- a/pkg/deploy/action/storage_operate/dr_deploy_operate/update_dr_params.py +++ b/pkg/deploy/action/storage_operate/dr_deploy_operate/update_dr_params.py @@ -25,7 +25,9 @@ class UpdateDRParams(object): self.storage_metadata_fs = self.deploy_params.get("storage_metadata_fs") self.storage_share_fs = self.deploy_params.get("storage_share_fs") self.cluster_name = self.deploy_params.get("cluster_name") - self.mysql_metadata_in_cantian = self.deploy_params.get("mysql_metadata_in_cantian") + self.mysql_metadata_in_cantian = self.deploy_params.get( + "mysql_metadata_in_cantian" + ) self.dbstor_fs_vstore_id = self.deploy_params.get("dbstor_fs_vstore_id") self.deploy_mode = self.deploy_params.get("deploy_mode") @@ -35,8 +37,10 @@ class UpdateDRParams(object): 容灾告警需要重启cantian_exporter :return: """ - cmd = "ps -ef | grep \"python3 /opt/cantian/ct_om/service/cantian_exporter/exporter/execute.py\"" \ - " | grep -v grep | awk '{print $2}' | xargs kill -9" + cmd = ( + 'ps -ef | grep "python3 /opt/cantian/ct_om/service/cantian_exporter/exporter/execute.py"' + " | grep -v grep | awk '{print $2}' | xargs kill -9" + ) exec_popen(cmd) def copy_dr_deploy_param_file(self): @@ -118,13 +122,18 @@ class UpdateDRParams(object): if not current_real_path.startswith(target_path): try: - shutil.copy(DEPLOY_PARAM_FILE, os.path.join(CURRENT_PATH, "../../../config")) + shutil.copy( + DEPLOY_PARAM_FILE, os.path.join(CURRENT_PATH, "../../../config") + ) except Exception as _err: LOG.info(f"copy DEPLOY_PARAM_FILE failed") encrypted_pwd = KmcResolve.kmc_resolve_password("encrypted", dm_passwd) dr_deploy_params["dm_pwd"] = encrypted_pwd write_json_config(DR_DEPLOY_CONFIG, dr_deploy_params) - os.chmod(os.path.join(CURRENT_PATH, "../../../config/dr_deploy_param.json"), mode=0o644) + os.chmod( + os.path.join(CURRENT_PATH, "../../../config/dr_deploy_param.json"), + mode=0o644, + ) if not current_real_path.startswith(target_path): try: shutil.copy(DR_DEPLOY_CONFIG, "/opt/cantian/config") @@ -149,7 +158,9 @@ class UpdateDRParams(object): dr_deploy_opt = DRDeployCommon(storage_operate) LOG.info(f"begin to check hyper metro domain[{hyper_domain_id}]") dr_deploy_opt.query_hyper_metro_domain_info(hyper_domain_id) - LOG.info(f"begin to check hyper metro vstore pair[{hyper_metro_vstore_pair_id}]") + LOG.info( + f"begin to check hyper metro vstore pair[{hyper_metro_vstore_pair_id}]" + ) dr_deploy_opt.query_hyper_metro_vstore_pair_info(hyper_metro_vstore_pair_id) LOG.info(f"begin to check hyper metro filesystem pair[{ulog_fs_pair_id}]") dr_deploy_opt.query_hyper_metro_filesystem_pair_info_by_pair_id(ulog_fs_pair_id) @@ -158,4 +169,3 @@ class UpdateDRParams(object): if not self.mysql_metadata_in_cantian: LOG.info(f"begin to check remote replication pair[{meta_fs_pair_id}]") dr_deploy_opt.query_remote_replication_pair_info_by_pair_id(meta_fs_pair_id) - diff --git a/pkg/deploy/action/storage_operate/dr_operate_interface.py b/pkg/deploy/action/storage_operate/dr_operate_interface.py index 80cfb356..1582e352 100644 --- a/pkg/deploy/action/storage_operate/dr_operate_interface.py +++ b/pkg/deploy/action/storage_operate/dr_operate_interface.py @@ -3,23 +3,32 @@ import traceback import sys from om_log import LOGGER as LOG -from storage_operate.dr_deploy_operate.dr_deploy_pre_check import DRDeployPreCheck, ParamCheck +from storage_operate.dr_deploy_operate.dr_deploy_pre_check import ( + DRDeployPreCheck, + ParamCheck, +) from storage_operate.dr_deploy_operate.dr_deploy import DRDeploy from storage_operate.dr_deploy_operate.dr_undeploy import UNDeploy from storage_operate.dr_deploy_operate.dr_deploy_progress_query import ProgressQuery -from storage_operate.dr_deploy_operate.dr_deploy_switchover import SwitchOver, DRRecover, FailOver +from storage_operate.dr_deploy_operate.dr_deploy_switchover import ( + SwitchOver, + DRRecover, + FailOver, +) from storage_operate.dr_deploy_operate.update_dr_params import UpdateDRParams -HELP_MSG = ("example:\n" - " sh appctl.sh dr_operate pre_check active/standby --conf=config_file_path\n" - " sh appctl.sh dr_operate deploy standby/active -" - "-mysql_cmd='/usr/local/mysql/bin/mysql' --mysql_user=myuser\n" - " sh appctl.sh dr_operate progress_query --action=deploy --display=table/json\n" - " sh appctl.sh dr_operate undeploy active/standby\n" - " sh appctl.sh dr_operate switch_over\n" - " sh appctl.sh dr_operate recover [full_sync]\n" - " sh appctl.sh dr_operate fail_over\n" - " sh appctl.sh dr_operate update_conf\n") +HELP_MSG = ( + "example:\n" + " sh appctl.sh dr_operate pre_check active/standby --conf=config_file_path\n" + " sh appctl.sh dr_operate deploy standby/active -" + "-mysql_cmd='/usr/local/mysql/bin/mysql' --mysql_user=myuser\n" + " sh appctl.sh dr_operate progress_query --action=deploy --display=table/json\n" + " sh appctl.sh dr_operate undeploy active/standby\n" + " sh appctl.sh dr_operate switch_over\n" + " sh appctl.sh dr_operate recover [full_sync]\n" + " sh appctl.sh dr_operate fail_over\n" + " sh appctl.sh dr_operate update_conf\n" +) class DRDeployOperate(object): @@ -87,8 +96,9 @@ class DRDeployOperate(object): def main(): - err_msg = "Failed to parse the DR setup command, deploy operate commands " \ - + HELP_MSG + err_msg = ( + "Failed to parse the DR setup command, deploy operate commands " + HELP_MSG + ) dr_deploy_operate = DRDeployOperate() if len(sys.argv) <= 1: raise Exception(err_msg) diff --git a/pkg/deploy/action/storage_operate/migrate_file_system.py b/pkg/deploy/action/storage_operate/migrate_file_system.py index 658e0717..e905be3e 100644 --- a/pkg/deploy/action/storage_operate/migrate_file_system.py +++ b/pkg/deploy/action/storage_operate/migrate_file_system.py @@ -41,7 +41,9 @@ class MigrateFileSystem(StorageInf): """ kerberos_type = self.new_config_info.get("kerberos_key") params = f" -o sec={kerberos_type},timeo=50,nosuid,nodev " - self.mount_file_system(file_system_name, logic_ip, prefix="share", params=params) + self.mount_file_system( + file_system_name, logic_ip, prefix="share", params=params + ) def create_share_file_system_nfs_share(self, share_fs_id, share_path, vstore_id): """ @@ -54,7 +56,7 @@ class MigrateFileSystem(StorageInf): data = { "SHAREPATH": f"/{share_path}/", "vstoreId": vstore_id, - "FSID": share_fs_id + "FSID": share_fs_id, } return self.create_nfs_share(data) @@ -65,7 +67,7 @@ class MigrateFileSystem(StorageInf): "ROOTSQUASH": 1, "PARENTID": parent_id, "vstoreId": vstore_id, - "NAME": client_name + "NAME": client_name, } self.add_nfs_client(data) @@ -76,6 +78,7 @@ class MigrateFileSystem(StorageInf): :param vstore_id: 租户ID :return: """ + def _exec_cmd(): _cmd = f"change vstore view id={vstore_id}" res = self.ssh_client.execute_cmd(_cmd, expect=":/>", timeout=10) @@ -95,6 +98,7 @@ class MigrateFileSystem(StorageInf): err_msg = "Execute cmd[%s], details:%s" % (_cmd, res) LOGGER.error(err_msg) raise Exception(err_msg) + self.ssh_client.create_client() try: _exec_cmd() @@ -112,17 +116,24 @@ class MigrateFileSystem(StorageInf): clone_file_system_name = self.new_config_info.get("storage_share_fs") clone_file_system_share_logic_ip = self.new_config_info.get("share_logic_ip") clone_share_file_system_vstore_id = self.new_config_info.get("vstore_id") - logic_port_info = self.query_logical_port_info(clone_file_system_share_logic_ip, - vstore_id=clone_share_file_system_vstore_id) + logic_port_info = self.query_logical_port_info( + clone_file_system_share_logic_ip, + vstore_id=clone_share_file_system_vstore_id, + ) if logic_port_info is None: - err_msg = "Logic port info[%s] is not exist" % clone_file_system_share_logic_ip + err_msg = ( + "Logic port info[%s] is not exist" % clone_file_system_share_logic_ip + ) LOGGER.error(err_msg) raise Exception(err_msg) - clone_file_system_info = self.query_filesystem_info(clone_file_system_name, - vstore_id=clone_share_file_system_vstore_id) + clone_file_system_info = self.query_filesystem_info( + clone_file_system_name, vstore_id=clone_share_file_system_vstore_id + ) if clone_file_system_info: - err_msg = "Clone share file system[%s] is exist, details: %s " % (clone_file_system_name, - clone_file_system_info) + err_msg = "Clone share file system[%s] is exist, details: %s " % ( + clone_file_system_name, + clone_file_system_info, + ) LOGGER.error(err_msg) raise Exception(err_msg) LOGGER.info("Success to check migrate share fs info") @@ -148,41 +159,59 @@ class MigrateFileSystem(StorageInf): file_system_info = self.query_filesystem_info(file_system_name, vstore_id=0) file_system_id = file_system_info.get("ID") - clone_file_system_info = self.query_filesystem_info(file_system_name, - vstore_id=clone_file_system_vstore_id) + clone_file_system_info = self.query_filesystem_info( + file_system_name, vstore_id=clone_file_system_vstore_id + ) if not clone_file_system_info: - clone_file_system_info = self.create_clone_file_system(file_system_id, - file_system_name, - clone_file_system_vstore_id) + clone_file_system_info = self.create_clone_file_system( + file_system_id, file_system_name, clone_file_system_vstore_id + ) clone_file_system_id = clone_file_system_info.get("ID") split_status = clone_file_system_info.get("SPLITSTATUS") split_enable = clone_file_system_info.get("SPLITENABLE") if int(split_status) == 1 and split_enable == "false": - self.split_clone_file_system(clone_file_system_id, action=1, vstore_id=clone_file_system_vstore_id) - self.query_split_clone_file_system_process(clone_file_system_name, vstore_id=clone_file_system_vstore_id) - clone_file_system_nfs_info = self.query_nfs_info(clone_file_system_id, - vstore_id=clone_file_system_vstore_id) + self.split_clone_file_system( + clone_file_system_id, + action=1, + vstore_id=clone_file_system_vstore_id, + ) + self.query_split_clone_file_system_process( + clone_file_system_name, vstore_id=clone_file_system_vstore_id + ) + clone_file_system_nfs_info = self.query_nfs_info( + clone_file_system_id, vstore_id=clone_file_system_vstore_id + ) if clone_file_system_nfs_info: clone_share_file_system_nfs_id = clone_file_system_nfs_info[0].get("ID") - self.delete_nfs_share(clone_share_file_system_nfs_id, vstore_id=clone_file_system_vstore_id) + self.delete_nfs_share( + clone_share_file_system_nfs_id, + vstore_id=clone_file_system_vstore_id, + ) clone_file_system_nfs_share_id = self.create_share_file_system_nfs_share( - clone_file_system_id, file_system_name, clone_file_system_vstore_id) + clone_file_system_id, file_system_name, clone_file_system_vstore_id + ) share_nfs_info = self.query_nfs_info(file_system_id, vstore_id=0) if not share_nfs_info: err_msg = "Failed to query fs[%s] nfs share info." % file_system_name LOGGER.error(err_msg) raise Exception(err_msg) share_nfs_id = share_nfs_info[0].get("ID") - share_nfs_client_config = self.query_nfs_share_auth_client(share_nfs_id, vstore_id=0) + share_nfs_client_config = self.query_nfs_share_auth_client( + share_nfs_id, vstore_id=0 + ) share_nfs_client_name = share_nfs_client_config[0].get("NAME") - self.add_share_file_system_nfs_client(clone_file_system_nfs_share_id, - clone_file_system_vstore_id, - share_nfs_client_name) + self.add_share_file_system_nfs_client( + clone_file_system_nfs_share_id, + clone_file_system_vstore_id, + share_nfs_client_name, + ) time.sleep(2) self.umount_share_file_system(file_system_name) clone_file_system_logic_ip = self.new_config_info.get(logic_ip_key) time.sleep(2) - self.mount_share_file_system(clone_file_system_name, clone_file_system_logic_ip) + self.mount_share_file_system( + clone_file_system_name, clone_file_system_logic_ip + ) self.config_mandatory_lock_switch(clone_file_system_vstore_id) def rollback(self): @@ -205,16 +234,22 @@ class MigrateFileSystem(StorageInf): self.umount_share_file_system(clone_file_system_name) time.sleep(2) self.mount_share_file_system(file_system_name, file_system_logic_ip) - clone_file_system_info = self.query_filesystem_info(clone_file_system_name, - vstore_id=clone_share_file_system_vstore_id) + clone_file_system_info = self.query_filesystem_info( + clone_file_system_name, vstore_id=clone_share_file_system_vstore_id + ) if not clone_file_system_info: return clone_file_system_id = clone_file_system_info.get("ID") - clone_file_system_nfs_share_info = self.query_nfs_info(clone_file_system_id, - clone_share_file_system_vstore_id) + clone_file_system_nfs_share_info = self.query_nfs_info( + clone_file_system_id, clone_share_file_system_vstore_id + ) if clone_file_system_nfs_share_info: - clone_file_system_nfs_share_id = clone_file_system_nfs_share_info[0].get("ID") - self.delete_nfs_share(clone_file_system_nfs_share_id, clone_share_file_system_vstore_id) + clone_file_system_nfs_share_id = clone_file_system_nfs_share_info[ + 0 + ].get("ID") + self.delete_nfs_share( + clone_file_system_nfs_share_id, clone_share_file_system_vstore_id + ) self.delete_file_system(clone_file_system_id) @@ -223,11 +258,11 @@ def check_version(old_config): return True config_dir = os.path.dirname(old_config) versions_path = os.path.join(config_dir, "..", "versions.yml") - with open(versions_path, 'r', encoding='utf-8') as file: + with open(versions_path, "r", encoding="utf-8") as file: source_version_info = file.readlines() - version = '' + version = "" for line in source_version_info: - if 'Version:' in line: + if "Version:" in line: version = line.split()[-1] return version.startswith("2.0.0") @@ -244,7 +279,9 @@ def main(): ip_addr = input() user_name = input() passwd = input() - migrate_file_system = MigrateFileSystem(ip_addr, user_name, passwd, new_config, old_config) + migrate_file_system = MigrateFileSystem( + ip_addr, user_name, passwd, new_config, old_config + ) migrate_file_system.login() try: getattr(migrate_file_system, action)() diff --git a/pkg/deploy/action/storage_operate/split_dbstore_fs.py b/pkg/deploy/action/storage_operate/split_dbstore_fs.py index 1a32c443..130839a2 100644 --- a/pkg/deploy/action/storage_operate/split_dbstore_fs.py +++ b/pkg/deploy/action/storage_operate/split_dbstore_fs.py @@ -89,7 +89,9 @@ class StorageFileSystemSplit(StorageInf): 2、将ulog_root_dir内部目录上移一层，然后删除ulog_root_di :return: """ - log_namespace_path = f"/mnt/dbdata/remote/{self.storage_dbstor_fs}/{self.namespace}" + log_namespace_path = ( + f"/mnt/dbdata/remote/{self.storage_dbstor_fs}/{self.namespace}" + ) page_pool_root_dir = os.path.join(log_namespace_path, "page_pool_root_dir") ulog_root_dir = os.path.join(log_namespace_path, "ulog_root_dir") if os.path.exists(page_pool_root_dir): @@ -107,7 +109,9 @@ class StorageFileSystemSplit(StorageInf): 3、将page_pool_root_dir内的目录上移后，删除page_pool_root_dir :return: """ - page_namespace_path = f"/mnt/dbdata/remote/{self.storage_dbstor_page_fs}/{self.namespace}" + page_namespace_path = ( + f"/mnt/dbdata/remote/{self.storage_dbstor_page_fs}/{self.namespace}" + ) namespace_file = os.path.join(page_namespace_path, self.namespace) page_pool_root_dir = os.path.join(page_namespace_path, "page_pool_root_dir") ulog_root_dir = os.path.join(page_namespace_path, "ulog_root_dir") @@ -126,7 +130,7 @@ class StorageFileSystemSplit(StorageInf): "ROOTSQUASH": 1, "PARENTID": share_id, "vstoreId": self.vstore_id, - "NAME": "*" + "NAME": "*", } self.add_nfs_client(data) @@ -144,7 +148,7 @@ class StorageFileSystemSplit(StorageInf): data = { "SHAREPATH": f"/{share_path}/", "vstoreId": self.vstore_id, - "FSID": clone_fs_id + "FSID": clone_fs_id, } share_id = self.create_nfs_share(data) return share_id @@ -156,8 +160,11 @@ class StorageFileSystemSplit(StorageInf): :param clone_fs_id:克隆文件系统ID :return: """ - LOGGER.info("Begin to clear dbstor nfs share.fs_name:[%s], clone_fs_name:[%s]", self.storage_dbstor_fs, - self.storage_dbstor_page_fs) + LOGGER.info( + "Begin to clear dbstor nfs share.fs_name:[%s], clone_fs_name:[%s]", + self.storage_dbstor_fs, + self.storage_dbstor_page_fs, + ) for _id in [fs_id, clone_fs_id]: share_info = self.query_nfs_info(_id, vstore_id=self.vstore_id) if share_info: @@ -167,7 +174,9 @@ class StorageFileSystemSplit(StorageInf): def pre_upgrade(self): LOGGER.info("Begin to check dbstor page fs info") - page_file_system_info = self.query_filesystem_info(self.storage_dbstor_page_fs, vstore_id=self.vstore_id) + page_file_system_info = self.query_filesystem_info( + self.storage_dbstor_page_fs, vstore_id=self.vstore_id + ) if page_file_system_info: err_msg = "File system [%s] is exist." % self.storage_dbstor_page_fs LOGGER.error(err_msg) @@ -190,12 +199,18 @@ class StorageFileSystemSplit(StorageInf): 10、删除共享 :return: """ - fs_info = self.query_filesystem_info(self.storage_dbstor_fs, vstore_id=self.vstore_id) + fs_info = self.query_filesystem_info( + self.storage_dbstor_fs, vstore_id=self.vstore_id + ) fs_id = fs_info.get("ID") - clone_fs_info = self.query_filesystem_info(self.storage_dbstor_page_fs, vstore_id=self.vstore_id) + clone_fs_info = self.query_filesystem_info( + self.storage_dbstor_page_fs, vstore_id=self.vstore_id + ) if not clone_fs_info: - clone_fs_info = self.create_clone_file_system(fs_id, self.storage_dbstor_page_fs, vstore_id=self.vstore_id) + clone_fs_info = self.create_clone_file_system( + fs_id, self.storage_dbstor_page_fs, vstore_id=self.vstore_id + ) clone_fs_id = clone_fs_info.get("ID") split_status = clone_fs_info.get("SPLITSTATUS") @@ -204,10 +219,14 @@ class StorageFileSystemSplit(StorageInf): if int(split_status) == 1 and split_enable == "false": self.split_clone_file_system(clone_fs_id) - self.query_split_clone_file_system_process(self.storage_dbstor_page_fs, vstore_id=self.vstore_id) + self.query_split_clone_file_system_process( + self.storage_dbstor_page_fs, vstore_id=self.vstore_id + ) - for _fs_id, _fs_name in [(fs_id, self.storage_dbstor_fs), - (clone_fs_id, self.storage_dbstor_page_fs)]: + for _fs_id, _fs_name in [ + (fs_id, self.storage_dbstor_fs), + (clone_fs_id, self.storage_dbstor_page_fs), + ]: _share_id = self.create_clone_share(_fs_id, _fs_name) self.add_clone_share_client(_share_id) @@ -216,7 +235,9 @@ class StorageFileSystemSplit(StorageInf): self.clear_dbstor_nfs_share(_fs_id, clone_fs_id) def rollback(self): - page_fs_info = self.query_filesystem_info(self.storage_dbstor_page_fs, vstore_id=self.vstore_id) + page_fs_info = self.query_filesystem_info( + self.storage_dbstor_page_fs, vstore_id=self.vstore_id + ) if not page_fs_info: return file_system_id = page_fs_info.get("ID") @@ -231,10 +252,16 @@ class StorageFileSystemSplit(StorageInf): cantian_user = i.split("=")[1] if "cantian_user" in i else cantian_user cantian_group = i.split("=")[1] if "cantian_group" in i else cantian_group dbstor_fs_path = f"/mnt/dbdata/remote/{self.storage_dbstor_fs}/{self.namespace}" - dbstor_page_fs_path = f"/mnt/dbdata/remote/{self.storage_dbstor_page_fs}/{self.namespace}" - LOGGER.info("Start change owner of %s and %s", dbstor_fs_path, dbstor_page_fs_path) - cmd = f"chown -hR {cantian_user}:{cantian_group} {dbstor_fs_path} &&" \ - f" chown -hR {cantian_user}:{cantian_group} {dbstor_page_fs_path}" + dbstor_page_fs_path = ( + f"/mnt/dbdata/remote/{self.storage_dbstor_page_fs}/{self.namespace}" + ) + LOGGER.info( + "Start change owner of %s and %s", dbstor_fs_path, dbstor_page_fs_path + ) + cmd = ( + f"chown -hR {cantian_user}:{cantian_group} {dbstor_fs_path} &&" + f" chown -hR {cantian_user}:{cantian_group} {dbstor_page_fs_path}" + ) return_code, _, stderr = exec_popen(cmd) if return_code: err_msg = f"Failed chown {dbstor_fs_path} {dbstor_page_fs_path}, details:{stderr}" @@ -246,7 +273,7 @@ def check_version(): try: version_check.read_source_version_info() except Exception as _err: - err_msg = 'obtain source version failed with error: %s', str(_err) + err_msg = "obtain source version failed with error: %s", str(_err) raise Exception(err_msg) from _err return version_check.source_version.startswith("2.0.0") diff --git a/pkg/deploy/action/update_config.py b/pkg/deploy/action/update_config.py index 4672d1c3..be0ae1ff 100644 --- a/pkg/deploy/action/update_config.py +++ b/pkg/deploy/action/update_config.py @@ -27,8 +27,13 @@ def _exec_popen(cmd, values=None): if not values: values = [] bash_cmd = ["bash"] - pobj = subprocess.Popen(bash_cmd, shell=False, stdin=subprocess.PIPE, - stdout=subprocess.PIPE, stderr=subprocess.PIPE) + pobj = subprocess.Popen( + bash_cmd, + shell=False, + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + ) pobj.stdin.write(cmd.encode()) pobj.stdin.write(os.linesep.encode()) for value in values: @@ -53,15 +58,17 @@ def get_ctencrypt_passwd(passwd): file_path = "/opt/cantian/action/cantian/install_config.json" flags = os.O_RDONLY modes = stat.S_IWUSR | stat.S_IRUSR - with os.fdopen(os.open(file_path, flags, modes), 'r') as fp: + with os.fdopen(os.open(file_path, flags, modes), "r") as fp: json_data = json.load(fp) - install_path = json_data['R_INSTALL_PATH'].strip() + install_path = json_data["R_INSTALL_PATH"].strip() cmd = "source ~/.bashrc && %s/bin/ctencrypt -e PBKDF2" % install_path values = [passwd, passwd] ret_code, stdout, stderr = _exec_popen(cmd, values) if ret_code: - raise OSError("Failed to encrypt password of user [sys]." - " Error: %s" % (stderr + os.linesep + stderr)) + raise OSError( + "Failed to encrypt password of user [sys]." + " Error: %s" % (stderr + os.linesep + stderr) + ) # Example of output: # Please enter password to encrypt: # ********* @@ -127,7 +134,7 @@ def update_dbstor_conf(action, key, value=None): "/opt/cantian/cms/dbstor/conf/dbs/dbstor_config_tool_1.ini", "/opt/cantian/cms/dbstor/conf/dbs/dbstor_config_tool_2.ini", "/opt/cantian/cms/dbstor/conf/dbs/dbstor_config_tool_3.ini", - "/opt/cantian/cms/dbstor/conf/dbs/dbstor_config.ini" + "/opt/cantian/cms/dbstor/conf/dbs/dbstor_config.ini", ] opt_dbstor_config = "/opt/cantian/dbstor/tools/dbstor_config.ini" file_list.append(opt_dbstor_config) @@ -190,42 +197,56 @@ def update_cms_ini_conf(action, key, value): def update_ctsql_config(action, key, value): ctsql_passwd = input() encrypt_passwd = get_ctencrypt_passwd(ctsql_passwd) - update_cmd = f'source ~/.bashrc && echo -e {ctsql_passwd} | ctsql sys@127.0.0.1:1611 -q -c ' \ - f'"alter system set _sys_password=\'{encrypt_passwd}\'"' + update_cmd = ( + f"source ~/.bashrc && echo -e {ctsql_passwd} | ctsql sys@127.0.0.1:1611 -q -c " + f"\"alter system set _sys_password='{encrypt_passwd}'\"" + ) ret_code, stdout, stderr = _exec_popen(update_cmd) stderr = str(stderr) stderr.replace(ctsql_passwd, "****") if ret_code: - raise OSError("Failed to encrypt password of user [sys]." - " Error: %s" % (stderr + os.linesep + stderr)) + raise OSError( + "Failed to encrypt password of user [sys]." + " Error: %s" % (stderr + os.linesep + stderr) + ) if "Succeed" not in stdout: raise Exception("Update ctsql _sys_passwd failed") def update_ctsql_passwd(action, key, value): def _check_passwd(): - check_cmd = f'source ~/.bashrc && echo -e {ctsql_passwd} | ctsql sys@127.0.0.1:1611 -q -c ' \ - '"select version();"' + check_cmd = ( + f"source ~/.bashrc && echo -e {ctsql_passwd} | ctsql sys@127.0.0.1:1611 -q -c " + '"select version();"' + ) ret_code, stdout, stderr = _exec_popen(check_cmd) stderr = str(stderr) stderr.replace(ctsql_passwd, "*****") if ret_code: - raise Exception("Check passwd failed, please ensure that the password is entered correctly.") + raise Exception( + "Check passwd failed, please ensure that the password is entered correctly." + ) + file_path = "/opt/cantian/action/cantian/install_config.json" - with open(file_path, 'r') as fp: + with open(file_path, "r") as fp: json_data = json.load(fp) - data_path = json_data['D_DATA_PATH'].strip() + data_path = json_data["D_DATA_PATH"].strip() sys.path.append(os.path.join(CUR_PATH, "dbstor")) from kmc_adapter import CApiWrapper + primary_keystore = "/opt/cantian/common/config/primary_keystore_bak.ks" standby_keystore = "/opt/cantian/common/config/standby_keystore_bak.ks" kmc_adapter = CApiWrapper(primary_keystore, standby_keystore) kmc_adapter.initialize() ctsql_passwd = getpass.getpass("please input new passwd:").strip() ctencrypt_passwd = kmc_adapter.encrypt(ctsql_passwd.strip()) - split_env = os.environ['LD_LIBRARY_PATH'].split(":") - filtered_env = [single_env for single_env in split_env if "/opt/cantian/dbstor/lib" not in single_env] - os.environ['LD_LIBRARY_PATH'] = ":".join(filtered_env) + split_env = os.environ["LD_LIBRARY_PATH"].split(":") + filtered_env = [ + single_env + for single_env in split_env + if "/opt/cantian/dbstor/lib" not in single_env + ] + os.environ["LD_LIBRARY_PATH"] = ":".join(filtered_env) _check_passwd() _conf_files = os.path.join(data_path, "cfg", "*sql.ini") conf_file = glob.glob(_conf_files)[0] @@ -242,11 +263,24 @@ def update_ctsql_passwd(action, key, value): def main(): update_parse = argparse.ArgumentParser() - update_parse.add_argument("-c", "--component", dest="component", - choices=["dbstor", "cms", "cantian", "cantian_ini", "cms_ini", "ctsql", "ctsql_pwd"], - required=True) - update_parse.add_argument("-a", "--action", dest="action", choices=["del", "add", "update"], - required=True) + update_parse.add_argument( + "-c", + "--component", + dest="component", + choices=[ + "dbstor", + "cms", + "cantian", + "cantian_ini", + "cms_ini", + "ctsql", + "ctsql_pwd", + ], + required=True, + ) + update_parse.add_argument( + "-a", "--action", dest="action", choices=["del", "add", "update"], required=True + ) update_parse.add_argument("-k", "--key", dest="key", required=True) update_parse.add_argument("-v", "--value", dest="value", required=False) args = update_parse.parse_args() diff --git a/pkg/deploy/action/upgrade_version_check.py b/pkg/deploy/action/upgrade_version_check.py index 916a03a1..488ecb46 100644 --- a/pkg/deploy/action/upgrade_version_check.py +++ b/pkg/deploy/action/upgrade_version_check.py @@ -6,20 +6,22 @@ from string import digits from om_log import LOGGER as LOG CUR_PATH = os.path.dirname(os.path.realpath(__file__)) -VERSION_PREFIX = 'Version:' -SUB_VERSION_PREFIX = ('B', 'SP') +VERSION_PREFIX = "Version:" +SUB_VERSION_PREFIX = ("B", "SP") class UpgradeVersionCheck: def __init__(self, white_list=None): self.white_list_file = white_list - self.source_version_file = str(Path('/opt/cantian/versions.yml')) - self.white_list_dict = {} # 格式：{SOURCE-VERSION: [UPGRADE-MODE, CHANGE_SYSTEM]} - self.source_version = '' + self.source_version_file = str(Path("/opt/cantian/versions.yml")) + self.white_list_dict = ( + {} + ) # 格式：{SOURCE-VERSION: [UPGRADE-MODE, CHANGE_SYSTEM]} + self.source_version = "" def process_white_list(self): - with open(self.white_list_file, 'r', encoding='utf-8') as file: + with open(self.white_list_file, "r", encoding="utf-8") as file: white_list_info = file.readlines() for white_list_detail in white_list_info[1:]: @@ -27,8 +29,8 @@ class UpgradeVersionCheck: self.white_list_dict[details[0]] = [details[1], details[2]] def read_source_version_info(self): - version = '' - with open(self.source_version_file, 'r', encoding='utf-8') as file: + version = "" + with open(self.source_version_file, "r", encoding="utf-8") as file: source_version_info = file.readlines() for line in source_version_info: @@ -40,51 +42,61 @@ class UpgradeVersionCheck: def source_version_check(self): result = "" for white_list_version, white_list_detail in self.white_list_dict.items(): - *white_main_version, white_sub_version = white_list_version.split('.') - *source_main_version, source_sub_version = self.source_version.split('.') + *white_main_version, white_sub_version = white_list_version.split(".") + *source_main_version, source_sub_version = self.source_version.split(".") if source_main_version != white_main_version: continue - if white_sub_version == '*' or white_sub_version == source_sub_version: - result = "{} {} {}".format(self.source_version, white_list_detail[0], white_list_detail[1]) + if white_sub_version == "*" or white_sub_version == source_sub_version: + result = "{} {} {}".format( + self.source_version, white_list_detail[0], white_list_detail[1] + ) break - if '-' in white_sub_version: - min_version, max_version = white_sub_version.split('-') - trans_map = str.maketrans('', '', digits) + if "-" in white_sub_version: + min_version, max_version = white_sub_version.split("-") + trans_map = str.maketrans("", "", digits) source_pre_fix = source_sub_version.translate(trans_map) - if source_pre_fix not in SUB_VERSION_PREFIX: # 源版本号开头不是B或者SPH返回结果为空 + if ( + source_pre_fix not in SUB_VERSION_PREFIX + ): # 源版本号开头不是B或者SPH返回结果为空 break - sub_version_min_num = min_version.replace(source_pre_fix, '') - sub_version_max_num = max_version.replace(source_pre_fix, '') - sub_source_version_num = source_sub_version.replace(source_pre_fix, '') - if sub_version_min_num.isdigit() \ - and sub_version_max_num.isdigit() \ - and int(sub_version_max_num) >= int(sub_source_version_num) >= int(sub_version_min_num): - result = "{} {} {}".format(self.source_version, white_list_detail[0], white_list_detail[1]) + sub_version_min_num = min_version.replace(source_pre_fix, "") + sub_version_max_num = max_version.replace(source_pre_fix, "") + sub_source_version_num = source_sub_version.replace(source_pre_fix, "") + if ( + sub_version_min_num.isdigit() + and sub_version_max_num.isdigit() + and int(sub_version_max_num) + >= int(sub_source_version_num) + >= int(sub_version_min_num) + ): + result = "{} {} {}".format( + self.source_version, white_list_detail[0], white_list_detail[1] + ) break return result -if __name__ == '__main__': +if __name__ == "__main__": white_list_input = sys.argv[1] version_check = UpgradeVersionCheck(white_list_input) try: version_check.process_white_list() except Exception as err: - LOG.error(f'obtain source version white list failed with error: {str(err)}') - exit('') + LOG.error(f"obtain source version white list failed with error: {str(err)}") + exit("") try: version_check.read_source_version_info() except Exception as err: - LOG.error(f'obtain source version failed with error: {str(err)}') - exit('') + LOG.error(f"obtain source version failed with error: {str(err)}") + exit("") try: print(version_check.source_version_check()) except Exception as err: - LOG.error(f'source version check failed with error: {str(err)}') - exit('') + LOG.error(f"source version check failed with error: {str(err)}") + exit("") diff --git a/pkg/deploy/action/utils/client/response_parse.py b/pkg/deploy/action/utils/client/response_parse.py index f1871a38..535a07ae 100644 --- a/pkg/deploy/action/utils/client/response_parse.py +++ b/pkg/deploy/action/utils/client/response_parse.py @@ -1,5 +1,6 @@ # coding=utf-8 + class ResponseParse(object): def __init__(self, res): """ @@ -15,11 +16,11 @@ class ResponseParse(object): if status_code == 200: res = self.res.json() if "error" in res: - ret_result = res.get('error') + ret_result = res.get("error") else: - ret_result = res.get('result') - error_code = ret_result['code'] - error_des = ret_result['description'] + ret_result = res.get("result") + error_code = ret_result["code"] + error_des = ret_result["description"] if error_des is None or error_code == 0: error_des = "success" return status_code, int(error_code), error_des @@ -32,10 +33,10 @@ class ResponseParse(object): if status_code == 200: rsp_code = 0 if "error" in self.res.json(): - ret_result = self.res.json().get('error') + ret_result = self.res.json().get("error") if "result" in self.res.json(): - ret_result = self.res.json().get('result') - ret_data = self.res.json().get('data') + ret_result = self.res.json().get("result") + ret_data = self.res.json().get("data") return rsp_code, ret_result, ret_data def get_omtask_rsp_data(self): @@ -46,8 +47,8 @@ class ResponseParse(object): if status_code == 200: rsp_code = 0 if "error" in self.res.json(): - ret_result = self.res.json().get('error') + ret_result = self.res.json().get("error") if "result" in self.res.json(): - ret_result = self.res.json().get('result') - ret_data = self.res.json().get('data') + ret_result = self.res.json().get("result") + ret_data = self.res.json().get("data") return rsp_code, ret_result, ret_data diff --git a/pkg/deploy/action/utils/client/rest_client.py b/pkg/deploy/action/utils/client/rest_client.py index 76ff37d3..fae06084 100644 --- a/pkg/deploy/action/utils/client/rest_client.py +++ b/pkg/deploy/action/utils/client/rest_client.py @@ -21,7 +21,11 @@ NORMAL_STATE, ABNORMAL_STATE = 0, 1 def get_cur_timestamp(): utc_now = datetime.utcnow() - return utc_now.replace(tzinfo=timezone.utc).astimezone(tz=None).strftime('%Y%m%d%H%M%S') + return ( + utc_now.replace(tzinfo=timezone.utc) + .astimezone(tz=None) + .strftime("%Y%m%d%H%M%S") + ) class ExecutionError(Exception): @@ -41,67 +45,81 @@ class RestClient: def gen_timestamp(): utc_now = datetime.utcnow() cur_time = utc_now.replace(tzinfo=timezone.utc).astimezone(tz=None) - return str(cur_time.strftime('%Y%m%d%H%M%S')) + return str(cur_time.strftime("%Y%m%d%H%M%S")) @staticmethod def exception_handler(err_msg=None, cur_mode=None): - err_info = '[current_mode] {}, [err_info] {}'.format(cur_mode, err_msg) + err_info = "[current_mode] {}, [err_info] {}".format(cur_mode, err_msg) LOG.error(err_info) raise ExecutionError(err_info) @staticmethod def response_parse(res_data): status_code = res_data.status_code - err_code, err_details = -1, 'failed' + err_code, err_details = -1, "failed" if status_code == 200: exec_res = res_data.json() - err_code, err_details = \ - exec_res.get('error').get('code'), exec_res.get('error').get('description') + err_code, err_details = exec_res.get("error").get("code"), exec_res.get( + "error" + ).get("description") return status_code, int(err_code), err_details def update_cookies(self, res): - res_body, set_cookie = res.json().get('data'), res.headers.get('Set-Cookie') + res_body, set_cookie = res.json().get("data"), res.headers.get("Set-Cookie") - self.token, self.device_id = res_body.get('iBaseToken'), res_body.get('deviceid') + self.token, self.device_id = res_body.get("iBaseToken"), res_body.get( + "deviceid" + ) - match_res = re.findall(r'session=ismsession=\w+;', set_cookie) + match_res = re.findall(r"session=ismsession=\w+;", set_cookie) if match_res: self.ism_session = match_res[0][:-1] - def make_header(self, content_type='application/json'): - header = {'Content-type': content_type} + def make_header(self, content_type="application/json"): + header = {"Content-type": content_type} if self.token: - header['iBaseToken'] = self.token + header["iBaseToken"] = self.token if self.ism_session: - header['Cookie'] = self.ism_session + header["Cookie"] = self.ism_session return header def login(self, keep_session=False): - url = '{}{}:{}{}'.format(RestElemConstant.HTTPS, self.ip_addr, RestElemConstant.PORT, RestElemConstant.LOGIN) + url = "{}{}:{}{}".format( + RestElemConstant.HTTPS, + self.ip_addr, + RestElemConstant.PORT, + RestElemConstant.LOGIN, + ) user_info = { - 'username': self.user_name, - 'password': self.passwd, - 'scope': 0, - 'loginMode': 3, - 'timeConversion': 0, - 'isEncrypt': 'false' + "username": self.user_name, + "password": self.passwd, + "scope": 0, + "loginMode": 3, + "timeConversion": 0, + "isEncrypt": "false", } login_header = { - 'Content-type': 'application/json', - 'Cookie': '__LANGUAGE_KEY__=zh-CN; __IBASE_LANGUAGE_KEY__=zh-CN' + "Content-type": "application/json", + "Cookie": "__LANGUAGE_KEY__=zh-CN; __IBASE_LANGUAGE_KEY__=zh-CN", } requests.packages.urllib3.disable_warnings() with requests.session() as session: - res = session.post(url, data=json.dumps(user_info), headers=login_header, verify=False) + res = session.post( + url, data=json.dumps(user_info), headers=login_header, verify=False + ) status_code, err_code, err_details = self.response_parse(res) if err_code: - err_msg = ('Login DM failed {}, status_code: {}, err_code: {}, ' - 'err_details: {}'.format(self.ip_addr, status_code, err_code, err_details)) + err_msg = ( + "Login DM failed {}, status_code: {}, err_code: {}, " + "err_details: {}".format( + self.ip_addr, status_code, err_code, err_details + ) + ) raise Exception(err_msg) self.update_cookies(res) @@ -112,11 +130,11 @@ class RestClient: else: res.close() - return NORMAL_STATE, 'success' + return NORMAL_STATE, "success" def logout(self): url = RestElemConstant.LOGOUT.format(deviceId=self.device_id) - res = self.normal_request(url, 'delete') + res = self.normal_request(url, "delete") result = ResponseParse(res) status_code, error_code, error_des = result.get_res_code() if status_code != 200 or error_code != 0: @@ -137,27 +155,35 @@ class RestClient: url = Constant.HTTPS + self.ip_addr + ":" + Constant.PORT + url if keep_session: req = self.session - self.token = self.res_login.get('data').get('ibasetoken') + self.token = self.res_login.get("data").get("ibasetoken") else: req = requests.session() headers = self.make_header() with req as session: - if method == 'put': - res = session.put(url, data=data, headers=headers, verify=False, timeout=timeout) - elif method == 'post': - res = session.post(url, data=data, headers=headers, verify=False, timeout=timeout) - elif method == 'get': - res = session.get(url, data=data, headers=headers, verify=False, timeout=timeout) - elif method == 'delete': - res = session.delete(url, data=data, headers=headers, verify=False, timeout=timeout) + if method == "put": + res = session.put( + url, data=data, headers=headers, verify=False, timeout=timeout + ) + elif method == "post": + res = session.post( + url, data=data, headers=headers, verify=False, timeout=timeout + ) + elif method == "get": + res = session.get( + url, data=data, headers=headers, verify=False, timeout=timeout + ) + elif method == "delete": + res = session.delete( + url, data=data, headers=headers, verify=False, timeout=timeout + ) res.close() return res def read_helper(file_path): - with open(file_path, 'r', encoding='utf-8') as f_handler: + with open(file_path, "r", encoding="utf-8") as f_handler: deploy_data = f_handler.read() return deploy_data @@ -166,8 +192,8 @@ def write_helper(file_path, data): modes = stat.S_IWRITE | stat.S_IRUSR flags = os.O_WRONLY | os.O_TRUNC | os.O_CREAT if data: - with os.fdopen(os.open(file_path, flags, modes), 'w', encoding='utf-8') as file: + with os.fdopen(os.open(file_path, flags, modes), "w", encoding="utf-8") as file: file.write(json.dumps(data, indent=4)) else: - with os.fdopen(os.open(file_path, flags, modes), 'w', encoding='utf-8') as file: + with os.fdopen(os.open(file_path, flags, modes), "w", encoding="utf-8") as file: file.truncate() diff --git a/pkg/deploy/action/utils/client/ssh_client.py b/pkg/deploy/action/utils/client/ssh_client.py index 0eee8e3e..78400c9d 100644 --- a/pkg/deploy/action/utils/client/ssh_client.py +++ b/pkg/deploy/action/utils/client/ssh_client.py @@ -19,12 +19,20 @@ def convert(code): :param code: :return: """ - code = code.decode('utf-8', errors='ignore') if isinstance(code, bytes) else code + code = code.decode("utf-8", errors="ignore") if isinstance(code, bytes) else code return code class SshClient(object): - def __init__(self, ip, username, passwd=None, root_passwd=None, port=22, private_key_file=None): + def __init__( + self, + ip, + username, + passwd=None, + root_passwd=None, + port=22, + private_key_file=None, + ): self.username = username self.passwd = passwd self.root_pwd = root_passwd @@ -39,8 +47,11 @@ class SshClient(object): if trans: trans.close() except Exception as ex: - logger.error("failed to close connection after creating SSH client failed: {}." - .format(ex)) + logger.error( + "failed to close connection after creating SSH client failed: {}.".format( + ex + ) + ) def create_client(self, timeout=180, width=300): trans = None @@ -62,21 +73,35 @@ class SshClient(object): time.sleep(interval) t += interval if session.closed: - output += convert(session.recv(SSH_RECV_BUFFER).decode('utf-8')).\ - replace(' \r', '\r').replace('\r', '') + output += ( + convert(session.recv(SSH_RECV_BUFFER).decode("utf-8")) + .replace(" \r", "\r") + .replace("\r", "") + ) break if not session.recv_ready(): continue - output += convert(session.recv(SSH_RECV_BUFFER).decode('utf-8')).\ - replace(' \r', '\r').replace('\r', '') + output += ( + convert(session.recv(SSH_RECV_BUFFER).decode("utf-8")) + .replace(" \r", "\r") + .replace("\r", "") + ) return output.split(cmd)[-1] def close_client(self): try: - if self.ssh_client and isinstance(self.ssh_client, dict) and 'client' in self.ssh_client: - self.ssh_client['client'].close() - if self.ssh_client and isinstance(self.ssh_client, dict) and 'sshClient' in self.ssh_client: - self.ssh_client['sshClient'].close() + if ( + self.ssh_client + and isinstance(self.ssh_client, dict) + and "client" in self.ssh_client + ): + self.ssh_client["client"].close() + if ( + self.ssh_client + and isinstance(self.ssh_client, dict) + and "sshClient" in self.ssh_client + ): + self.ssh_client["sshClient"].close() except Exception as err: err_msg = "Close ssh client err, details:%s" % str(err) logger.error(err_msg) @@ -89,7 +114,9 @@ class SshClient(object): if self.passwd: trans.connect(username=self.username, password=self.passwd) else: - private_key = paramiko.RSAKey.from_private_key_file(self.private_key_file) + private_key = paramiko.RSAKey.from_private_key_file( + self.private_key_file + ) trans.connect(username=self.username, pkey=private_key) except Exception as err: err_mgs = "Create ssh client failed, details: %s" % str(err) @@ -100,15 +127,15 @@ class SshClient(object): channel.settimeout(timeout) channel.get_pty(width=width) channel.invoke_shell() - stdout = channel.makefile('r', -1) - self.ssh_client['ip'] = self.ip - self.ssh_client['port'] = self.port - self.ssh_client['username'] = self.username - self.ssh_client['timeout'] = timeout - self.ssh_client['width'] = width - self.ssh_client['client'] = trans - self.ssh_client['channel'] = channel - self.ssh_client['stdout'] = stdout + stdout = channel.makefile("r", -1) + self.ssh_client["ip"] = self.ip + self.ssh_client["port"] = self.port + self.ssh_client["username"] = self.username + self.ssh_client["timeout"] = timeout + self.ssh_client["width"] = width + self.ssh_client["client"] = trans + self.ssh_client["channel"] = channel + self.ssh_client["stdout"] = stdout logger.info("Create ssh client host[%s] success" % self.ip) return trans @@ -120,7 +147,7 @@ class SshClient(object): """ sftp = None try: - sftp = paramiko.SFTPClient.from_transport(self.ssh_client['client']) + sftp = paramiko.SFTPClient.from_transport(self.ssh_client["client"]) sftp.put(source, dest) except Exception as err: err_msg = f"Upload failed from {source} to {dest}, details: {err}" @@ -144,7 +171,7 @@ class SshClient(object): dest = os.path.join(dest, filename) if os.path.exists(dest): os.remove(dest) - sftp = paramiko.SFTPClient.from_transport(self.ssh_client['client']) + sftp = paramiko.SFTPClient.from_transport(self.ssh_client["client"]) sftp.get(source, dest) except Exception as err: err_msg = f"download failed from {source} to {dest}, details: {err}" @@ -153,4 +180,3 @@ class SshClient(object): finally: if sftp is not None: sftp.close() - diff --git a/pkg/deploy/action/utils/config/rest_constant.py b/pkg/deploy/action/utils/config/rest_constant.py index 47a08b34..2f0e097a 100644 --- a/pkg/deploy/action/utils/config/rest_constant.py +++ b/pkg/deploy/action/utils/config/rest_constant.py @@ -1,56 +1,84 @@ class Constant: - PORT = '8088' - HTTPS = 'https://' - LOGIN = '/deviceManager/rest/xxxxx/login' - LOGOUT = '/deviceManager/rest/{deviceId}/sessions' - QUERY_POOL = '/deviceManager/rest/{deviceId}/storagepool' - CREATE_FS = '/deviceManager/rest/{deviceId}/filesystem' - QUERY_FILE_SYSTEM_NUM = '/deviceManager/rest/{deviceId}/filesystem/count' - DELETE_FS = '/deviceManager/rest/{deviceId}/filesystem/{id}' - NFS_SERVICE = '/deviceManager/rest/{deviceId}/nfsservice' - NFS_SHARE_ADD = '/deviceManager/rest/{deviceId}/NFSSHARE' - NFS_SHARE_ADD_CLIENT = '/deviceManager/rest/{deviceId}/NFS_SHARE_AUTH_CLIENT' - NFS_SHARE_DELETE = '/deviceManager/rest/{deviceId}/NFSSHARE/{id}' - NFS_SHARE_DEL_CLIENT = '/deviceManager/rest/{deviceId}/NFS_SHARE_AUTH_CLIENT/{id}' - NFS_SHARE_QUERY = '/deviceManager/rest/{deviceId}/NFSSHARE' - QUERY_VSTORE = '/deviceManager/rest/{deviceId}/vstore/count' - CREATE_VSTORE = '/deviceManager/rest/{deviceId}/vstore' - DELETE_VSTORE = '/deviceManager/rest/{deviceId}/vstore/{id}' + PORT = "8088" + HTTPS = "https://" + LOGIN = "/deviceManager/rest/xxxxx/login" + LOGOUT = "/deviceManager/rest/{deviceId}/sessions" + QUERY_POOL = "/deviceManager/rest/{deviceId}/storagepool" + CREATE_FS = "/deviceManager/rest/{deviceId}/filesystem" + QUERY_FILE_SYSTEM_NUM = "/deviceManager/rest/{deviceId}/filesystem/count" + DELETE_FS = "/deviceManager/rest/{deviceId}/filesystem/{id}" + NFS_SERVICE = "/deviceManager/rest/{deviceId}/nfsservice" + NFS_SHARE_ADD = "/deviceManager/rest/{deviceId}/NFSSHARE" + NFS_SHARE_ADD_CLIENT = "/deviceManager/rest/{deviceId}/NFS_SHARE_AUTH_CLIENT" + NFS_SHARE_DELETE = "/deviceManager/rest/{deviceId}/NFSSHARE/{id}" + NFS_SHARE_DEL_CLIENT = "/deviceManager/rest/{deviceId}/NFS_SHARE_AUTH_CLIENT/{id}" + NFS_SHARE_QUERY = "/deviceManager/rest/{deviceId}/NFSSHARE" + QUERY_VSTORE = "/deviceManager/rest/{deviceId}/vstore/count" + CREATE_VSTORE = "/deviceManager/rest/{deviceId}/vstore" + DELETE_VSTORE = "/deviceManager/rest/{deviceId}/vstore/{id}" CREATE_LIF = "/deviceManager/rest/{deviceId}/lif" DELETE_LIF = "/deviceManager/rest/{deviceId}/lif?NAME={name}" CREATE_CLONE_FS = "/deviceManager/rest/{deviceId}/filesystem" SPLIT_CLONE_FS = "/deviceManager/rest/{deviceId}/clone_fs_split" CREATE_FSSNAPSHOT = "/deviceManager/rest/{deviceId}/fssnapshot" ROLLBACK_SNAPSHOT = "/deviceManager/rest/{deviceId}/fssnapshot/rollback_fssnapshot" - QUERY_ROLLBACK_SNAPSHOT_PROCESS = "/deviceManager/rest/{deviceId}/FSSNAPSHOT/" \ - "query_fs_snapshot_rollback?PARENTNAME={fs_name}" + QUERY_ROLLBACK_SNAPSHOT_PROCESS = ( + "/deviceManager/rest/{deviceId}/FSSNAPSHOT/" + "query_fs_snapshot_rollback?PARENTNAME={fs_name}" + ) QUERY_LOGIC_PORT_INFO = "/deviceManager/rest/{deviceId}/lif" # 容灾查询操作 QUERY_SYSTEM_INFO = "/deviceManager/rest/{deviceId}/system/" QUERY_REMOTE_DEVICE_INFO = "/deviceManager/rest/{deviceId}/remote_device" QUERY_LICENSE_FEATURE = "/deviceManager/rest/{deviceId}/license/feature" - QUERY_HYPER_METRO_FILE_SYSTEM_PAIR = "/deviceManager/rest/{deviceId}/HyperMetroPair/associate" - QUERY_HYPER_METRO_FILE_SYSTEM_COUNT = "/deviceManager/rest/{deviceId}/HyperMetroPair/count" - QUERY_REPLICATION_FILE_SYSTEM_PAIR = "/deviceManager/rest/{deviceId}/replicationpair/associate" - QUERY_FILESYSTEM_FOR_REPLICATION = "/deviceManager/rest/{deviceId}/filesystem_for_replication" + QUERY_HYPER_METRO_FILE_SYSTEM_PAIR = ( + "/deviceManager/rest/{deviceId}/HyperMetroPair/associate" + ) + QUERY_HYPER_METRO_FILE_SYSTEM_COUNT = ( + "/deviceManager/rest/{deviceId}/HyperMetroPair/count" + ) + QUERY_REPLICATION_FILE_SYSTEM_PAIR = ( + "/deviceManager/rest/{deviceId}/replicationpair/associate" + ) + QUERY_FILESYSTEM_FOR_REPLICATION = ( + "/deviceManager/rest/{deviceId}/filesystem_for_replication" + ) # 容灾搭建操作 HYPER_METRO_DOMAIN = "/deviceManager/rest/{deviceId}/FsHyperMetroDomain" HYPER_METRO_VSTORE_PAIR = "/deviceManager/rest/{deviceId}/vstore_pair" HYPER_METRO_FILESYSTEM_PAIR = "/deviceManager/rest/{deviceId}/HyperMetroPair" - SPLIT_REMOTE_REPLICATION_FILESYSTEM_PAIR = "/deviceManager/rest/{deviceId}/REPLICATIONPAIR/split" - SYNC_REMOTE_REPLICATION_FILESYSTEM_PAIR = "/deviceManager/rest/{deviceId}/REPLICATIONPAIR/sync" - REMOTE_REPLICATION_FILESYSTEM_PAIR_OPT = "/deviceManager/rest/{deviceId}/REPLICATIONPAIR/{id}" + SPLIT_REMOTE_REPLICATION_FILESYSTEM_PAIR = ( + "/deviceManager/rest/{deviceId}/REPLICATIONPAIR/split" + ) + SYNC_REMOTE_REPLICATION_FILESYSTEM_PAIR = ( + "/deviceManager/rest/{deviceId}/REPLICATIONPAIR/sync" + ) + REMOTE_REPLICATION_FILESYSTEM_PAIR_OPT = ( + "/deviceManager/rest/{deviceId}/REPLICATIONPAIR/{id}" + ) DELETE_HYPER_METRO_PAIR = "/deviceManager/rest/{deviceId}/HyperMetroPair/{id}" DELETE_HYPER_METRO_VSTORE_PAIR = "/deviceManager/rest/{deviceId}/vstore_pair/{id}" - SPLIT_FILESYSTEM_HYPER_METRO_DOMAIN = "/deviceManager/rest/{deviceId}/SplitFsHyperMetroDomain" - DELETE_FILESYSTEM_HYPER_METRO_DOMAIN = "/deviceManager/rest/{deviceId}/FsHyperMetroDomain/{id}" - CANCEL_SECONDARY_WRITE_LOCK = "/deviceManager/rest/{deviceId}/REPLICATIONPAIR/CANCEL_SECONDARY_WRITE_LOCK" - SET_SECONDARY_WRITE_LOCK = "/deviceManager/rest/{deviceId}/REPLICATIONPAIR/SET_SECONDARY_WRITE_LOCK" - SWAP_ROLE_FS_HYPER_METRO_DOMAIN = "/deviceManager/rest/{deviceId}/SwapRoleFsHyperMetroDomain" + SPLIT_FILESYSTEM_HYPER_METRO_DOMAIN = ( + "/deviceManager/rest/{deviceId}/SplitFsHyperMetroDomain" + ) + DELETE_FILESYSTEM_HYPER_METRO_DOMAIN = ( + "/deviceManager/rest/{deviceId}/FsHyperMetroDomain/{id}" + ) + CANCEL_SECONDARY_WRITE_LOCK = ( + "/deviceManager/rest/{deviceId}/REPLICATIONPAIR/CANCEL_SECONDARY_WRITE_LOCK" + ) + SET_SECONDARY_WRITE_LOCK = ( + "/deviceManager/rest/{deviceId}/REPLICATIONPAIR/SET_SECONDARY_WRITE_LOCK" + ) + SWAP_ROLE_FS_HYPER_METRO_DOMAIN = ( + "/deviceManager/rest/{deviceId}/SwapRoleFsHyperMetroDomain" + ) SWAP_ROLE_REPLICATION_PAIR = "/deviceManager/rest/{deviceId}/REPLICATIONPAIR/switch" - CHANGE_FS_HYPER_METRO_DOMAIN_SECOND_ACCESS = "/deviceManager/rest/{deviceId}/ChangeFsHyperMetroDomainSecondAccess" + CHANGE_FS_HYPER_METRO_DOMAIN_SECOND_ACCESS = ( + "/deviceManager/rest/{deviceId}/ChangeFsHyperMetroDomainSecondAccess" + ) JOIN_FS_HYPER_METRO_DOMAIN = "/deviceManager/rest/{deviceId}/JoinFsHyperMetroDomain" # omtask rest api @@ -59,24 +87,21 @@ class Constant: QUERY_TASK_PROCESS = "/api/v2/task/{id}" CREATE_REMOTE_REPLICATION_FILESYSTEM_PAIR = "/api/v2/task/protection/nas" # cdp操作 - DELETE_FS_CDP_SCHEDULE = '/deviceManager/rest/{deviceId}/filesystem/remove_associate' + DELETE_FS_CDP_SCHEDULE = ( + "/deviceManager/rest/{deviceId}/filesystem/remove_associate" + ) FULL_SYNC_MAX_TIME = 1500 CANTIAN_DOMAIN_PREFIX = "CantianDomain_%s%s" -SPEED = { - "low": 1, - "medium": 2, - "high": 3, - "highest": 4 -} +SPEED = {"low": 1, "medium": 2, "high": 3, "highest": 4} class HealthStatus: - Normal = "1" # 正常 - Faulty = "2" # 故障 - Invalid = "14" # 失效 + Normal = "1" # 正常 + Faulty = "2" # 故障 + Invalid = "14" # 失效 class SystemRunningStatus: @@ -88,21 +113,21 @@ class SystemRunningStatus: class RemoteDeviceStatus: - LinkUp = "10" # 已连接 - LinkDown = "11" # 未连接 - Disabled = "31" # 已禁用 - Connecting = "101" # 正在连接 - AirGapLinkDown = "118" # Air Gap断开 + LinkUp = "10" # 已连接 + LinkDown = "11" # 未连接 + Disabled = "31" # 已禁用 + Connecting = "101" # 正在连接 + AirGapLinkDown = "118" # Air Gap断开 class ReplicationRunningStatus: - Normal = "1" # 正常 - Synchronizing = "23" # 正在同步 - TobeRecovered = "33" # 待恢复 - Interrupted = "34" # 异常断开 - Split = "26" # 已分裂 - Invalid = "35" # 失效 - Standby = "110" # 备用 + Normal = "1" # 正常 + Synchronizing = "23" # 正在同步 + TobeRecovered = "33" # 待恢复 + Interrupted = "34" # 异常断开 + Split = "26" # 已分裂 + Invalid = "35" # 失效 + Standby = "110" # 备用 class FilesystemRunningStatus: @@ -113,68 +138,68 @@ class FilesystemRunningStatus: class MetroDomainRunningStatus: - Normal = "0" # 正常 - Recovering = "1" # 恢复中 - Faulty = "2" # 故障 - Split = "3" # 分裂 - ForceStarted = "4" # 强制拉起 - Invalid = "5" # 失效 + Normal = "0" # 正常 + Recovering = "1" # 恢复中 + Faulty = "2" # 故障 + Split = "3" # 分裂 + ForceStarted = "4" # 强制拉起 + Invalid = "5" # 失效 class VstorePairRunningStatus: - Normal = "1" # 正常 - Unsynchronized = "25" # 未同步 - Split = "26" # 分裂 - Invalid = "35" # 失效 - ForceStarted = "93" # 强制启动 + Normal = "1" # 正常 + Unsynchronized = "25" # 未同步 + Split = "26" # 分裂 + Invalid = "35" # 失效 + ForceStarted = "93" # 强制启动 class VstorePairConfigStatus: - Normal = "0" # 正常 - Synchronizing = "1" # 同步中 - Unsynchronizing = "2" # 待同步 + Normal = "0" # 正常 + Synchronizing = "1" # 同步中 + Unsynchronizing = "2" # 待同步 class FilesystemPairRunningStatus: - Normal = "1" # 正常 - Synchronizing = "23" # 同步中 - Invalid = "35" # 失效 - Paused = "41" # 暂停 - ForceStarted = "93" # 强制启动 + Normal = "1" # 正常 + Synchronizing = "23" # 同步中 + Invalid = "35" # 失效 + Paused = "41" # 暂停 + ForceStarted = "93" # 强制启动 ToBeSynchronized = "100" # 待同步 - Creating = "119" # 创建中 + Creating = "119" # 创建中 class SecresAccess: - AccessDenied = "1" # 禁止访问 - ReadOnly = "2" # 只读 - ReadAndWrite = "3" # 读写 + AccessDenied = "1" # 禁止访问 + ReadOnly = "2" # 只读 + ReadAndWrite = "3" # 读写 class PoolStatus: - PreCopy = "14" # 预拷贝 - Rebuilt = "16" # 重构 - Online = "27" # 在线 - Offline = "28" # 离线 - Balancing = "32" # 正在均衡 - Initializing = "53" # 初始化中 - Deleting = "106" # 删除中 + PreCopy = "14" # 预拷贝 + Rebuilt = "16" # 重构 + Online = "27" # 在线 + Offline = "28" # 离线 + Balancing = "32" # 正在均衡 + Initializing = "53" # 初始化中 + Deleting = "106" # 删除中 class PoolHealth: - Normal = "1" # 正常 - Faulty = "2" # 故障 - Degraded = "5" # 降级 + Normal = "1" # 正常 + Faulty = "2" # 故障 + Degraded = "5" # 降级 class DomainAccess: ReadAndWrite = "3" # 读写 - ReadOnly = "1" # 只读 + ReadOnly = "1" # 只读 class ConfigRole: - Secondary = "0" # 从端 - Primary = "1" # 主端 + Secondary = "0" # 从端 + Primary = "1" # 主端 class DataIntegrityStatus: @@ -185,4 +210,3 @@ class DataIntegrityStatus: class RepFileSystemNameRule: NamePrefix = "ct_" NameSuffix = "_rep" - diff --git a/pkg/deploy/action/write_config.py b/pkg/deploy/action/write_config.py index a4bd0fc6..b1269ce0 100644 --- a/pkg/deploy/action/write_config.py +++ b/pkg/deploy/action/write_config.py @@ -19,7 +19,7 @@ JVViGOl8yXdK5nFdBq+Oj4KDE5FMensDnZm0HZIyDxm9e2foOmHBYAKZN5novclj""" def read_install_file(): - with open(INSTALL_FILE, 'r', encoding='utf8') as file_path: + with open(INSTALL_FILE, "r", encoding="utf8") as file_path: _tmp = file_path.read() info = json.loads(_tmp) return info @@ -28,12 +28,12 @@ def read_install_file(): def write_install_file(write_data): modes = stat.S_IRWXU | stat.S_IROTH | stat.S_IRGRP flag = os.O_RDWR | os.O_CREAT | os.O_TRUNC - with os.fdopen(os.open(INSTALL_FILE, flag, modes), 'w') as file_path: + with os.fdopen(os.open(INSTALL_FILE, flag, modes), "w") as file_path: config_params = json.dumps(write_data, indent=4) file_path.write(config_params) -if __name__ == '__main__': +if __name__ == "__main__": config_key = sys.argv[1] config_value = sys.argv[2] install_file_data = read_install_file() diff --git a/pkg/install/Common.py b/pkg/install/Common.py index 093739f5..2d816977 100644 --- a/pkg/install/Common.py +++ b/pkg/install/Common.py @@ -10,6 +10,7 @@ import sys + sys.dont_write_bytecode = True try: @@ -25,8 +26,10 @@ class DefaultValue(object): """ Default value of some variables """ + def __init__(self): pass + # file mode MAX_FILE_MODE = 640 MIN_FILE_MODE = 400 @@ -34,7 +37,7 @@ class DefaultValue(object): MID_FILE_MODE = 500 KEY_DIRECTORY_MODE = 700 MAX_DIRECTORY_MODE = 750 - KEY_DIRECTORY_MODE_STR = '0700' + KEY_DIRECTORY_MODE_STR = "0700" MIN_FILE_PERMISSION = 0o400 MID_FILE_PERMISSION = 0o500 KEY_FILE_PERMISSION = 0o600 @@ -43,7 +46,7 @@ class DefaultValue(object): DOCKER_SHARE_DIR = "/home/regress/cantian_data" DOCKER_DATA_DIR = "{}/data".format(DOCKER_SHARE_DIR) DOCKER_GCC_DIR = "{}/gcc_home".format(DOCKER_SHARE_DIR) - + # get os version and python version CURRENT_OS = platform.system() PY_VERSION = platform.python_version() @@ -63,7 +66,7 @@ class DefaultValue(object): while True: # find the top path to be created (tmpDir, topDirName) = os.path.split(tmpDir) - if (os.path.exists(tmpDir) or topDirName == ""): + if os.path.exists(tmpDir) or topDirName == "": tmpDir = os.path.join(tmpDir, topDirName) break return tmpDir @@ -110,8 +113,13 @@ class DefaultValue(object): bash_cmd = ["bash"] if not stdin_list: stdin_list = [] - pobj = subprocess.Popen(bash_cmd, shell=False, stdin=subprocess.PIPE, - stdout=subprocess.PIPE, stderr=subprocess.PIPE) + pobj = subprocess.Popen( + bash_cmd, + shell=False, + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + ) # in python 3, the stand output and stand error is # unicode, we must decode it before return diff --git a/pkg/install/funclib.py b/pkg/install/funclib.py index 6569eade..57ee7585 100644 --- a/pkg/install/funclib.py +++ b/pkg/install/funclib.py @@ -4,6 +4,7 @@ # Copyright © Huawei Technologies Co., Ltd. 2010-2018. All rights reserved. import sys + sys.dont_write_bytecode = True try: import os @@ -28,15 +29,17 @@ except ImportError as import_err: py_verion = platform.python_version() -SYS_PATH = os.environ["PATH"].split(':') +SYS_PATH = os.environ["PATH"].split(":") class CommonValue(object): """ common value for some variables """ + def __init__(self): pass + # file mode MAX_FILE_MODE = 640 MIN_FILE_MODE = 400 @@ -46,7 +49,7 @@ class CommonValue(object): KEY_DIRECTORY_MODE = 700 MAX_DIRECTORY_MODE = 750 - KEY_DIRECTORY_MODE_STR = '0700' + KEY_DIRECTORY_MODE_STR = "0700" MIN_FILE_PERMISSION = 0o400 MID_FILE_PERMISSION = 0o500 @@ -63,6 +66,7 @@ class DefaultConfigValue(object): """ default value for cantiand, cms, gss config """ + def __init__(self): pass @@ -124,7 +128,7 @@ class DefaultConfigValue(object): "_UNDO_ACTIVE_SEGMENTS": 64, "USE_LARGE_PAGES": "FALSE", "CTSTORE_MAX_OPEN_FILES": 40960, - "REPLAY_PRELOAD_PROCESSES":0, + "REPLAY_PRELOAD_PROCESSES": 0, "LOG_REPLAY_PROCESSES": 64, "_LOG_MAX_FILE_SIZE": "4G", "_LOG_BACKUP_FILE_COUNT": 128, @@ -144,19 +148,19 @@ class DefaultConfigValue(object): "REPL_SCRAM_AUTH": "TRUE", "ENABLE_ACCESS_DC": "FALSE", "REPLACE_PASSWORD_VERIFY": "TRUE", - "LOG_HOME": "", #generate by installer - "_SYS_PASSWORD": "", #input by user in command line parameter or from shell command interactively - "INTERCONNECT_ADDR": "", #input by user in command line parameter - "LSNR_ADDR": "", #input by user in command line parameter + "LOG_HOME": "", # generate by installer + "_SYS_PASSWORD": "", # input by user in command line parameter or from shell command interactively + "INTERCONNECT_ADDR": "", # input by user in command line parameter + "LSNR_ADDR": "", # input by user in command line parameter "SHARED_PATH": "+vg1", "ENABLE_IDX_KEY_LEN_CHECK": "FALSE", "EMPTY_STRING_AS_NULL": "FALSE", "MYSQL_METADATA_IN_CANTIAN": "TRUE", "CT_CLUSTER_STRICT_CHECK": "TRUE", "MYSQL_DEPLOY_GROUP_ID": "5000", - "SQL_STATISTIC_TIME_LIMIT": 1000000 + "SQL_STATISTIC_TIME_LIMIT": 1000000, } - + GSS_CONFIG = { "INTERCONNECT_PORT": "1621", "MAX_SESSION_NUMS": 4096, @@ -164,38 +168,38 @@ class DefaultConfigValue(object): "_LOG_LEVEL": 4294967295, "STORAGE_MODE": "CLUSTER_RAID", "INST_ID": 0, - "LOG_HOME": "", #generate by installer - "INTERCONNECT_ADDR": "", #input by user in command line parameter, same as CANTIAND_CONFIG#INTERCONNECT_ADDR + "LOG_HOME": "", # generate by installer + "INTERCONNECT_ADDR": "", # input by user in command line parameter, same as CANTIAND_CONFIG#INTERCONNECT_ADDR } - + CMS_CONFIG = { "NODE_ID": 0, - "GCC_HOME": "", #generate by installer - "GCC_TYPE": "", #generate by installer + "GCC_HOME": "", # generate by installer + "GCC_TYPE": "", # generate by installer "CMS_LOG": "/opt/cantian/log/cms", "_PORT": 14587, - "_IP": "", #input by user in command line parameter, same as CANTIAND_CONFIG#LSNR_ADDR + "_IP": "", # input by user in command line parameter, same as CANTIAND_CONFIG#LSNR_ADDR "_LOG_LEVEL": 7, "_SPLIT_BRAIN": "TRUE", "_LOG_MAX_FILE_SIZE": "1G", "_DETECT_DISK_TIMEOUT": 6000, "_DISK_DETECT_FILE": "gcc_file,", - "_STOP_RERUN_CMS_SCRIPT" : "/opt/cantian/common/script/cms_reg.sh", + "_STOP_RERUN_CMS_SCRIPT": "/opt/cantian/common/script/cms_reg.sh", "_EXIT_NUM_COUNT_FILE": "/home/cantiandba/data/exit_num.txt", - "_CMS_MES_THREAD_NUM" : "5", - "_CMS_MES_MAX_SESSION_NUM" : "40", - "_CMS_MES_MESSAGE_POOL_COUNT" : "1", - "_CMS_MES_MESSAGE_QUEUE_COUNT" : "1", - "_CMS_MES_MESSAGE_BUFF_COUNT" : "4096", - "_CMS_MES_MESSAGE_CHANNEL_NUM" : "1", - "_CMS_NODE_FAULT_THRESHOLD" : "5", - "_USE_DBSTOR" : "FALSE", - "_DBSTOR_NAMESPACE" : "", - "_CMS_MES_PIPE_TYPE" : "TCP", - "_CMS_MES_CRC_CHECK_SWITCH" : "TRUE", + "_CMS_MES_THREAD_NUM": "5", + "_CMS_MES_MAX_SESSION_NUM": "40", + "_CMS_MES_MESSAGE_POOL_COUNT": "1", + "_CMS_MES_MESSAGE_QUEUE_COUNT": "1", + "_CMS_MES_MESSAGE_BUFF_COUNT": "4096", + "_CMS_MES_MESSAGE_CHANNEL_NUM": "1", + "_CMS_NODE_FAULT_THRESHOLD": "5", + "_USE_DBSTOR": "FALSE", + "_DBSTOR_NAMESPACE": "", + "_CMS_MES_PIPE_TYPE": "TCP", + "_CMS_MES_CRC_CHECK_SWITCH": "TRUE", "SHARED_PATH": "/home/cantiandba/data/data", } - + CANTIAND_DBG_CONFIG = { "DBWR_PROCESSES": 8, "SESSIONS": 8192, @@ -229,7 +233,7 @@ class DefaultConfigValue(object): "CR_POOL_SIZE": "2G", "CR_POOL_COUNT": 4, "VARIANT_MEMORY_AREA_SIZE": "1G", - "REPLAY_PRELOAD_PROCESSES":0, + "REPLAY_PRELOAD_PROCESSES": 0, "LOG_REPLAY_PROCESSES": 64, "_LOG_MAX_FILE_SIZE": "1G", "RECYCLEBIN": "FALSE", @@ -251,43 +255,63 @@ class DefaultConfigValue(object): "REPLACE_PASSWORD_VERIFY": "TRUE", "INTERCONNECT_ADDR": "127.0.0.1", "LSNR_ADDR": "127.0.0.1", - "SHARED_PATH": "", #generate by installer - "LOG_HOME": "", #generate by installer - "_SYS_PASSWORD": "", #input by user in command line parameter or from shell command interactively + "SHARED_PATH": "", # generate by installer + "LOG_HOME": "", # generate by installer + "_SYS_PASSWORD": "", # input by user in command line parameter or from shell command interactively "ENABLE_IDX_KEY_LEN_CHECK": "FALSE", "EMPTY_STRING_AS_NULL": "FALSE", "MYSQL_METADATA_IN_CANTIAN": "TRUE", "CT_CLUSTER_STRICT_CHECK": "TRUE", - "MYSQL_DEPLOY_GROUP_ID": "5000" + "MYSQL_DEPLOY_GROUP_ID": "5000", } class SingleNodeConfig(object): @staticmethod - def get_config(in_container = False): - cantiand_cfg = DefaultConfigValue.CANTIAND_CONFIG if not in_container else DefaultConfigValue.CANTIAND_DBG_CONFIG - return cantiand_cfg, DefaultConfigValue.CMS_CONFIG, DefaultConfigValue.GSS_CONFIG + def get_config(in_container=False): + cantiand_cfg = ( + DefaultConfigValue.CANTIAND_CONFIG + if not in_container + else DefaultConfigValue.CANTIAND_DBG_CONFIG + ) + return ( + cantiand_cfg, + DefaultConfigValue.CMS_CONFIG, + DefaultConfigValue.GSS_CONFIG, + ) class ClusterNode0Config(object): @staticmethod - def get_config(in_container = False): - cantiand_cfg = DefaultConfigValue.CANTIAND_CONFIG if not in_container else DefaultConfigValue.CANTIAND_DBG_CONFIG + def get_config(in_container=False): + cantiand_cfg = ( + DefaultConfigValue.CANTIAND_CONFIG + if not in_container + else DefaultConfigValue.CANTIAND_DBG_CONFIG + ) if in_container: cantiand_cfg["LSNR_ADDR"] = "192.168.86.1" cantiand_cfg["INTERCONNECT_ADDR"] = "192.168.86.1;192.168.86.2" cantiand_cfg["INTERCONNECT_PORT"] = "1601,1602" DefaultConfigValue.GSS_CONFIG["INTERCONNECT_PORT"] = "1621,1621" - return cantiand_cfg, DefaultConfigValue.CMS_CONFIG, DefaultConfigValue.GSS_CONFIG + return ( + cantiand_cfg, + DefaultConfigValue.CMS_CONFIG, + DefaultConfigValue.GSS_CONFIG, + ) class ClusterNode1Config(object): @staticmethod - def get_config(in_container = False): - cantiand_cfg = DefaultConfigValue.CANTIAND_CONFIG if not in_container else DefaultConfigValue.CANTIAND_DBG_CONFIG + def get_config(in_container=False): + cantiand_cfg = ( + DefaultConfigValue.CANTIAND_CONFIG + if not in_container + else DefaultConfigValue.CANTIAND_DBG_CONFIG + ) if in_container: cantiand_cfg["LSNR_ADDR"] = "192.168.86.2" cantiand_cfg["INTERCONNECT_ADDR"] = "192.168.86.1;192.168.86.2" @@ -296,7 +320,11 @@ class ClusterNode1Config(object): DefaultConfigValue.GSS_CONFIG["INST_ID"] = 1 DefaultConfigValue.GSS_CONFIG["INTERCONNECT_PORT"] = "1621,1621" DefaultConfigValue.CMS_CONFIG["NODE_ID"] = 1 - return cantiand_cfg, DefaultConfigValue.CMS_CONFIG, DefaultConfigValue.GSS_CONFIG + return ( + cantiand_cfg, + DefaultConfigValue.CMS_CONFIG, + DefaultConfigValue.GSS_CONFIG, + ) class SshToolException(Exception): @@ -312,8 +340,13 @@ def exec_popen(cmd): :return: status code, standard output, error output """ bash_cmd = ["bash"] - pobj = subprocess.Popen(bash_cmd, shell=False, stdin=subprocess.PIPE, - stdout=subprocess.PIPE, stderr=subprocess.PIPE) + pobj = subprocess.Popen( + bash_cmd, + shell=False, + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + ) if py_verion[0] == "3": stdout, stderr = pobj.communicate(cmd.encode()) @@ -358,43 +391,47 @@ def get_abs_path(_file): def check_path(path_type_in): path_len = len(path_type_in) i = 0 - a_ascii = ord('a') - z_ascii = ord('z') - A_ascii = ord('A') - Z_ascii = ord('Z') - num0_ascii = ord('0') - num9_ascii = ord('9') - blank_ascii = ord(' ') + a_ascii = ord("a") + z_ascii = ord("z") + A_ascii = ord("A") + Z_ascii = ord("Z") + num0_ascii = ord("0") + num9_ascii = ord("9") + blank_ascii = ord(" ") sep1_ascii = ord(os.sep) - sep2_ascii = ord('_') - sep3_ascii = ord(':') - sep4_ascii = ord('-') - sep5_ascii = ord('.') + sep2_ascii = ord("_") + sep3_ascii = ord(":") + sep4_ascii = ord("-") + sep5_ascii = ord(".") CURRENT_OS = platform.system() if CURRENT_OS == "Linux": for i in range(0, path_len): char_check = ord(path_type_in[i]) - if(not (a_ascii <= char_check <= z_ascii - or A_ascii <= char_check <= Z_ascii - or num0_ascii <= char_check <= num9_ascii - or char_check == blank_ascii - or char_check == sep1_ascii - or char_check == sep2_ascii - or char_check == sep4_ascii - or char_check == sep5_ascii)): + if not ( + a_ascii <= char_check <= z_ascii + or A_ascii <= char_check <= Z_ascii + or num0_ascii <= char_check <= num9_ascii + or char_check == blank_ascii + or char_check == sep1_ascii + or char_check == sep2_ascii + or char_check == sep4_ascii + or char_check == sep5_ascii + ): return False elif CURRENT_OS == "Windows": for i in range(0, path_len): char_check = ord(path_type_in[i]) - if(not (a_ascii <= char_check <= z_ascii - or A_ascii <= char_check <= Z_ascii - or num0_ascii <= char_check <= num9_ascii - or char_check == blank_ascii - or char_check == sep1_ascii - or char_check == sep2_ascii - or char_check == sep3_ascii - or char_check == sep4_ascii)): + if not ( + a_ascii <= char_check <= z_ascii + or A_ascii <= char_check <= Z_ascii + or num0_ascii <= char_check <= num9_ascii + or char_check == blank_ascii + or char_check == sep1_ascii + or char_check == sep2_ascii + or char_check == sep3_ascii + or char_check == sep4_ascii + ): return False else: print("Error: Can not support this platform.") @@ -403,10 +440,10 @@ def check_path(path_type_in): def check_ssh_connection(ips): - ''' + """ check ssh connection without password, if success to connect the node user trust to the node has be created - ''' + """ failed_ip = [] success_ip = [] ssh = get_abs_path("ssh") @@ -418,30 +455,39 @@ def check_ssh_connection(ips): cmd += "-o ServerAliveInterval=100 -o ServerAliveCountMax=36 " cmd += "-n 'echo Last login'" process = Execution(cmd) - idx =\ - process.expect(['Permission denied', - 'Last login', - 'Are you sure you want to continue connecting', - 'Password', 'ssh:', TimeoutException, - EOFException], 60) + idx = process.expect( + [ + "Permission denied", + "Last login", + "Are you sure you want to continue connecting", + "Password", + "ssh:", + TimeoutException, + EOFException, + ], + 60, + ) if idx == 0: failed_ip.append(ip) elif idx == 1: success_ip.append(ip) process.sendLine("exit") elif idx == 2: - process.sendLine('yes') - idx = process.expect(['Permission denied', 'Last login', - 'Password', 'ssh:'], 60) + process.sendLine("yes") + idx = process.expect( + ["Permission denied", "Last login", "Password", "ssh:"], 60 + ) if idx == 0: failed_ip.append(ip) elif idx == 1: success_ip.append(ip) process.sendLine("exit") elif idx == 2: - raise Exception("Check ssh connection" - " failed,check your ssh" - " configure file please.") + raise Exception( + "Check ssh connection" + " failed,check your ssh" + " configure file please." + ) elif idx == 3: raise Exception(str(process.context_buffer)) @@ -455,10 +501,12 @@ def check_ssh_connection(ips): # password authentication method, # so we must expect Password key word # to avoid to wait to timeout - raise Exception("Check ssh" - " connection failed," - " check your ssh" - " configure file please.") + raise Exception( + "Check ssh" + " connection failed," + " check your ssh" + " configure file please." + ) elif idx == 4: raise Exception(str(process.context_buffer)) @@ -472,6 +520,7 @@ class CommandTool(object): """ class for CommandTool """ + def __init__(self, log): self.log = log @@ -484,16 +533,16 @@ class CommandTool(object): raise SshToolException("Can't find bash command.") def __execute(self, arg): - ''' + """ execute shell command by ssh to login remote host arg - list for argument, ip address and shell command - ''' + """ ip = arg[0] cmd = arg[1] ssh_options = " -o ServerAliveInterval=100 " ssh_options += " -o ServerAliveCountMax=36 " cmd = "export TMOUT=0; %s" % cmd - ssh_cmd = "ssh %s %s \"%s\"" % (ssh_options, ip, cmd) + ssh_cmd = 'ssh %s %s "%s"' % (ssh_options, ip, cmd) return [ip, exec_popen(ssh_cmd)] def __scp(self, arg): @@ -511,14 +560,14 @@ class CommandTool(object): def __interactive_input(self, process, ip, pw1, pw2): - pw_str = 'Please enter password' + pw_str = "Please enter password" self.log("Expect(%s) on: [%s]" % (ip, pw_str)) - process.expect(['Please enter password']) + process.expect(["Please enter password"]) self.log("Send(%s) password." % ip) process.sendLine(pw1) if pw2: self.log("Expect(%s) on: [%s]" % (ip, pw_str)) - process.expect(['Please enter password']) + process.expect(["Please enter password"]) self.log("Send(%s) password." % ip) process.sendLine(pw2) @@ -544,9 +593,7 @@ class CommandTool(object): if ip: process = Execution("%s %s %s" % (self.ssh, ssh_options, ip)) pdict = user[1] - self.log("ssh session info:\n%s %s %s" % (self.ssh, - ssh_options, - ip)) + self.log("ssh session info:\n%s %s %s" % (self.ssh, ssh_options, ip)) else: process = Execution("%s" % (self.bash)) self.log("bash session") @@ -562,26 +609,26 @@ class CommandTool(object): if user: if instlist: for inst in instlist: - p0 = pdict[str(ip)+"_"+inst][0] - p1 = pdict[str(ip)+"_"+inst][1] + p0 = pdict[str(ip) + "_" + inst][0] + p1 = pdict[str(ip) + "_" + inst][1] self.__interactive_input(process, ip, p0, p1) else: self.__interactive_input(process, ip, user[1], user[2]) self.log("Expect(%s) on: [Done, Upgrade Failed]" % ip) - idx = process.expect(['Done', 'Upgrade Failed'], timeout=51200) + idx = process.expect(["Done", "Upgrade Failed"], timeout=51200) if idx == 0: self.log("Expect(%s) received Done." % ip) - process.sendLine('exit') - return [ip, ('0', str(process.context_before))] + process.sendLine("exit") + return [ip, ("0", str(process.context_before))] self.log("Expect(%s) received Upgrade Failed." % ip) - process.sendLine('exit') - return [ip, ('1', str(process.context_buffer))] + process.sendLine("exit") + return [ip, ("1", str(process.context_buffer))] except (TimeoutException, EOFException) as err: self.log("Expect(%s) timeout." % ip) if process: - process.sendLine('exit') - return [ip, ('1', str(err)+'\n'+str(process.context_buffer))] + process.sendLine("exit") + return [ip, ("1", str(err) + "\n" + str(process.context_buffer))] def execute_local(self, cmd): ret_code, output, errput = exec_popen(cmd) @@ -589,9 +636,9 @@ class CommandTool(object): return ret_code, output def expect_execute(self, ip_cmd_map): - ''' + """ execute shell command with expect - ''' + """ try: pool = Pool(len(ip_cmd_map)) result = pool.map(self.__expect_execute, ip_cmd_map) @@ -600,15 +647,13 @@ class CommandTool(object): raise def execute_in_node(self, ip_cmd_map): - ''' - ''' + """ """ pool = Pool(len(ip_cmd_map)) result = pool.map(self.__execute, ip_cmd_map) return self.__parse(result) def scp_in_node(self, ip_dest_map): - ''' - ''' + """ """ pool = Pool(len(ip_dest_map)) result = pool.map(self.__scp, ip_dest_map) return self.__parse(result) @@ -623,23 +668,22 @@ class CommandTool(object): success_node = [] failed_node = [] for rs in result: - if str(rs[1][0]) != '0': + if str(rs[1][0]) != "0": ret_code = 1 failed_node.append(rs) success_node.append(rs) return ret_code, success_node, failed_node def expect_ctsql(self, ip_cmd_map): - ''' + """ expect execute ctsql and sql command - ''' + """ pool = Pool(len(ip_cmd_map)) result = pool.map(self.__expect_ctsql, ip_cmd_map) return self.__parse(result) def __expect_ctsql(self, arg): - ''' - ''' + """ """ ip = arg[0] ctsql = arg[1] sql = arg[2] @@ -655,21 +699,22 @@ class CommandTool(object): process.sendLine(ctsql) if passwd: - process.expect(['Please enter password']) + process.expect(["Please enter password"]) process.sendLine(passwd) - process.expect(['SQL>']) + process.expect(["SQL>"]) process.sendLine(sql) - idx = process.expect(['rows fetched', 'Succeed', 'CT-', 'SQL>'], - timeout=600) + idx = process.expect( + ["rows fetched", "Succeed", "CT-", "SQL>"], timeout=600 + ) if idx == 0 or idx == 1: - process.sendLine('exit') - return [ip, ('0', str(process.context_before))] - process.sendLine('exit') - return [ip, '1', str(process.context_buffer)] + process.sendLine("exit") + return [ip, ("0", str(process.context_before))] + process.sendLine("exit") + return [ip, "1", str(process.context_buffer)] except (TimeoutException, EOFException): if process: - process.sendLine('exit') - return [ip, ('1', str(process.context_buffer))] + process.sendLine("exit") + return [ip, ("1", str(process.context_buffer))] class ExpectException(Exception): @@ -694,13 +739,16 @@ class Execution(object): if py_verion[0] == "3": ALLOWED_STRING_TYPES = (str,) else: - ALLOWED_STRING_TYPES = (type(b''), type(''), type(u''),) + ALLOWED_STRING_TYPES = ( + type(b""), + type(""), + type(""), + ) LINE_SEPERATOR = os.linesep - CTRLF = '\r\n' + CTRLF = "\r\n" - def __init__(self, command, timeout=1800, maxReadSize=4096, - delimiter=None): + def __init__(self, command, timeout=1800, maxReadSize=4096, delimiter=None): self.matcher = None self.context_before = None @@ -724,11 +772,13 @@ class Execution(object): try: from termios import CEOF from termios import CINTR + (self._INTR, self._EOF) = (CINTR, CEOF) except ImportError: try: from termios import VEOF from termios import VINTR + fp = sys.__stdin__.fileno() self._INTR = ord(termios.tcgetattr(fp)[6][VINTR]) self._EOF = ord(termios.tcgetattr(fp)[6][VEOF]) @@ -739,13 +789,13 @@ class Execution(object): @staticmethod def _ascii(content): if not isinstance(content, bytes): - return content.encode('ascii') + return content.encode("ascii") return content @staticmethod def _utf8(content): if not isinstance(content, bytes): - return content.encode('utf-8') + return content.encode("utf-8") return content def __del__(self): @@ -757,29 +807,29 @@ class Execution(object): def __str__(self): s = list() - s.append('%r' % self) - s.append('after: %r' % self.context_after) - s.append('pid: %s' % str(self.childPid)) - s.append('child_fd: %s' % str(self.childFd)) - s.append('closed: %s' % str(self.is_closed)) - s.append('timeout: %s' % str(self.timeout)) - s.append('delimiter: %s' % str(self.delimiter)) - s.append('maxReadSize: %s' % str(self.maxReadSize)) - return '\n'.join(s) + s.append("%r" % self) + s.append("after: %r" % self.context_after) + s.append("pid: %s" % str(self.childPid)) + s.append("child_fd: %s" % str(self.childFd)) + s.append("closed: %s" % str(self.is_closed)) + s.append("timeout: %s" % str(self.timeout)) + s.append("delimiter: %s" % str(self.delimiter)) + s.append("maxReadSize: %s" % str(self.maxReadSize)) + return "\n".join(s) def _excute(self, command): self.args = shlex.split(command) if self.childPid is not None: - raise ExpectException('The pid member must be None.') + raise ExpectException("The pid member must be None.") if self.command is None: - raise ExpectException('The command member must not be None.') + raise ExpectException("The command member must not be None.") try: self.childPid, self.childFd = pty.fork() except OSError as err: # pragma: no cover - raise ExpectException('pty.fork() failed: ' + str(err)) + raise ExpectException("pty.fork() failed: " + str(err)) if self.childPid == pty.CHILD: # child @@ -819,19 +869,18 @@ class Execution(object): # give kernel time to update process status. time.sleep(self.closeDelay) if self.isAlive() and not self.terminate(): - raise ExpectException('Could not terminate the child.') + raise ExpectException("Could not terminate the child.") self.childFd = -1 self.is_closed = True def setEcho(self, state): - err_msg = ('method setEcho() may not be available on' - ' this operating system.') + err_msg = "method setEcho() may not be available on" " this operating system." try: child_attr = termios.tcgetattr(self.childFd) except termios.error as e: if e.args[0] == errno.EINVAL: - raise IOError(e.args[0], '%s: %s.' % (e.args[1], err_msg)) + raise IOError(e.args[0], "%s: %s." % (e.args[1], err_msg)) raise if state: @@ -843,12 +892,12 @@ class Execution(object): termios.tcsetattr(self.childFd, termios.TCSANOW, child_attr) except IOError as e: if e.args[0] == errno.EINVAL: - raise IOError(e.args[0], '%s: %s.' % (e.args[1], err_msg)) + raise IOError(e.args[0], "%s: %s." % (e.args[1], err_msg)) raise def readNonBlock(self, size=1, timeout=-1): if self.is_closed: - raise ValueError('I/O operation on closed file.') + raise ValueError("I/O operation on closed file.") if timeout == -1: timeout = self.timeout @@ -858,16 +907,16 @@ class Execution(object): rfds, _, _ = self.__select([self.childFd], [], [], 0) if not rfds: self.eofFlag = True - raise EOFException('End Of File (EOF). Braindead platform.') + raise EOFException("End Of File (EOF). Braindead platform.") rfds, _, _ = self.__select([self.childFd], [], [], timeout) if not rfds: if not self.isAlive(): self.eofFlag = True - raise EOFException('Reach end of File (EOF).') + raise EOFException("Reach end of File (EOF).") else: - raise TimeoutException('Timeout exceeded.') + raise TimeoutException("Timeout exceeded.") if self.childFd in rfds: try: @@ -875,17 +924,19 @@ class Execution(object): except OSError as e: if e.args[0] == errno.EIO: self.eofFlag = True - raise EOFException('Reach End Of File (EOF). ' - 'Exception style platform.') + raise EOFException( + "Reach End Of File (EOF). " "Exception style platform." + ) raise - if child_data == b'': + if child_data == b"": self.eofFlag = True - raise EOFException('Reach end Of File (EOF).' - ' Empty string style platform.') + raise EOFException( + "Reach end Of File (EOF)." " Empty string style platform." + ) return child_data - raise ExpectException('Reached an unexpected state.') + raise ExpectException("Reached an unexpected state.") # pragma: no cover def read(self, size=-1, timeout=-1): @@ -902,7 +953,7 @@ class Execution(object): def _send(self, content): return os.write(self.childFd, content) - def sendLine(self, content=''): + def sendLine(self, content=""): sendCount = self.send(content) sendCount = sendCount + self.send(self.LINE_SEPERATOR) return sendCount @@ -944,18 +995,17 @@ class Execution(object): except OSError as e: # No child processes if e.errno == errno.ECHILD: - raise ExpectException('process already not exist.') + raise ExpectException("process already not exist.") else: raise e if childPid == 0: try: - childPid, childStatus = os.waitpid(self.childPid, - waitpidOptions) + childPid, childStatus = os.waitpid(self.childPid, waitpidOptions) except OSError as err: # pragma: no cover if err.errno == errno.ECHILD: - raise ExpectException('process already not exist.') + raise ExpectException("process already not exist.") else: raise @@ -968,7 +1018,7 @@ class Execution(object): if os.WIFEXITED(childStatus) or os.WIFSIGNALED(childStatus): self.is_terminated = True elif os.WIFSTOPPED(childStatus): - raise ExpectException('process already been stopped.') + raise ExpectException("process already been stopped.") return False @@ -985,10 +1035,13 @@ class Execution(object): def raisePatternTypeError(self, pattern): raise TypeError( - 'got %s as pattern, must be one' - ' of: %s, pexpect.EOFException, pexpect.TIMEOUTException' - % (type(pattern), ', '.join([str(ast) for ast in - self.ALLOWED_STRING_TYPES]))) + "got %s as pattern, must be one" + " of: %s, pexpect.EOFException, pexpect.TIMEOUTException" + % ( + type(pattern), + ", ".join([str(ast) for ast in self.ALLOWED_STRING_TYPES]), + ) + ) def compilePatternList(self, pattern_list): if not pattern_list: @@ -1005,7 +1058,7 @@ class Execution(object): patternList.append(EOFException) elif pattern is TimeoutException: patternList.append(TimeoutException) - elif isinstance(pattern, type(re.compile(''))): + elif isinstance(pattern, type(re.compile(""))): patternList.append(pattern) else: self.raisePatternTypeError(pattern) @@ -1030,16 +1083,17 @@ class Execution(object): while True: matchIndex = re_searcher.search(context_buffer) if matchIndex > -1: - self.context_buffer = context_buffer[re_searcher.end:] + self.context_buffer = context_buffer[re_searcher.end :] self.context_before = context_buffer[: re_searcher.start] - self.context_after = context_buffer[re_searcher.start: - re_searcher.end] + self.context_after = context_buffer[ + re_searcher.start : re_searcher.end + ] self.context_match = re_searcher.context_match self.matchIndex = matchIndex return self.matchIndex # no match at this point if (timeout is not None) and (timeout < 0): - raise TimeoutException('Timeout exceeded in loopExpect().') + raise TimeoutException("Timeout exceeded in loopExpect().") # not timed out, continue read more_context = self.readNonBlock(self.maxReadSize, timeout) time.sleep(0.0001) @@ -1080,8 +1134,8 @@ class Execution(object): raise def setWinSize(self, rows, cols): - win_size = getattr(termios, 'TIOCSWINSZ', -2146929561) - s_size = struct.pack('HHHH', rows, cols, 0, 0) + win_size = getattr(termios, "TIOCSWINSZ", -2146929561) + s_size = struct.pack("HHHH", rows, cols, 0, 0) fcntl.ioctl(self.fileno(), win_size, s_size) def __select(self, inputs, outputs, errputs, timeout=None): @@ -1095,7 +1149,7 @@ class Execution(object): if timeout is not None: timeout = endTime - time.time() if timeout < 0: - return([], [], []) + return ([], [], []) else: raise @@ -1108,8 +1162,7 @@ class RESearcher(object): self.start = None self.context_match = None self.end = None - for index, pattern_item in zip(list(range(len(pattern_list))), - pattern_list): + for index, pattern_item in zip(list(range(len(pattern_list))), pattern_list): if pattern_item is EOFException: self.eof_index = index continue @@ -1122,21 +1175,23 @@ class RESearcher(object): result_list = list() for index, pattern_item in self._searches: try: - result_list.append((index, ' %d: re.compile("%s")' % - (index, pattern_item.pattern))) + result_list.append( + (index, ' %d: re.compile("%s")' % (index, pattern_item.pattern)) + ) except UnicodeEncodeError: - result_list.append((index, ' %d: re.compile(%r)' % - (index, pattern_item.pattern))) - result_list.append((-1, 'RESearcher:')) + result_list.append( + (index, " %d: re.compile(%r)" % (index, pattern_item.pattern)) + ) + result_list.append((-1, "RESearcher:")) if self.eof_index >= 0: - result_list.append((self.eof_index, ' %d: EOF' % - self.eof_index)) + result_list.append((self.eof_index, " %d: EOF" % self.eof_index)) if self.timeout_index >= 0: - result_list.append((self.timeout_index, ' %d: TIMEOUT' % - self.timeout_index)) + result_list.append( + (self.timeout_index, " %d: TIMEOUT" % self.timeout_index) + ) result_list.sort() s_result_list = list(zip(*result_list))[1] - return '\n'.join(s_result_list) + return "\n".join(s_result_list) def search(self, content): first_match_index = None diff --git a/pkg/install/install.py b/pkg/install/install.py index c3ed8e3d..c9b50d71 100644 --- a/pkg/install/install.py +++ b/pkg/install/install.py @@ -5,6 +5,7 @@ import sys + sys.dont_write_bytecode = True try: @@ -26,7 +27,13 @@ try: import json import threading import signal - from funclib import CommonValue, SingleNodeConfig, ClusterNode0Config, ClusterNode1Config, DefaultConfigValue + from funclib import ( + CommonValue, + SingleNodeConfig, + ClusterNode0Config, + ClusterNode1Config, + DefaultConfigValue, + ) import argparse PYTHON242 = "2.4.2" @@ -38,8 +45,7 @@ try: elif gPyVersion >= PYTHON25: import hashlib else: - print("This install script can not support python version: %s" - % gPyVersion) + print("This install script can not support python version: %s" % gPyVersion) sys.exit(1) sys.path.append(os.path.split(os.path.realpath(__file__))[0]) @@ -52,18 +58,31 @@ CURRENT_OS = platform.system() CANTIAND = "cantiand" CANTIAND_WITH_MYSQL = "cantiand_with_mysql" -CANTIAND_WITH_MYSQL_ST = "cantiand_with_mysql_st" #single process mode with mysql llt +CANTIAND_WITH_MYSQL_ST = "cantiand_with_mysql_st" # single process mode with mysql llt CANTIAND_IN_CLUSTER = "cantiand_in_cluster" CANTIAND_WITH_MYSQL_IN_CLUSTER = "cantiand_with_mysql_in_cluster" MYSQLD = "mysqld" -INSTALL_SCRIPT = os.path.join(os.path.dirname(os.path.abspath(__file__)), "installdb.sh") +INSTALL_SCRIPT = os.path.join( + os.path.dirname(os.path.abspath(__file__)), "installdb.sh" +) -VALID_RUNNING_MODE = {CANTIAND, CANTIAND_WITH_MYSQL, CANTIAND_WITH_MYSQL_ST, CANTIAND_IN_CLUSTER, CANTIAND_WITH_MYSQL_IN_CLUSTER, MYSQLD} +VALID_RUNNING_MODE = { + CANTIAND, + CANTIAND_WITH_MYSQL, + CANTIAND_WITH_MYSQL_ST, + CANTIAND_IN_CLUSTER, + CANTIAND_WITH_MYSQL_IN_CLUSTER, + MYSQLD, +} -VALID_SINGLE_MYSQL_RUNNING_MODE = {CANTIAND_WITH_MYSQL_IN_CLUSTER, CANTIAND_WITH_MYSQL_ST, CANTIAND_WITH_MYSQL} +VALID_SINGLE_MYSQL_RUNNING_MODE = { + CANTIAND_WITH_MYSQL_IN_CLUSTER, + CANTIAND_WITH_MYSQL_ST, + CANTIAND_WITH_MYSQL, +} -CLUSTER_SIZE = 2 # default to 2, 4 node cluster mode need add parameter to specify this +CLUSTER_SIZE = 2 # default to 2, 4 node cluster mode need add parameter to specify this INSTALL_SCPRIT_DIR = os.path.dirname(os.path.abspath(__file__)) PKG_DIR = os.path.abspath(os.path.join(INSTALL_SCPRIT_DIR, "../..")) @@ -81,13 +100,17 @@ MYSQL_BIN_DIR = "/usr/local/mysql" CTC_LIB_DIR = "/opt/cantian/mysql/server/" MYSQL_LOG_FILE = os.path.join(MYSQL_DATA_DIR, "mysql.log") -MYSQL_LIB_OUTPUT_DIR = os.path.join(PKG_DIR, "cantian-connector-mysql/mysql-source/bld_debug/plugin_output_directory") +MYSQL_LIB_OUTPUT_DIR = os.path.join( + PKG_DIR, "cantian-connector-mysql/mysql-source/bld_debug/plugin_output_directory" +) CTC_PLUGIN = "ha_ctc.so" + class Options(object): """ command line options """ + def __init__(self): self.log_file = "" self.install_user_privilege = "" @@ -114,10 +137,10 @@ class Options(object): # flag indicate user using gss storage self.use_gss = False - + # flag indicate user using dbstor storage self.use_dbstor = False - + self.link_type = "TCP" self.link_type_from_para = False @@ -138,23 +161,31 @@ class Options(object): # In slave cluster: self.slave_cluster = False - + # is re-launch mysql server self.is_relaunch = False + g_opts = Options() + def check_directories(): global MYSQL_CODE_DIR global MYSQL_VERSION - global MYSQL_DATA_DIR + global MYSQL_DATA_DIR global MYSQL_BIN_DIR global CONFIG_FILE global MYSQL_LOG_FILE if os.path.exists("/.dockerenv") and (g_opts.cantian_in_container != "1"): MYSQL_CODE_DIR = os.path.join(PKG_DIR, "cantian-connector-mysql/mysql-source") - is_mysql_metadata_in_cantian = (DefaultConfigValue.CANTIAND_CONFIG["MYSQL_METADATA_IN_CANTIAN"] == "TRUE") - MYSQL_VERSION = VERSION_DOCKER_META if is_mysql_metadata_in_cantian else VERSION_DOCKER_NOMETA + is_mysql_metadata_in_cantian = ( + DefaultConfigValue.CANTIAND_CONFIG["MYSQL_METADATA_IN_CANTIAN"] == "TRUE" + ) + MYSQL_VERSION = ( + VERSION_DOCKER_META + if is_mysql_metadata_in_cantian + else VERSION_DOCKER_NOMETA + ) MYSQL_DATA_DIR = "/data/data" MYSQL_BIN_DIR = "/usr/local/mysql" CTC_LIB_DIR = "/home/regress/cantian-connector-mysql/mysql-source/" @@ -165,13 +196,17 @@ def check_directories(): _tmp = conf.read() info = json.loads(_tmp) is_mysql_metadata_in_cantian = info.get("mysql_metadata_in_cantian") - MYSQL_VERSION = VERSION_ENV_META if is_mysql_metadata_in_cantian else VERSION_ENV_NOMETA - MYSQL_DATA_DIR = "/mnt/dbdata/remote/metadata_{metadata}/node{node}"\ - .format(metadata=info.get("storage_metadata_fs", ""), node=info["node_id"]) + MYSQL_VERSION = ( + VERSION_ENV_META if is_mysql_metadata_in_cantian else VERSION_ENV_NOMETA + ) + MYSQL_DATA_DIR = "/mnt/dbdata/remote/metadata_{metadata}/node{node}".format( + metadata=info.get("storage_metadata_fs", ""), node=info["node_id"] + ) MYSQL_BIN_DIR = "/opt/cantian/mysql/install/mysql" MYSQL_LOG_FILE = os.path.join(MYSQL_DATA_DIR, "mysql.log") + def check_kernel_parameter(para): """Is kernel parameter invalid?""" pattern = re.compile("^[A-Z_][A-Z0-9_]+$") @@ -186,10 +221,10 @@ def check_invalid_symbol(para): :param para: parameter's value :return: NA """ - symbols = ["|", "&", "$", ">", "<", "\"", "'", "`"] + symbols = ["|", "&", "$", ">", "<", '"', "'", "`"] for symbol in symbols: if para.find(symbol) > -1: - print("There is invalid symbol \"%s\" in %s" % (symbol, para)) + print('There is invalid symbol "%s" in %s' % (symbol, para)) sys.exit(1) @@ -201,7 +236,7 @@ def all_zero_addr_after_ping(nodeIp): """ if not nodeIp: return False - allowed_chars = set('0:.') + allowed_chars = set("0:.") if set(nodeIp).issubset(allowed_chars): return True else: @@ -215,64 +250,92 @@ def checkPath(path_type_in): :return: weather validity """ path_len = len(path_type_in) - a_ascii = ord('a') - z_ascii = ord('z') - A_ascii = ord('A') - Z_ascii = ord('Z') - num0_ascii = ord('0') - num9_ascii = ord('9') - blank_ascii = ord(' ') + a_ascii = ord("a") + z_ascii = ord("z") + A_ascii = ord("A") + Z_ascii = ord("Z") + num0_ascii = ord("0") + num9_ascii = ord("9") + blank_ascii = ord(" ") sep1_ascii = ord(os.sep) - sep2_ascii = ord('_') - sep3_ascii = ord(':') - sep4_ascii = ord('-') - sep5_ascii = ord('.') - char_check_list1 = [blank_ascii, - sep1_ascii, - sep2_ascii, - sep4_ascii, - sep5_ascii - ] - - char_check_list2 = [blank_ascii, - sep1_ascii, - sep2_ascii, - sep3_ascii, - sep4_ascii - ] + sep2_ascii = ord("_") + sep3_ascii = ord(":") + sep4_ascii = ord("-") + sep5_ascii = ord(".") + char_check_list1 = [blank_ascii, sep1_ascii, sep2_ascii, sep4_ascii, sep5_ascii] + + char_check_list2 = [blank_ascii, sep1_ascii, sep2_ascii, sep3_ascii, sep4_ascii] if CURRENT_OS == "Linux": - return checkPathLinux(path_len, path_type_in, a_ascii, z_ascii, - A_ascii, Z_ascii, num0_ascii, num9_ascii, - char_check_list1) + return checkPathLinux( + path_len, + path_type_in, + a_ascii, + z_ascii, + A_ascii, + Z_ascii, + num0_ascii, + num9_ascii, + char_check_list1, + ) elif CURRENT_OS == "Windows": - return checkPathWindows(path_len, path_type_in, a_ascii, z_ascii, - A_ascii, Z_ascii, num0_ascii, num9_ascii, - char_check_list2) + return checkPathWindows( + path_len, + path_type_in, + a_ascii, + z_ascii, + A_ascii, + Z_ascii, + num0_ascii, + num9_ascii, + char_check_list2, + ) else: print("Error: Can not support this platform.") sys.exit(1) -def checkPathLinux(path_len, path_type_in, a_ascii, z_ascii, - A_ascii, Z_ascii, num0_ascii, num9_ascii, char_check_list): +def checkPathLinux( + path_len, + path_type_in, + a_ascii, + z_ascii, + A_ascii, + Z_ascii, + num0_ascii, + num9_ascii, + char_check_list, +): for i in range(0, path_len): char_check = ord(path_type_in[i]) - if (not (a_ascii <= char_check <= z_ascii - or A_ascii <= char_check <= Z_ascii - or num0_ascii <= char_check <= num9_ascii - or char_check in char_check_list)): + if not ( + a_ascii <= char_check <= z_ascii + or A_ascii <= char_check <= Z_ascii + or num0_ascii <= char_check <= num9_ascii + or char_check in char_check_list + ): return False return True -def checkPathWindows(path_len, path_type_in, a_ascii, z_ascii, A_ascii, - Z_ascii, num0_ascii, num9_ascii, char_check_list): +def checkPathWindows( + path_len, + path_type_in, + a_ascii, + z_ascii, + A_ascii, + Z_ascii, + num0_ascii, + num9_ascii, + char_check_list, +): for i in range(0, path_len): char_check = ord(path_type_in[i]) - if (not (a_ascii <= char_check <= z_ascii - or A_ascii <= char_check <= Z_ascii - or num0_ascii <= char_check <= num9_ascii - or char_check in char_check_list)): + if not ( + a_ascii <= char_check <= z_ascii + or A_ascii <= char_check <= Z_ascii + or num0_ascii <= char_check <= num9_ascii + or char_check in char_check_list + ): return False return True @@ -286,8 +349,13 @@ def _exec_popen(cmd, values=None): if not values: values = [] bash_cmd = ["bash"] - pobj = subprocess.Popen(bash_cmd, shell=False, stdin=subprocess.PIPE, - stdout=subprocess.PIPE, stderr=subprocess.PIPE) + pobj = subprocess.Popen( + bash_cmd, + shell=False, + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + ) if gPyVersion[0] == "3": pobj.stdin.write(cmd.encode()) @@ -350,22 +418,29 @@ def check_runner(): if owner_uid == 0: if runner_uid != 0: runner = pwd.getpwuid(runner_uid).pw_name - print("Error: The owner of install.py has root privilege," - " can't run it by user [%s]." % runner) + print( + "Error: The owner of install.py has root privilege," + " can't run it by user [%s]." % runner + ) sys.exit(1) else: if runner_uid == 0: owner = pwd.getpwuid(owner_uid).pw_name - print("Error: The owner of install.py is [%s]," - " can't run it by root." % owner) + print( + "Error: The owner of install.py is [%s]," + " can't run it by root." % owner + ) sys.exit(1) elif runner_uid != owner_uid: runner = pwd.getpwuid(runner_uid).pw_name owner = pwd.getpwuid(owner_uid).pw_name - print("Error: The owner of install.py [%s] is different" - " with the executor [%s]." % (owner, runner)) + print( + "Error: The owner of install.py [%s] is different" + " with the executor [%s]." % (owner, runner) + ) sys.exit(1) + def persist_environment_variable(var_name, var_value, config_file=None): """ 将环境变量持久化到指定的配置文件中。 @@ -380,7 +455,7 @@ def persist_environment_variable(var_name, var_value, config_file=None): # 读取现有的配置文件内容 try: - with open(config_file, 'r') as file: + with open(config_file, "r") as file: lines = file.readlines() except FileNotFoundError: lines = [] @@ -398,7 +473,7 @@ def persist_environment_variable(var_name, var_value, config_file=None): lines.append(f"\nexport {var_name}={var_value}\n") # 写回配置文件 - with open(config_file, 'w') as file: + with open(config_file, "w") as file: file.writelines(lines) # 使更改生效 @@ -406,14 +481,15 @@ def persist_environment_variable(var_name, var_value, config_file=None): print(f"{var_name} set to {var_value} and made persistent.") + def get_random_num(lower_num, upper_num): # range of values differ_num = upper_num - lower_num + 1 # get the first ten rows of random numbers cmd = "cat /dev/random | head -n 10 | cksum | awk -F ' ' '{print $1}'" - p = subprocess.Popen(["bash", "-c", cmd], shell=False, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE) + p = subprocess.Popen( + ["bash", "-c", cmd], shell=False, stdout=subprocess.PIPE, stderr=subprocess.PIPE + ) # format result to string result_num = p.stdout.read().strip().decode() # string to int @@ -454,53 +530,53 @@ def generate_password(): def usage(): """install.py is a utility to install cantiand server. -Usage: - python install.py --help - python install.py -U user:group -R installpath -M cantiand -N 0 -D DATADIR [-O] [-c] - [-C PARAMETER=VALUE] [-g withoutroot] [-f SQLFILE] [-d] [-p] - python install.py -U user:group -R installpath -M cantiand_in_cluster -N 0 -D DATADIR [-O] [-c] - [-C 'PARAMETER=VALUE'] [-g withoutroot] [-f SQLFILE] [-d] [-p] - python install.py -U user:group -R installpath -M cantiand_with_mysql -N 0 -D DATADIR [-O] [-c] - [-C 'PARAMETER=VALUE'] [-g withoutroot] [-f SQLFILE] [-d] [-p] - python install.py -U user:group -R installpath -M cantiand_in_cluster -N 1 -D DATADIR [-O] [-c] - [-C \"PARAMETER=VALUE\"] [-g withoutroot] [-f SQLFILE] [-d] [-p] - -Common options: - -U the database program and cluster owner - -R the database program path - -M the database running mode, case insensitive, default value [cantiand] - cantiand: running cantiand in single mode; - cantiand_with_mysql: running cantiand with mysql connector in single mode; - cantiand_with_mysql_st: running cantiand(mysql llt) with mysql connector in single mode; - cantiand_in_cluster: running cantiand in cluster mode; - cantiand_with_mysql_in_cluster: running cantiand with mysql connector in cluster mode. - -N node id, value is [0, 1] - -O only install database program, and omit other optional parameters - -D location of the database cluster storage area - -g run install script without root privilege, - but you must have permission of installation folder - note: use \"-g withoutroot\" exactly - -Z configure the database cluster config, eg: -Z "LSNR_PORT=1611", - for more detail information see documentation. - -C configure the database cluster cms config, eg: -C "GCC_HOME=/dev/cms-disk1", - for more detail information see documentation. - -G configure the database cluster gss config, eg: -G "STORAGE_MODE=CLUSTER_RAID", - for more detail information see documentation. - -W configure the database ip white list, eg: -W "127.0.0.1". - -c not use SSL-based secure connections - -s using gss as storage, default using file system - -l specify the cantiand install log file path and name, - if not, use the default - disable it(not recommended) - -P Compatibility parameter, which does not take effect. - -f specify a customized create database sql file. - if not, use default create database sql file. - -d install inside docker container - -p if ignore pkg version check - -m assign the config file of mysql - --help show this help, then exit - -If all the optional parameters are not specified, -O option will be used. + Usage: + python install.py --help + python install.py -U user:group -R installpath -M cantiand -N 0 -D DATADIR [-O] [-c] + [-C PARAMETER=VALUE] [-g withoutroot] [-f SQLFILE] [-d] [-p] + python install.py -U user:group -R installpath -M cantiand_in_cluster -N 0 -D DATADIR [-O] [-c] + [-C 'PARAMETER=VALUE'] [-g withoutroot] [-f SQLFILE] [-d] [-p] + python install.py -U user:group -R installpath -M cantiand_with_mysql -N 0 -D DATADIR [-O] [-c] + [-C 'PARAMETER=VALUE'] [-g withoutroot] [-f SQLFILE] [-d] [-p] + python install.py -U user:group -R installpath -M cantiand_in_cluster -N 1 -D DATADIR [-O] [-c] + [-C \"PARAMETER=VALUE\"] [-g withoutroot] [-f SQLFILE] [-d] [-p] + + Common options: + -U the database program and cluster owner + -R the database program path + -M the database running mode, case insensitive, default value [cantiand] + cantiand: running cantiand in single mode; + cantiand_with_mysql: running cantiand with mysql connector in single mode; + cantiand_with_mysql_st: running cantiand(mysql llt) with mysql connector in single mode; + cantiand_in_cluster: running cantiand in cluster mode; + cantiand_with_mysql_in_cluster: running cantiand with mysql connector in cluster mode. + -N node id, value is [0, 1] + -O only install database program, and omit other optional parameters + -D location of the database cluster storage area + -g run install script without root privilege, + but you must have permission of installation folder + note: use \"-g withoutroot\" exactly + -Z configure the database cluster config, eg: -Z "LSNR_PORT=1611", + for more detail information see documentation. + -C configure the database cluster cms config, eg: -C "GCC_HOME=/dev/cms-disk1", + for more detail information see documentation. + -G configure the database cluster gss config, eg: -G "STORAGE_MODE=CLUSTER_RAID", + for more detail information see documentation. + -W configure the database ip white list, eg: -W "127.0.0.1". + -c not use SSL-based secure connections + -s using gss as storage, default using file system + -l specify the cantiand install log file path and name, + if not, use the default + disable it(not recommended) + -P Compatibility parameter, which does not take effect. + -f specify a customized create database sql file. + if not, use default create database sql file. + -d install inside docker container + -p if ignore pkg version check + -m assign the config file of mysql + --help show this help, then exit + + If all the optional parameters are not specified, -O option will be used. """ print(usage.__doc__) @@ -523,19 +599,22 @@ def check_user(user, group): print("Parameter input error: -U, the user does not match the group.") sys.exit(1) elif user == "root" or user_.pw_uid == 0: - print("Parameter input error: -U, can not install program to" - " root user.") + print("Parameter input error: -U, can not install program to" " root user.") sys.exit(1) elif group == "root" or user_.pw_gid == 0: - print("Parameter input error: -U, can not install program to" - " user with root privilege.") + print( + "Parameter input error: -U, can not install program to" + " user with root privilege." + ) sys.exit(1) runner_uid = os.getuid() if runner_uid != 0 and runner_uid != user_.pw_uid: runner = pwd.getpwuid(runner_uid).pw_name - print("Parameter input error: -U, has to be the same as the" - " executor [%s]" % runner) + print( + "Parameter input error: -U, has to be the same as the" + " executor [%s]" % runner + ) sys.exit(1) @@ -564,8 +643,11 @@ def parse_parameter(): try: # Parameters are passed into argv. After parsing, they are stored # in opts as binary tuples. Unresolved parameters are stored in args. - opts, args = getopt.getopt(sys.argv[1:], - "U:R:M:N:OD:Z:C:G:W:cg:sdl:Ppf:m:S:r", ["help", "dbstor", "linktype="]) + opts, args = getopt.getopt( + sys.argv[1:], + "U:R:M:N:OD:Z:C:G:W:cg:sdl:Ppf:m:S:r", + ["help", "dbstor", "linktype="], + ) if args: print("Parameter input error: " + str(args[0])) exit(1) @@ -643,7 +725,7 @@ def is_mlnx(): log("exec ofed_info cmd failed") return False - if 'MLNX_OFED_LINUX-5.5' in stdout: + if "MLNX_OFED_LINUX-5.5" in stdout: log("Is mlnx 5.5") return True @@ -685,8 +767,10 @@ def check_parameter(): """ if g_opts.install_user_privilege != "withoutroot": if os.getuid(): - print("Error: User has no root privilege, do install," - " need specify parameter '-g withoutroot'.") + print( + "Error: User has no root privilege, do install," + " need specify parameter '-g withoutroot'." + ) sys.exit(1) # Check the log path. check_log_path() @@ -706,17 +790,30 @@ def check_parameter(): print(str(ex)) sys.exit(1) # Check running mode - if len(g_opts.running_mode) == 0 or g_opts.running_mode.lower() not in VALID_RUNNING_MODE: + if ( + len(g_opts.running_mode) == 0 + or g_opts.running_mode.lower() not in VALID_RUNNING_MODE + ): print("Invalid running mode: " + g_opts.running_mode) sys.exit(1) - if g_opts.node_id not in [0 ,1]: + if g_opts.node_id not in [0, 1]: print("Invalid node id: " + g_opts.node_id) sys.exit(1) - if g_opts.running_mode.lower() in [CANTIAND, CANTIAND_WITH_MYSQL, CANTIAND_WITH_MYSQL_ST] and g_opts.node_id == 1: - print("Invalid node id: " + g_opts.node_id + ", this node id can only run in cluster mode") + if ( + g_opts.running_mode.lower() + in [CANTIAND, CANTIAND_WITH_MYSQL, CANTIAND_WITH_MYSQL_ST] + and g_opts.node_id == 1 + ): + print( + "Invalid node id: " + + g_opts.node_id + + ", this node id can only run in cluster mode" + ) sys.exit(1) # Check docker option - if (g_opts.in_container ^ os.path.exists("/.dockerenv")) and (g_opts.cantian_in_container != "1"): + if (g_opts.in_container ^ os.path.exists("/.dockerenv")) and ( + g_opts.cantian_in_container != "1" + ): print("Wrong docker container env option of -d") sys.exit(1) @@ -749,12 +846,16 @@ def check_log_path(): if not os.path.isdir(dir_path): g_opts.log_file = "" - print("Specified log path: \"%s\" does not exist, " - "choose the default path instead." % dir_path) + print( + 'Specified log path: "%s" does not exist, ' + "choose the default path instead." % dir_path + ) elif not base_name: g_opts.log_file = "" - print("Log file does not been specified, " - "choose the default logfile instead.") + print( + "Log file does not been specified, " + "choose the default logfile instead." + ) def use_default_log_path(): @@ -770,8 +871,9 @@ def use_default_log_path(): if not os.path.exists(os.path.realpath(stdout)): print("Cant get the home path of current user.") sys.exit(1) - g_opts.log_file = os.path.join(os.path.realpath(os.path.normpath(stdout)), - "cantiandinstall.log") + g_opts.log_file = os.path.join( + os.path.realpath(os.path.normpath(stdout)), "cantiandinstall.log" + ) def log(msg, is_screen=False): @@ -796,8 +898,10 @@ def logExit(msg): """ log("Error: " + msg) print("Error: " + msg) - print("Please refer to install log \"%s\" for more detailed information." - % g_opts.log_file) + print( + 'Please refer to install log "%s" for more detailed information.' + % g_opts.log_file + ) sys.exit(1) @@ -805,32 +909,46 @@ class Platform(object): """ get dist name/version/id from /etc/*release """ + def __init__(self): pass - SUPPORTED_DISTS = ('suse', 'debian', 'fedora', 'redhat', 'centos', - 'mandrake', 'mandriva', 'rocks', 'slackware', - 'yellowdog', 'gentoo', 'unitedlinux', - 'turbolinux', 'arch', 'mageia', 'openeuler', - 'neokylin', 'euleros', 'kylin') - UNIXCONFDIR = '/etc' + SUPPORTED_DISTS = ( + "suse", + "debian", + "fedora", + "redhat", + "centos", + "mandrake", + "mandriva", + "rocks", + "slackware", + "yellowdog", + "gentoo", + "unitedlinux", + "turbolinux", + "arch", + "mageia", + "openeuler", + "neokylin", + "euleros", + "kylin", + ) + UNIXCONFDIR = "/etc" @staticmethod - def _parse_release_file(firstline, version='', dst_id=''): + def _parse_release_file(firstline, version="", dst_id=""): """ function: parse first line of /etc/*release input: string output: tuple(string, string, string) """ - lsb_release_version_re = r'(.+) release ([\d.]+)[^(]*(?:\((.+)\))?' - release_version_re = (r'([^0-9]+)(?: release )?([\d.]+)[^(]*' - r'(?:\((.+)\))?') + lsb_release_version_re = r"(.+) release ([\d.]+)[^(]*(?:\((.+)\))?" + release_version_re = r"([^0-9]+)(?: release )?([\d.]+)[^(]*" r"(?:\((.+)\))?" try: - lsb_release_version = re.compile(lsb_release_version_re, - re.ASCII) - release_version = re.compile(release_version_re, - re.ASCII) + lsb_release_version = re.compile(lsb_release_version_re, re.ASCII) + release_version = re.compile(release_version_re, re.ASCII) except AttributeError: lsb_release_version = re.compile(lsb_release_version_re) release_version = re.compile(release_version_re) @@ -853,7 +971,7 @@ class Platform(object): if len(line_list) > 1: dst_id = line_list[1] - return '', version, dst_id + return "", version, dst_id @staticmethod def dist(): @@ -872,9 +990,9 @@ class Platform(object): return "", "", "" try: - release_re = re.compile(r'(\w+)[-_](release|version)', re.ASCII) + release_re = re.compile(r"(\w+)[-_](release|version)", re.ASCII) except AttributeError: - release_re = re.compile(r'(\w+)[-_](release|version)') + release_re = re.compile(r"(\w+)[-_](release|version)") for etc_file in etc: m = release_re.match(etc_file) @@ -886,14 +1004,16 @@ class Platform(object): # read the first line try: etc_file_name = os.path.join(Platform.UNIXCONFDIR, etc_file) - with open(etc_file_name, 'r') as f: + with open(etc_file_name, "r") as f: firstline = f.readline() except Exception: continue # when euler, has centos-release - if (_distname.lower() == "centos" and - _distname.lower() not in firstline.lower()): + if ( + _distname.lower() == "centos" + and _distname.lower() not in firstline.lower() + ): continue if _distname.lower() in Platform.SUPPORTED_DISTS: @@ -934,8 +1054,7 @@ class ParameterContainer(object): index = self.ifiles.index(value) para_index = 0 for _ in range(index + 1): - para_index = self.parameters.index(self.IFILE, - para_index + 1) + para_index = self.parameters.index(self.IFILE, para_index + 1) self.parameters.pop(para_index) self.ifiles.pop(index) self.parameters.append(key) @@ -1027,17 +1146,21 @@ class SslCertConstructor(object): status, stdout, stderr = _exec_popen(cmd, values) except Exception as error: err_msg = str(error).replace(self.passwd, "*") - raise OSError("Failed to generate {}." - " Error: {}".format(cert_name, err_msg)) + raise OSError( + "Failed to generate {}." " Error: {}".format(cert_name, err_msg) + ) output = stdout + stderr output = output.replace(self.passwd, "*") if status: - raise OSError("Failed to generate {}." - " Error: {}".format(cert_name, output)) + raise OSError( + "Failed to generate {}." " Error: {}".format(cert_name, output) + ) if not self.check_certificate_files_exist([cert_name]): - raise OSError("Failed to generate {}. The file does not exist now." - " Error: {}".format(cert_name, output)) + raise OSError( + "Failed to generate {}. The file does not exist now." + " Error: {}".format(cert_name, output) + ) # Verify key files. if cert_name.endswith(".key") or cert_name == "cakey.pem": @@ -1045,8 +1168,9 @@ class SslCertConstructor(object): try: self.verify_ssl_key(path) except (OSError, ValueError) as error: - raise OSError("Failed to verify {}." - " Error: {}".format(cert_name, error)) + raise OSError( + "Failed to verify {}." " Error: {}".format(cert_name, error) + ) def __generate_cert_file(self, cmd, cert_name): """Generate cert file.""" @@ -1062,8 +1186,7 @@ class SslCertConstructor(object): def check_certificate_files_exist(self, cert_names): """Check whether the certificate file is generated.""" - log("Check whether the certificate files %s are generated." - % cert_names) + log("Check whether the certificate files %s are generated." % cert_names) for cert_name in cert_names: cert_path = os.path.join(self.keys_path, cert_name) if not os.path.exists(cert_path): @@ -1097,15 +1220,19 @@ class SslCertConstructor(object): log("Generate ca keys.") # cakey.pem - cmd = ('openssl genrsa -aes256 -f4 -passout stdin' - ' -out {0}/cakey.pem 2048'.format(self.keys_path)) + cmd = ( + "openssl genrsa -aes256 -f4 -passout stdin" + " -out {0}/cakey.pem 2048".format(self.keys_path) + ) self.__generate_cert_file(cmd, "cakey.pem") # cacert.pem - cmd = ('openssl req -new -x509 -passin stdin -days 10950' - ' -key {0}/cakey.pem -out {0}/cacert.pem' - ' -subj "/C=CN/ST=NULL/L=NULL/O=NULL/OU=NULL/' - 'CN=CA"'.format(self.keys_path)) + cmd = ( + "openssl req -new -x509 -passin stdin -days 10950" + " -key {0}/cakey.pem -out {0}/cacert.pem" + ' -subj "/C=CN/ST=NULL/L=NULL/O=NULL/OU=NULL/' + 'CN=CA"'.format(self.keys_path) + ) self.__generate_cert_file(cmd, "cacert.pem") def _generate_cert(self, role): @@ -1117,23 +1244,28 @@ class SslCertConstructor(object): log("Generate %s keys." % role) # key - cmd = ("openssl genrsa -aes256 -passout stdin -out {0}/{1}.key" - " 2048".format(self.keys_path, role)) + cmd = "openssl genrsa -aes256 -passout stdin -out {0}/{1}.key" " 2048".format( + self.keys_path, role + ) cert_name = "{}.key".format(role) self.__generate_cert_file(cmd, cert_name) # csr - cmd = ('openssl req -new -key {0}/{1}.key -passin stdin -out ' - '{0}/{1}.csr -subj "/C=CN/ST=NULL/L=NULL/O=NULL/OU=NULL/' - 'CN={1}"'.format(self.keys_path, role)) + cmd = ( + "openssl req -new -key {0}/{1}.key -passin stdin -out " + '{0}/{1}.csr -subj "/C=CN/ST=NULL/L=NULL/O=NULL/OU=NULL/' + 'CN={1}"'.format(self.keys_path, role) + ) cert_name = "{}.csr".format(role) self.__generate_cert_file(cmd, cert_name) # crt - cmd = ('openssl x509 -req -days 10950 -in {0}/{1}.csr' - ' -CA {0}/cacert.pem -CAkey {0}/cakey.pem -passin stdin' - ' -CAserial {0}/cacert.srl -CAcreateserial -out {0}/{1}.crt' - ' -extfile {0}/openssl.cnf'.format(self.keys_path, role)) + cmd = ( + "openssl x509 -req -days 10950 -in {0}/{1}.csr" + " -CA {0}/cacert.pem -CAkey {0}/cakey.pem -passin stdin" + " -CAserial {0}/cacert.srl -CAcreateserial -out {0}/{1}.crt" + " -extfile {0}/openssl.cnf".format(self.keys_path, role) + ) cert_name = "{}.crt".format(role) self.__generate_cert_file(cmd, cert_name) @@ -1143,22 +1275,19 @@ class SslCertConstructor(object): def _check_all_keys_exists(self): """All keys must be exists.""" - keys = ["cacert.pem", "server.crt", "server.key", - "client.crt", "client.key"] + keys = ["cacert.pem", "server.crt", "server.key", "client.crt", "client.key"] keys_ = set(keys) files = set([_ for _ in os.listdir(self.keys_path)]) lack_keys = keys_ - files if lack_keys: - raise Exception("Failed to generate keys: %s" - % " ".join(lack_keys)) + raise Exception("Failed to generate keys: %s" % " ".join(lack_keys)) def _clean_useless_path(self): """ Clean useless dirs and files, chmod the target files :return: NA """ - keys = ["cacert.pem", "server.crt", "server.key", - "client.crt", "client.key"] + keys = ["cacert.pem", "server.crt", "server.key", "client.crt", "client.key"] for filename in os.listdir(self.keys_path): if filename in [os.curdir, os.pardir]: continue @@ -1207,13 +1336,15 @@ class SslCertConstructor(object): except Exception as ssl_err: self.clean_all() err_msg = str(ssl_err).replace(self.passwd, "*") - raise Exception("Failed to generate ssl certificate. Error: %s" - % err_msg) + raise Exception("Failed to generate ssl certificate. Error: %s" % err_msg) log("Complete to generate ssl certificate.") def skip_execute_in_node_1(): - if g_opts.running_mode in [CANTIAND_IN_CLUSTER, CANTIAND_WITH_MYSQL_IN_CLUSTER] and g_opts.node_id == 1: + if ( + g_opts.running_mode in [CANTIAND_IN_CLUSTER, CANTIAND_WITH_MYSQL_IN_CLUSTER] + and g_opts.node_id == 1 + ): return True return False @@ -1223,6 +1354,7 @@ def skip_execute_in_slave_cluster(): return True return False + def create_dir_if_needed(condition, dir): if condition: return @@ -1231,14 +1363,16 @@ def create_dir_if_needed(condition, dir): def check_command(cmd): try: - output = subprocess.check_output(['/usr/bin/which', cmd], stderr=subprocess.STDOUT) + output = subprocess.check_output( + ["/usr/bin/which", cmd], stderr=subprocess.STDOUT + ) return True except subprocess.CalledProcessError as e: return False class Installer: - """ This is Cantiand installer. """ + """This is Cantiand installer.""" # Defining a constant identifies which step the installer failed to take. # For roll back. @@ -1277,7 +1411,7 @@ class Installer: IPV_TYPE = "ipv4" def __init__(self, user, group): - """ Constructor for the Installer class. """ + """Constructor for the Installer class.""" log("Begin init...") log("Installer runs on python version : " + gPyVersion) @@ -1343,7 +1477,7 @@ class Installer: self.repl_scram_auth = True # ENABLE_ACCESS_DC self.enable_access_dc = False - + # replace_password_verify self.replace_password_verify = True @@ -1357,11 +1491,11 @@ class Installer: self.factor_key = "" self.os_type = platform.machine() - self.have_numactl = check_command('numactl') + self.have_numactl = check_command("numactl") self.numactl_str = "" - if self.os_type == 'aarch64' and self.have_numactl == True: + if self.os_type == "aarch64" and self.have_numactl == True: last_cpu_core = os.cpu_count() - 1 - self.numactl_str = "numactl -C 0-1,6-11,16-" + str(last_cpu_core) + " " + self.numactl_str = "numactl -C 0-1,6-11,16-" + str(last_cpu_core) + " " log("End init") @@ -1374,24 +1508,24 @@ class Installer: find_files.append(file_name) if not find_files: - return '' + return "" if len(find_files) > 1: - raise Exception("More than one target found in %s: %s\n" - "Please remove the unused files." - % (path, ' ;'.join(find_files))) + raise Exception( + "More than one target found in %s: %s\n" + "Please remove the unused files." % (path, " ;".join(find_files)) + ) # file exists, return absolute file name file_name = os.path.realpath(os.path.join(path, find_files[0])) if not os.path.isfile(file_name): - raise Exception("%s is not file, please check your package." - % file_name) + raise Exception("%s is not file, please check your package." % file_name) return file_name def get_decompress_tarname(self, tar_file): - ''' + """ decompress a.tar.gz, then get file name :return: - ''' + """ # get real directory name in tar file tars = tarfile.open(tar_file) basename = tars.getnames()[0] @@ -1409,19 +1543,21 @@ class Installer: self.dirName = os.path.dirname(installFile) # get run.tar.gz package - run_pattern = ("^(%s|%s)-[A-Z0-9]+-64bit.tar.gz$" - % (self.RUN_VERSION_A, self.RUN_VERSION_B)) + run_pattern = "^(%s|%s)-[A-Z0-9]+-64bit.tar.gz$" % ( + self.RUN_VERSION_A, + self.RUN_VERSION_B, + ) self.runFile = self.find_file(self.dirName, run_pattern) if not self.runFile: - raise Exception("Can not get correct run package in path %s" - % self.dirName) + raise Exception("Can not get correct run package in path %s" % self.dirName) # get run.sha256 file - sha256_pattern = ("^(%s|%s)-[A-Z0-9]+-64bit.sha256$" - % (self.RUN_VERSION_A, self.RUN_VERSION_B)) + sha256_pattern = "^(%s|%s)-[A-Z0-9]+-64bit.sha256$" % ( + self.RUN_VERSION_A, + self.RUN_VERSION_B, + ) self.runSha256File = self.find_file(self.dirName, sha256_pattern) if not self.runSha256File: - raise Exception("Can not get correct sha256 file in path %s" - % self.dirName) + raise Exception("Can not get correct sha256 file in path %s" % self.dirName) # get run file name without suffix # compress package name is run.tar.gz, # decompress is run, remove .tar.gz @@ -1430,20 +1566,20 @@ class Installer: log("Using run file as : %s" % self.runFile) def is_readable(self, file_name, user): - ''' + """ :param path: :param user: :return: - ''' + """ user_info = pwd.getpwnam(user) uid = user_info.pw_uid gid = user_info.pw_gid s = os.stat(file_name) mode = s[stat.ST_MODE] return ( - ((s[stat.ST_UID] == uid) and (mode & stat.S_IRUSR > 0)) or - ((s[stat.ST_GID] == gid) and (mode & stat.S_IRGRP > 0)) or - (mode & stat.S_IROTH > 0) + ((s[stat.ST_UID] == uid) and (mode & stat.S_IRUSR > 0)) + or ((s[stat.ST_GID] == gid) and (mode & stat.S_IRGRP > 0)) + or (mode & stat.S_IROTH > 0) ) def clean_dir(self, dir_path): @@ -1464,35 +1600,37 @@ class Installer: pass def checkCreatedbFile(self): - ''' + """ check it is a file; user has read permission, :return: - ''' + """ # check -f parameter if self.option != self.INS_ALL: - raise Exception("Error: -f parameter should be used without" - " -O parameter ") + raise Exception( + "Error: -f parameter should be used without" " -O parameter " + ) # check it is a file if not os.path.isfile(self.create_db_file): - raise Exception("Error: %s does not exists or is not a file" - " or permission is not right." - % self.create_db_file) + raise Exception( + "Error: %s does not exists or is not a file" + " or permission is not right." % self.create_db_file + ) if not checkPath(self.create_db_file): - raise Exception("Error: %s file path invalid: " - % self.create_db_file) + raise Exception("Error: %s file path invalid: " % self.create_db_file) # if execute user is root, check common user has read permission file_path = os.path.dirname(self.create_db_file) # check path of create db sql file that user can cd permission_ok, _ = self.checkPermission(file_path, True) if not permission_ok: - raise Exception("Error: %s can not access %s" - % (self.user, file_path)) + raise Exception("Error: %s can not access %s" % (self.user, file_path)) # check create db file is readable for user if not self.is_readable(self.create_db_file, self.user): - raise Exception("Error: %s is not readable for user %s" - % (self.create_db_file, self.user)) + raise Exception( + "Error: %s is not readable for user %s" + % (self.create_db_file, self.user) + ) # change file to a realpath file self.create_db_file = os.path.realpath(self.create_db_file) @@ -1501,17 +1639,35 @@ class Installer: Parse cantiand, cms, gss default config :return: cantiand config, cms config, gss config """ - if g_opts.running_mode in [CANTIAND, CANTIAND_WITH_MYSQL, CANTIAND_WITH_MYSQL_ST]: - self.cantiandConfigs, self.cmsConfigs, self.gssConfigs = SingleNodeConfig.get_config(g_opts.in_container) - if g_opts.running_mode in [CANTIAND_IN_CLUSTER, CANTIAND_WITH_MYSQL_IN_CLUSTER] and g_opts.node_id == 0: - self.cantiandConfigs, self.cmsConfigs, self.gssConfigs = ClusterNode0Config.get_config(g_opts.in_container) - if g_opts.running_mode in [CANTIAND_IN_CLUSTER, CANTIAND_WITH_MYSQL_IN_CLUSTER] and g_opts.node_id == 1: - self.cantiandConfigs, self.cmsConfigs, self.gssConfigs = ClusterNode1Config.get_config(g_opts.in_container) + if g_opts.running_mode in [ + CANTIAND, + CANTIAND_WITH_MYSQL, + CANTIAND_WITH_MYSQL_ST, + ]: + self.cantiandConfigs, self.cmsConfigs, self.gssConfigs = ( + SingleNodeConfig.get_config(g_opts.in_container) + ) + if ( + g_opts.running_mode in [CANTIAND_IN_CLUSTER, CANTIAND_WITH_MYSQL_IN_CLUSTER] + and g_opts.node_id == 0 + ): + self.cantiandConfigs, self.cmsConfigs, self.gssConfigs = ( + ClusterNode0Config.get_config(g_opts.in_container) + ) + if ( + g_opts.running_mode in [CANTIAND_IN_CLUSTER, CANTIAND_WITH_MYSQL_IN_CLUSTER] + and g_opts.node_id == 1 + ): + self.cantiandConfigs, self.cmsConfigs, self.gssConfigs = ( + ClusterNode1Config.get_config(g_opts.in_container) + ) def addConfigForGss(self): - self.cantiandConfigs["CONTROL_FILES"] = "{0}, {1}, {2}".format(os.path.join(self.data, "data/ctrl1"), - os.path.join(self.data, "data/ctrl2"), - os.path.join(self.data, "data/ctrl3")) + self.cantiandConfigs["CONTROL_FILES"] = "{0}, {1}, {2}".format( + os.path.join(self.data, "data/ctrl1"), + os.path.join(self.data, "data/ctrl2"), + os.path.join(self.data, "data/ctrl3"), + ) self.gssConfigs["INTERCONNECT_ADDR"] = self.cantiandConfigs["INTERCONNECT_ADDR"] self.gssConfigs["LOG_HOME"] = self.cantiandConfigs["LOG_HOME"] del self.gssConfigs["STORAGE_MODE"] @@ -1519,9 +1675,11 @@ class Installer: self.cmsConfigs["GCC_HOME"] = os.path.join(self.data, "gcc_home") self.cmsConfigs["GCC_TYPE"] = "FILE" self.cmsConfigs["CMS_LOG"] = self.cantiandConfigs["LOG_HOME"] - + if g_opts.use_gss: - self.cantiandConfigs["CONTROL_FILES"] = "(+vg1/ctrl1, +vg1/ctrl2, +vg1/ctrl3)" + self.cantiandConfigs["CONTROL_FILES"] = ( + "(+vg1/ctrl1, +vg1/ctrl2, +vg1/ctrl3)" + ) self.gssConfigs["STORAGE_MODE"] = "CLUSTER_RAID" self.cmsConfigs["GCC_HOME"] = "/dev/gcc-disk" self.cmsConfigs["GCC_TYPE"] = "SD" @@ -1556,7 +1714,7 @@ class Installer: # Check user if not self.user: logExit("Parameter input error, need -U parameter.") - os.environ['cantiand_user'] = str(self.user) + os.environ["cantiand_user"] = str(self.user) # User must be exist. strCmd = "id -u ${cantiand_user}" ret_code, _, _ = _exec_popen(strCmd) @@ -1584,10 +1742,14 @@ class Installer: self.cantiandConfigs["LOG_HOME"] = os.path.join(self.data, "log") if len(self.cantiandConfigs["SHARED_PATH"]) == 0: self.cantiandConfigs["SHARED_PATH"] = os.path.join(self.data, "data") - self.cantiandConfigs["CT_CLUSTER_STRICT_CHECK"] = self.cantiandConfigs["CT_CLUSTER_STRICT_CHECK"].upper() + self.cantiandConfigs["CT_CLUSTER_STRICT_CHECK"] = self.cantiandConfigs[ + "CT_CLUSTER_STRICT_CHECK" + ].upper() if self.cantiandConfigs["CT_CLUSTER_STRICT_CHECK"] not in ["TRUE", "FALSE"]: self.cantiandConfigs["CT_CLUSTER_STRICT_CHECK"] = "TRUE" - self.cantiandConfigs["MYSQL_DEPLOY_GROUP_ID"] = grp.getgrnam(g_opts.os_group).gr_gid + self.cantiandConfigs["MYSQL_DEPLOY_GROUP_ID"] = grp.getgrnam( + g_opts.os_group + ).gr_gid self.addConfigForGss() self.gcc_home = self.cmsConfigs["GCC_HOME"] self.ssl_path = os.path.join(self.installPath, "sslkeys") @@ -1633,30 +1795,56 @@ class Installer: self.option = self.INS_PROGRAM # Get the kernel parameter elif key == "-Z": - _value = value.strip().split('=') + _value = value.strip().split("=") if len(_value) != 2: - log("Warning: kernel parameter will not take effect reason is invalid parameter: " + value, True) + log( + "Warning: kernel parameter will not take effect reason is invalid parameter: " + + value, + True, + ) continue self.cantiandConfigs[_value[0].strip().upper()] = _value[1].strip() elif key == "-C": - _value = value.strip().split('=') + _value = value.strip().split("=") if len(_value) != 2: - log("Warning: cms parameter will not take effect reason is invalid parameter: " + value, True) + log( + "Warning: cms parameter will not take effect reason is invalid parameter: " + + value, + True, + ) continue self.cmsConfigs[_value[0].strip().upper()] = _value[1].strip() elif key == "-G": - _value = value.strip().split('=') + _value = value.strip().split("=") if len(_value) != 2: - log("Warning: gss parameter will not take effect reason is invalid parameter: " + value, True) + log( + "Warning: gss parameter will not take effect reason is invalid parameter: " + + value, + True, + ) continue self.gssConfigs[_value[0].strip().upper()] = _value[1].strip() elif key == "-P": pass # Compatibility parameter - elif key in ["-g", "-l", "-U", "-M", "-W", "-s", "-N", "-d", "-p", "-m", "-r", "--dbstor", "--linktype"]: + elif key in [ + "-g", + "-l", + "-U", + "-M", + "-W", + "-s", + "-N", + "-d", + "-p", + "-m", + "-r", + "--dbstor", + "--linktype", + ]: pass elif key == "-f": self.create_db_file = value.strip() - elif key == '-c': + elif key == "-c": self.close_ssl = True else: logExit("Parameter input error: %s." % value) @@ -1671,9 +1859,8 @@ class Installer: log("Checking runner.", True) gid = os.getgid() uid = os.getuid() - log("Check runner user id and group id is : %s, %s" - % (str(uid), str(gid))) - if(gid != 0 and uid != 0): + log("Check runner user id and group id is : %s, %s" % (str(uid), str(gid))) + if gid != 0 and uid != 0: logExit("Only user with root privilege can run this script") log("End check runner is root") @@ -1690,19 +1877,32 @@ class Installer: os.chown(g_opts.log_file, uid, gid) except Exception as ex: logExit("Can not change log file's owner. Output:%s" % str(ex)) - + def chownDataDir(self): """ chown data and gcc dirs :return: """ - cmd = "chown %s:%s -R \"%s\" \"%s\";" % (self.user, self.group, self.data, self.gcc_home) + cmd = 'chown %s:%s -R "%s" "%s";' % ( + self.user, + self.group, + self.data, + self.gcc_home, + ) if g_opts.in_container: - cmd += "chown %s:%s -R \"%s\" \"%s\";" % (self.user, self.group, CommonValue.DOCKER_DATA_DIR, CommonValue.DOCKER_GCC_DIR) + cmd += 'chown %s:%s -R "%s" "%s";' % ( + self.user, + self.group, + CommonValue.DOCKER_DATA_DIR, + CommonValue.DOCKER_GCC_DIR, + ) log("Change owner cmd: %s" % cmd) ret_code, _, stderr = _exec_popen(cmd) if ret_code: - raise Exception("chown to %s:%s return: %s%s%s" % (self.user, self.group, str(ret_code), os.linesep, stderr)) + raise Exception( + "chown to %s:%s return: %s%s%s" + % (self.user, self.group, str(ret_code), os.linesep, stderr) + ) ########################################################################### # Is there a database installed by the user? If right, raise error @@ -1718,7 +1918,7 @@ class Installer: log("Checking old install.", True) # Check $CTDB_HOME. - if(g_opts.install_user_privilege == "withoutroot"): + if g_opts.install_user_privilege == "withoutroot": strCmd = "echo ~" else: strCmd = "su - '%s' -c \"echo ~\"" % self.user @@ -1727,7 +1927,7 @@ class Installer: logExit("Can not get user home.") # Get the profile of user. output = os.path.realpath(os.path.normpath(stdout)) - if (not checkPath(output)): + if not checkPath(output): logExit("The user home directory is invalid.") self.userProfile = os.path.join(output, ".bashrc") self.userHomePath = output @@ -1750,15 +1950,18 @@ class Installer: def dealwithCTDB(self, isFind, _file): while True: strLine = _file.readline() - if (not strLine): + if not strLine: break strLine = strLine.strip() - if (strLine.startswith("#")): + if strLine.startswith("#"): continue user_info = strLine.split() self.dealwithCTDB_DATA(user_info, strLine) - if (len(user_info) >= 2 and user_info[0] == "export" - and user_info[1].startswith("CTDB_HOME=") > 0): + if ( + len(user_info) >= 2 + and user_info[0] == "export" + and user_info[1].startswith("CTDB_HOME=") > 0 + ): isFind = True break else: @@ -1767,9 +1970,12 @@ class Installer: def dealwithCTDB_DATA(self, user_info, strLine): # deal with the CTDB_DATA with """ - if (len(user_info) >= 2 and user_info[0] == "export" - and user_info[1].startswith('CTDB_DATA="') > 0): - self.oldDataPath = strLine[strLine.find("=") + 2:-1] + if ( + len(user_info) >= 2 + and user_info[0] == "export" + and user_info[1].startswith('CTDB_DATA="') > 0 + ): + self.oldDataPath = strLine[strLine.find("=") + 2 : -1] self.oldDataPath = os.path.normpath(self.oldDataPath) realPath = os.path.realpath(self.oldDataPath) if not checkPath(realPath): @@ -1778,12 +1984,15 @@ class Installer: if self.option == self.INS_ALL and self.oldDataPath != self.data: logExit("User CTDB_DATA is different from -D parameter value") # deal with the CTDB_DATA path without """ - elif (len(user_info) >= 2 and user_info[0] == "export" - and user_info[1].startswith("CTDB_DATA=") > 0): - self.oldDataPath = strLine[strLine.find("=") + 1:] + elif ( + len(user_info) >= 2 + and user_info[0] == "export" + and user_info[1].startswith("CTDB_DATA=") > 0 + ): + self.oldDataPath = strLine[strLine.find("=") + 1 :] self.oldDataPath = os.path.normpath(self.oldDataPath) realPath = os.path.realpath(self.oldDataPath) - if (not checkPath(realPath)): + if not checkPath(realPath): logExit("The Path specified by CTDB_DATA is invalid.") log("Old data path: " + self.oldDataPath) if self.option == self.INS_ALL and self.oldDataPath != self.data: @@ -1815,10 +2024,10 @@ class Installer: """ log("Preparing path [%s]." % onePath) ownerPath = onePath - if(os.path.exists(onePath)): - if(checkEmpty): + if os.path.exists(onePath): + if checkEmpty: fileList = os.listdir(onePath) - if(len(fileList) != 0): + if len(fileList) != 0: logExit("Database path %s should be empty." % onePath) else: while True: @@ -1840,8 +2049,7 @@ class Installer: # will cause an error if ownerPath != onePath: cmd = "chown -R %s:%s %s; " % (self.user, self.group, ownerPath) - cmd += "chmod -R %s %s" % (CommonValue.MAX_DIRECTORY_MODE, - ownerPath) + cmd += "chmod -R %s %s" % (CommonValue.MAX_DIRECTORY_MODE, ownerPath) else: cmd = "chown %s:%s %s; " % (self.user, self.group, ownerPath) cmd += "chmod %s %s" % (CommonValue.MAX_DIRECTORY_MODE, ownerPath) @@ -1855,13 +2063,18 @@ class Installer: log("check [%s] user permission" % onePath) permission_ok, stderr = self.checkPermission(onePath) if not permission_ok: - logExit("Failed to check user [%s] path [%s] permission. Error: %s" - % (self.user, onePath, stderr)) + logExit( + "Failed to check user [%s] path [%s] permission. Error: %s" + % (self.user, onePath, stderr) + ) def checkMysqlDir(self, onePath): log("Checking Mysql path [%s]." % onePath) if not os.path.exists(onePath): - logExit("Database path %s does not exist or can not accessed. Please check it." % onePath) + logExit( + "Database path %s does not exist or can not accessed. Please check it." + % onePath + ) def checkPermission(self, originalPath, check_enter_only=False): """ @@ -1892,11 +2105,18 @@ class Installer: # check the user has write permission or not testFile = os.path.join(originalPath, "touch.tst") if g_opts.install_user_privilege == "withoutroot": - cmd = ("touch %s && chmod %s %s " - % (testFile, CommonValue.KEY_FILE_MODE, testFile)) + cmd = "touch %s && chmod %s %s " % ( + testFile, + CommonValue.KEY_FILE_MODE, + testFile, + ) else: - cmd = ("su - '%s' -c 'touch %s && chmod %s %s' " % - (self.user, testFile, CommonValue.KEY_FILE_MODE, testFile)) + cmd = "su - '%s' -c 'touch %s && chmod %s %s' " % ( + self.user, + testFile, + CommonValue.KEY_FILE_MODE, + testFile, + ) status, _, stderr = _exec_popen(cmd) if status != 0: @@ -1941,7 +2161,7 @@ class Installer: log("Checking directory.", True) # check if data or app path is too long(over 100 chars) - if(len(self.data) >= 110 or len(self.installPath) >= 110): + if len(self.data) >= 110 or len(self.installPath) >= 110: logExit("Install path or Data path is over 110 characters, exit.") # check install path is empty or not. self.prepareGivenPath(self.installPath) @@ -1949,59 +2169,62 @@ class Installer: if g_opts.slave_cluster: # Allow the data dir not empty in slave_cluster. self.prepareGivenPath(self.data, False) - else: + else: self.prepareGivenPath(self.data) # check install path vfs = os.statvfs(self.installPath) - availableSize = vfs.f_bavail * vfs.f_bsize / (1024*1024) - log("Database program install path available size: %sM" - % str(availableSize)) + availableSize = vfs.f_bavail * vfs.f_bsize / (1024 * 1024) + log("Database program install path available size: %sM" % str(availableSize)) if availableSize < 100: - logExit("Database program install path available size smaller" - " than 100M, current size is: %sM" - % str(availableSize)) + logExit( + "Database program install path available size smaller" + " than 100M, current size is: %sM" % str(availableSize) + ) # check data dir. if self.option == self.INS_ALL: # check partition of install path - strCmd1 = "df -h \"%s\"" % self.installPath - strCmd2 = "df -h \"%s\" | head -2" % self.installPath - strCmd3 = "df -h \"%s\" | head -2 |tail -1" % self.installPath - strCmd4 = ("df -h \"%s\" | head -2 |tail -1 | " - "awk -F\" \" '{print $1}'" % self.installPath) + strCmd1 = 'df -h "%s"' % self.installPath + strCmd2 = 'df -h "%s" | head -2' % self.installPath + strCmd3 = 'df -h "%s" | head -2 |tail -1' % self.installPath + strCmd4 = ( + 'df -h "%s" | head -2 |tail -1 | ' + "awk -F\" \" '{print $1}'" % self.installPath + ) cmds = [strCmd1, strCmd2, strCmd3, strCmd4] stdout = "" stdout_list = [] for cmd in cmds: ret_code, stdout, stderr = _exec_popen(strCmd1) if ret_code: - logExit("Can not get the partition of path: %s " - "%scommand: %s. %sError: %s" - % (self.installPath, os.linesep, - cmd, os.linesep, stderr)) + logExit( + "Can not get the partition of path: %s " + "%scommand: %s. %sError: %s" + % (self.installPath, os.linesep, cmd, os.linesep, stderr) + ) stdout_list.append(stdout) - log("The partition of path \"%s\": %s" - % (self.installPath, stdout)) + log('The partition of path "%s": %s' % (self.installPath, stdout)) self.checkPartitionOfDataDir(strCmd1, stdout_list) log("End check dir.") def checkPartitionOfDataDir(self, strCmd1, stdout_list): - strCmd5 = "df -h \"%s\"" % self.data - strCmd6 = "df -h \"%s\" | head -2" % self.data - strCmd7 = "df -h \"%s\" | head -2 |tail -1" % self.data - strCmd8 = ("df -h \"%s\" | head -2 |tail -1 " - "| awk -F\" \" '{print $1}'" % self.data) + strCmd5 = 'df -h "%s"' % self.data + strCmd6 = 'df -h "%s" | head -2' % self.data + strCmd7 = 'df -h "%s" | head -2 |tail -1' % self.data + strCmd8 = ( + 'df -h "%s" | head -2 |tail -1 ' "| awk -F\" \" '{print $1}'" % self.data + ) cmds = [strCmd5, strCmd6, strCmd7, strCmd8] stdout = "" for cmd in cmds: ret_code, stdout, stderr = _exec_popen(strCmd1) if ret_code: - logExit("Can not get the partition of path: %s " - "%scommand: %s. %sError: %s" - % (self.data, os.linesep, - cmd, os.linesep, stderr)) - log("The partition of path \"%s\": %s" - % (self.data, stdout)) + logExit( + "Can not get the partition of path: %s " + "%scommand: %s. %sError: %s" + % (self.data, os.linesep, cmd, os.linesep, stderr) + ) + log('The partition of path "%s": %s' % (self.data, stdout)) vfs = os.statvfs(self.data) availableSize = vfs.f_bavail * vfs.f_bsize / (1024 * 1024) @@ -2009,15 +2232,19 @@ class Installer: # check install path and data dir are in the same path or not if stdout_list[0] == stdout_list[1]: - if (availableSize < 20580): - logExit("The sum of database program and data directories" - " available size smaller than 20580M, " - "current size is: %sM" % str(availableSize)) + if availableSize < 20580: + logExit( + "The sum of database program and data directories" + " available size smaller than 20580M, " + "current size is: %sM" % str(availableSize) + ) else: - if (availableSize < 20480): - logExit("Database data directory available size smaller" - " than 20480M, current size is: " - "%sM" % str(availableSize)) + if availableSize < 20480: + logExit( + "Database data directory available size smaller" + " than 20480M, current size is: " + "%sM" % str(availableSize) + ) ######################################################################## # Check if the port is used in the installation parameters, and exit @@ -2054,21 +2281,31 @@ class Installer: # 98: Address already in use # 95: Operation not supported # 13: Permission denied - if (int(err.errno) == 98 or int(err.errno) == 95 - or int(err.errno) == 13): - log("Error: port %s has been used,the detail" - " information is as follows:" % value, True) + if ( + int(err.errno) == 98 + or int(err.errno) == 95 + or int(err.errno) == 13 + ): + log( + "Error: port %s has been used,the detail" + " information is as follows:" % value, + True, + ) strCmd = "netstat -unltp | grep %s" % value ret_code, stdout, _ = _exec_popen(strCmd) if ret_code: - logExit("can not get detail information of the" - " port, command: " + strCmd) + logExit( + "can not get detail information of the" + " port, command: " + strCmd + ) logExit(str(stdout)) except ValueError as ex: logExit("check port failed: " + str(ex)) else: - logExit("This install script can not support python version" - " : " + gPyVersion) + logExit( + "This install script can not support python version" + " : " + gPyVersion + ) def checkInnerPort(self, value): TIME_OUT = 2 @@ -2105,7 +2342,9 @@ class Installer: socket.inet_pton(socket.AF_INET6, nodeIp) self.IPV_TYPE = "ipv6" except socket.error: - logExit("The invalid IP address : %s is not ipv4 or ipv6 format." % nodeIp) + logExit( + "The invalid IP address : %s is not ipv4 or ipv6 format." % nodeIp + ) if self.IPV_TYPE == "ipv6": ping_cmd = "ping6" @@ -2116,9 +2355,11 @@ class Installer: cmd = "%s %s -i 1 -c 3 | grep ttl | wc -l" % (ping_cmd, nodeIp) ret_code, stdout, _ = _exec_popen(cmd) - if ret_code or stdout != '3': - logExit("The invalid IP address is %s. " - "ret_code : %s, stdout : %s" % (nodeIp, ret_code, stdout)) + if ret_code or stdout != "3": + logExit( + "The invalid IP address is %s. " + "ret_code : %s, stdout : %s" % (nodeIp, ret_code, stdout) + ) if all_zero_addr_after_ping(nodeIp): ip_is_found = 1 @@ -2127,11 +2368,12 @@ class Installer: ret_code, ip_is_found, _ = _exec_popen(ip_cmd) else: ip_is_found = 0 - - if ret_code or not int(ip_is_found): - logExit("The invalid IP address is %s. " - "ret_code : %s, ip_is_found : %s" % (nodeIp, ret_code, ip_is_found)) + if ret_code or not int(ip_is_found): + logExit( + "The invalid IP address is %s. " + "ret_code : %s, ip_is_found : %s" % (nodeIp, ret_code, ip_is_found) + ) log("checked the node IP address : %s" % nodeIp) @@ -2140,39 +2382,45 @@ class Installer: # not meet the requirements of the database。 ######################################################################### def set_numa_config(self): - if not os.path.exists('/usr/bin/lscpu'): + if not os.path.exists("/usr/bin/lscpu"): log("Warning: lscpu path get error") return - - _, result, _ = _exec_popen('/usr/bin/lscpu') + + _, result, _ = _exec_popen("/usr/bin/lscpu") if "NUMA node(s)" not in result: - err_code, ans, err_msg = _exec_popen('/usr/bin/lscpu | grep -i "On-line CPU(s) list"') - _ans = ans.strip().split(':') + err_code, ans, err_msg = _exec_popen( + '/usr/bin/lscpu | grep -i "On-line CPU(s) list"' + ) + _ans = ans.strip().split(":") if len(_ans) != 2: log("Warning: CPU(s) list get error, ans:%s" % ans) return self.cantiandConfigs["SHM_MYSQL_CPU_GROUP_INFO"] = _ans[1].strip() + " " self.cantiandConfigs["SHM_CPU_GROUP_INFO"] = _ans[1].strip() + " " return - - ret_code, result, stderr = _exec_popen('/usr/bin/lscpu | grep -i "NUMA node(s)"') + + ret_code, result, stderr = _exec_popen( + '/usr/bin/lscpu | grep -i "NUMA node(s)"' + ) if ret_code: logExit("can not get numa node parameters, err: %s" % stderr) - _result = result.strip().split(':') - + _result = result.strip().split(":") + if len(_result) != 2: log("Warning: numa get error, result:%s" % result) return numa_num = 0 numa_info = "" - #判断_result[1].strip() + # 判断_result[1].strip() if not _result[1].strip().isdigit(): log("Warning: numa(s) size get error, result:%s" % result) return while numa_num < int(_result[1].strip()): - err_code, ans, err_msg = _exec_popen('/usr/bin/lscpu | grep -i "NUMA node%s"' % numa_num) - _ans = ans.strip().split(':') + err_code, ans, err_msg = _exec_popen( + '/usr/bin/lscpu | grep -i "NUMA node%s"' % numa_num + ) + _ans = ans.strip().split(":") if len(_ans) != 2: log("Warning: numa node get error, ans:%s" % ans) return @@ -2190,16 +2438,17 @@ class Installer: """ log("Checking kernel parameters.", True) # GB MB kB - GB = 1024*1024*1024 - MB = 1024*1024 + GB = 1024 * 1024 * 1024 + MB = 1024 * 1024 KB = 1024 # The size of database - log_buffer_size = 4*MB - shared_pool_size = 128*MB - data_buffer_size = 128*MB - temp_buffer_size = 32*MB - sga_buff_size = (log_buffer_size + shared_pool_size + data_buffer_size - + temp_buffer_size) + log_buffer_size = 4 * MB + shared_pool_size = 128 * MB + data_buffer_size = 128 * MB + temp_buffer_size = 32 * MB + sga_buff_size = ( + log_buffer_size + shared_pool_size + data_buffer_size + temp_buffer_size + ) # getNuma self.set_numa_config() @@ -2213,12 +2462,18 @@ class Installer: check_kernel_parameter(key) check_invalid_symbol(value) # Unit conversion - sga_buff_size = self.doUnitConversion(GB, MB, KB, key, value, - sga_buff_size, - temp_buffer_size, - data_buffer_size, - shared_pool_size, - log_buffer_size) + sga_buff_size = self.doUnitConversion( + GB, + MB, + KB, + key, + value, + sga_buff_size, + temp_buffer_size, + data_buffer_size, + shared_pool_size, + log_buffer_size, + ) except ValueError as ex: logExit("check kernel parameter failed: " + str(ex)) @@ -2227,8 +2482,9 @@ class Installer: # Check the ip address for item in _list: if len(_list) != 1 and all_zero_addr_after_ping(item): - logExit("lsnr_addr contains all-zero ip," - " can not specify other ip.") + logExit( + "lsnr_addr contains all-zero ip," " can not specify other ip." + ) self.checkIPisVaild(item) else: # If this parameter is empty, the IPv4 is used by default. @@ -2253,18 +2509,23 @@ class Installer: if ret_code: logExit("can not get shmmax parameters, command: %s" % strCmd) - cmd = ("cat /proc/meminfo |grep -wE 'MemFree:|Buffers:|Cached:" - "|SwapCached' |awk '{sum += $2};END {print sum}'") + cmd = ( + "cat /proc/meminfo |grep -wE 'MemFree:|Buffers:|Cached:" + "|SwapCached' |awk '{sum += $2};END {print sum}'" + ) ret_code, cur_avi_memory, strerr = _exec_popen(cmd) if ret_code: - logExit("can not get shmmax parameters, command: %s, err: %s" % (cmd, stderr)) + logExit( + "can not get shmmax parameters, command: %s, err: %s" % (cmd, stderr) + ) if sga_buff_size < 114 * MB: - logExit("sga_buff_size should bigger than or equal to 114*MB," - " please check it!") + logExit( + "sga_buff_size should bigger than or equal to 114*MB," + " please check it!" + ) try: if sga_buff_size > int(cur_avi_memory) * KB: - logExit("sga_buff_size should smaller than shmmax," - " please check it!") + logExit("sga_buff_size should smaller than shmmax," " please check it!") except ValueError as ex: logExit("check kernel parameter failed: " + str(ex)) @@ -2290,28 +2551,37 @@ class Installer: elif key == "LSNR_PORT": self.lsnr_port = value elif key == "ENABLE_SYSDBA_LOGIN": - self.enableSysdbaLogin = Installer.check_pare_bool_value( - key, value) + self.enableSysdbaLogin = Installer.check_pare_bool_value(key, value) elif key == "REPL_AUTH": - self.repl_auth = Installer.check_pare_bool_value( - key, value) + self.repl_auth = Installer.check_pare_bool_value(key, value) elif key == "REPL_SCRAM_AUTH": - self.repl_scram_auth = Installer.check_pare_bool_value( - key, value) + self.repl_scram_auth = Installer.check_pare_bool_value(key, value) elif key == "ENABLE_ACCESS_DC": - self.enable_access_dc = Installer.check_pare_bool_value( - key, value) + self.enable_access_dc = Installer.check_pare_bool_value(key, value) elif key == "REPLACE_VERIFY_PASSWORD": - self.replace_password_verify = Installer.check_pare_bool_value( - key, value) + self.replace_password_verify = Installer.check_pare_bool_value(key, value) else: return - def doUnitConversion(self, GB, MB, KB, key, value, sga_buff_size, - temp_buffer_size, data_buffer_size, shared_pool_size, - log_buffer_size): - if key in ["TEMP_BUFFER_SIZE", "DATA_BUFFER_SIZE", - "SHARED_POOL_SIZE", "LOG_BUFFER_SIZE"]: + def doUnitConversion( + self, + GB, + MB, + KB, + key, + value, + sga_buff_size, + temp_buffer_size, + data_buffer_size, + shared_pool_size, + log_buffer_size, + ): + if key in [ + "TEMP_BUFFER_SIZE", + "DATA_BUFFER_SIZE", + "SHARED_POOL_SIZE", + "LOG_BUFFER_SIZE", + ]: if value[0:-1].isdigit() and value[-1:] in ["G", "M", "K"]: unit_map = { "G": GB, @@ -2350,24 +2620,26 @@ class Installer: with open(self.runFile, "rb") as _file: # Use sha256 when python version is lower higher 2.4 and # lower than 2.5. - if(gPyVersion >= PYTHON242 and gPyVersion < PYTHON25): + if gPyVersion >= PYTHON242 and gPyVersion < PYTHON25: sha256Obj = sha256.new() # Use hash when python version is higher than 2.5 - elif(gPyVersion >= PYTHON25): + elif gPyVersion >= PYTHON25: sha256Obj = hashlib.sha256() - if(sha256Obj is None): - logExit("check integrality of bin file failed, can not get" - " verification Obj.") + if sha256Obj is None: + logExit( + "check integrality of bin file failed, can not get" + " verification Obj." + ) while True: strRead = _file.read(8096) - if(not strRead): + if not strRead: break sha256Obj.update(strRead) strSHA256 = sha256Obj.hexdigest() with open(self.runSha256File, "r") as fileSHA256: strRead = fileSHA256.readline() oldSHA256 = strRead.strip() - if(strSHA256 == oldSHA256): + if strSHA256 == oldSHA256: isSameSHA256 = True else: isSameSHA256 = False @@ -2381,9 +2653,9 @@ class Installer: log("End check integrality of bin file") def execLogExit(self): - if (self.runFile == ""): + if self.runFile == "": logExit("Can not find run file.") - if (self.runSha256File == ""): + if self.runSha256File == "": logExit("Can not find verification file.") def changeAppPermission(self): @@ -2393,32 +2665,45 @@ class Installer: output: NA """ # change install path privilege to 700 - strCmd = "chmod %s %s -R" % (CommonValue.KEY_DIRECTORY_MODE, - self.installPath) + strCmd = "chmod %s %s -R" % (CommonValue.KEY_DIRECTORY_MODE, self.installPath) # chmod add-ons/ file 500 - strCmd += ("&& find '%s'/add-ons -type f | xargs chmod %s " - % (self.installPath, CommonValue.MID_FILE_MODE)) + strCmd += "&& find '%s'/add-ons -type f | xargs chmod %s " % ( + self.installPath, + CommonValue.MID_FILE_MODE, + ) # chmod admin/ file 600 - strCmd += ("&& find '%s'/admin -type f | xargs chmod %s " - % (self.installPath, CommonValue.KEY_FILE_MODE)) + strCmd += "&& find '%s'/admin -type f | xargs chmod %s " % ( + self.installPath, + CommonValue.KEY_FILE_MODE, + ) # chmod lib/ file 500 - strCmd += ("&& find '%s'/lib -type f | xargs chmod %s" - % (self.installPath, CommonValue.MID_FILE_MODE)) + strCmd += "&& find '%s'/lib -type f | xargs chmod %s" % ( + self.installPath, + CommonValue.MID_FILE_MODE, + ) # chmod bin/ file 500 - strCmd += ("&& find '%s'/bin -type f | xargs chmod %s " - % (self.installPath, CommonValue.MID_FILE_MODE)) + strCmd += "&& find '%s'/bin -type f | xargs chmod %s " % ( + self.installPath, + CommonValue.MID_FILE_MODE, + ) package_xml = os.path.join(self.installPath, "package.xml") if os.path.exists(package_xml): - strCmd += ("&& chmod %s '%s'/package.xml" - % (CommonValue.MIN_FILE_MODE, self.installPath)) + strCmd += "&& chmod %s '%s'/package.xml" % ( + CommonValue.MIN_FILE_MODE, + self.installPath, + ) log("Change app permission cmd: %s" % strCmd) ret_code, _, stderr = _exec_popen(strCmd) if ret_code: self.FAILED_POS = self.DECOMPRESS_BIN_FAILED self.rollBack() - logExit("chmod %s return: " % CommonValue.KEY_DIRECTORY_MODE - + str(ret_code) + os.linesep + stderr) + logExit( + "chmod %s return: " % CommonValue.KEY_DIRECTORY_MODE + + str(ret_code) + + os.linesep + + stderr + ) if not self.close_ssl: for file_name in os.listdir(self.ssl_path): @@ -2432,15 +2717,15 @@ class Installer: """ # eg 'length in [8-64]' if len(passwd) < shortest_len or len(passwd) > 64: - raise ValueError("The length of password must be %s to 64." - % shortest_len) + raise ValueError("The length of password must be %s to 64." % shortest_len) # Can't save with user name if passwd == self.user: raise ValueError("Error: Password can't be the same as username.") elif passwd == self.user[::-1]: - raise ValueError("Error: Password cannot be the same as username " - "in reverse order") + raise ValueError( + "Error: Password cannot be the same as username " "in reverse order" + ) upper_cases = set("ABCDEFGHIJKLMNOPQRSTUVWXYZ") lower_cases = set("abcdefghijklmnopqrstuvwxyz") @@ -2454,16 +2739,19 @@ class Installer: if passwd_set & cases: types += 1 if types < 3: - raise ValueError("Error: Password must contains at least three" - " different types of characters.") + raise ValueError( + "Error: Password must contains at least three" + " different types of characters." + ) # Only can contains enumerated cases all_cases = upper_cases | lower_cases | digits | special_cases un_cases = passwd_set - all_cases if un_cases: - raise ValueError("Error: There are characters that are not" - " allowed in the password: '%s'" - % "".join(un_cases)) + raise ValueError( + "Error: There are characters that are not" + " allowed in the password: '%s'" % "".join(un_cases) + ) def get_new_passwd(self, pw_prompt, user_prompt, shortest_len): """Get new passwd""" @@ -2498,9 +2786,11 @@ class Installer: # 0. "_SYS_PASSWORD" can't be set when ENABLE_SYSDBA_LOGIN is False sys_password = self.cantiandConfigs["_SYS_PASSWORD"] if not self.enableSysdbaLogin and len(sys_password) != 0: - raise ValueError("Can't use _SYS_PASSWORD to set the password of " - "user [SYS] in the installation script when " - "ENABLE_SYSDBA_LOGIN is False.") + raise ValueError( + "Can't use _SYS_PASSWORD to set the password of " + "user [SYS] in the installation script when " + "ENABLE_SYSDBA_LOGIN is False." + ) # 1. Get passed from parameter -C # Set passwd of SYS in cantiand.ini by parameter -C @@ -2511,9 +2801,8 @@ class Installer: if sys.stdin.isatty(): # If not pipe content, get passwd by interactive input g_opts.db_passwd = self.get_new_passwd( - pw_prompt="database password", - user_prompt="user [SYS]", - shortest_len=8) + pw_prompt="database password", user_prompt="user [SYS]", shortest_len=8 + ) else: try: # Get passwd from pipe @@ -2526,22 +2815,40 @@ class Installer: def copy_dbstor_path(self): str_cmd = "" if g_opts.use_dbstor: - os.makedirs("%s/dbstor/conf/dbs" % self.data, CommonValue.KEY_DIRECTORY_PERMISSION) - os.makedirs("%s/dbstor/conf/infra/config" % self.data, CommonValue.KEY_DIRECTORY_PERMISSION) - os.makedirs("%s/dbstor/data/logs" % self.data, CommonValue.KEY_DIRECTORY_PERMISSION) - os.makedirs("%s/dbstor/data/ftds" % self.data, CommonValue.KEY_DIRECTORY_PERMISSION) + os.makedirs( + "%s/dbstor/conf/dbs" % self.data, CommonValue.KEY_DIRECTORY_PERMISSION + ) + os.makedirs( + "%s/dbstor/conf/infra/config" % self.data, + CommonValue.KEY_DIRECTORY_PERMISSION, + ) + os.makedirs( + "%s/dbstor/data/logs" % self.data, CommonValue.KEY_DIRECTORY_PERMISSION + ) + os.makedirs( + "%s/dbstor/data/ftds" % self.data, CommonValue.KEY_DIRECTORY_PERMISSION + ) if is_rdma_startup(): - str_cmd += " && cp %s/cfg/node_config_rdma.xml %s/dbstor/conf/infra/config/node_config.xml" % ( - self.installPath, self.data) + str_cmd += ( + " && cp %s/cfg/node_config_rdma.xml %s/dbstor/conf/infra/config/node_config.xml" + % (self.installPath, self.data) + ) else: - str_cmd += " && cp %s/cfg/node_config_tcp.xml %s/dbstor/conf/infra/config/node_config.xml" % ( - self.installPath, self.data) - - str_cmd += " && cp %s/cfg/osd.cfg %s/dbstor/conf/infra/config/osd.cfg" % (self.installPath, self.data) - str_cmd += " && cp /home/conf/dbstor_config.ini %s/dbstor/conf/dbs/" % (self.data) - + str_cmd += ( + " && cp %s/cfg/node_config_tcp.xml %s/dbstor/conf/infra/config/node_config.xml" + % (self.installPath, self.data) + ) + + str_cmd += " && cp %s/cfg/osd.cfg %s/dbstor/conf/infra/config/osd.cfg" % ( + self.installPath, + self.data, + ) + str_cmd += " && cp /home/conf/dbstor_config.ini %s/dbstor/conf/dbs/" % ( + self.data + ) + return str_cmd - + ######################################################################### # Unzip the installation files to the installation directory. ######################################################################### @@ -2554,24 +2861,34 @@ class Installer: log("Decompressing run file.", True) # let bin executable - str_cmd = "chmod %s \"%s\"" % (CommonValue.KEY_DIRECTORY_MODE, - self.runFile) + str_cmd = 'chmod %s "%s"' % (CommonValue.KEY_DIRECTORY_MODE, self.runFile) log("decompress bin file executable cmd: %s" % str_cmd) ret_code, _, stderr = _exec_popen(str_cmd) if ret_code: self.FAILED_POS = self.DECOMPRESS_BIN_FAILED self.rollBack() - logExit("decompress bin file executable return: %s%s%s" - % (str(ret_code), os.linesep, stderr)) + logExit( + "decompress bin file executable return: %s%s%s" + % (str(ret_code), os.linesep, stderr) + ) # decompress bin file. cantian_pkg_file = "%s/%s" % (self.installPath, self.run_pkg_name) - str_cmd = "tar -xvf \"%s\" -C \"%s\"" % (self.runFile, self.installPath) - str_cmd = ("%s && cp -rf %s/add-ons %s/admin %s/bin %s/cfg %s/lib " - "%s/package.xml %s" - % (str_cmd, cantian_pkg_file, cantian_pkg_file, cantian_pkg_file, - cantian_pkg_file, cantian_pkg_file, cantian_pkg_file, - self.installPath)) - + str_cmd = 'tar -xvf "%s" -C "%s"' % (self.runFile, self.installPath) + str_cmd = ( + "%s && cp -rf %s/add-ons %s/admin %s/bin %s/cfg %s/lib " + "%s/package.xml %s" + % ( + str_cmd, + cantian_pkg_file, + cantian_pkg_file, + cantian_pkg_file, + cantian_pkg_file, + cantian_pkg_file, + cantian_pkg_file, + self.installPath, + ) + ) + str_cmd += self.copy_dbstor_path() str_cmd += " && rm -rf %s" % cantian_pkg_file log("Decompress cmd: " + str_cmd) @@ -2579,24 +2896,23 @@ class Installer: if ret_code: self.FAILED_POS = self.DECOMPRESS_BIN_FAILED self.rollBack() - logExit("Decompress bin return: " + str(ret_code) - + os.linesep + stderr) + logExit("Decompress bin return: " + str(ret_code) + os.linesep + stderr) # change app permission self.changeAppPermission() # change owner to user:group - str_cmd = "chown %s:%s -R %s " % (self.user, self.group, - self.installPath) + str_cmd = "chown %s:%s -R %s " % (self.user, self.group, self.installPath) # Change the owner log("Change owner cmd: %s" % str_cmd) ret_code, _, stderr = _exec_popen(str_cmd) if ret_code: self.FAILED_POS = self.DECOMPRESS_BIN_FAILED self.rollBack() - logExit("chown to %s: %s return: %s%s%s" - % (self.user, self.group, str(ret_code), - os.linesep, stderr)) + logExit( + "chown to %s: %s return: %s%s%s" + % (self.user, self.group, str(ret_code), os.linesep, stderr) + ) log("End decompress bin file.") @@ -2617,22 +2933,30 @@ class Installer: def exportUserEnv(self): try: with open(self.userProfile, "a") as _file: - _file.write("export CTDB_HOME=\"%s\"" % self.installPath) + _file.write('export CTDB_HOME="%s"' % self.installPath) _file.write(os.linesep) - _file.write("export PATH=\"%s\":\"%s\":$PATH" - % (os.path.join(self.installPath, "bin"), - os.path.join(MYSQL_BIN_DIR, "bin"))) + _file.write( + 'export PATH="%s":"%s":$PATH' + % ( + os.path.join(self.installPath, "bin"), + os.path.join(MYSQL_BIN_DIR, "bin"), + ) + ) _file.write(os.linesep) - _file.write("export LD_LIBRARY_PATH=\"%s\":\"%s\"" - ":$LD_LIBRARY_PATH" - % (os.path.join(self.installPath, "lib"), - os.path.join(self.installPath, "add-ons"))) + _file.write( + 'export LD_LIBRARY_PATH="%s":"%s"' + ":$LD_LIBRARY_PATH" + % ( + os.path.join(self.installPath, "lib"), + os.path.join(self.installPath, "add-ons"), + ) + ) _file.write(os.linesep) if self.oldDataPath == "": # set CTDB_DATA - _file.write("export CTDB_DATA=\"%s\"" % self.data) + _file.write('export CTDB_DATA="%s"' % self.data) _file.write(os.linesep) - _file.write("export CMS_HOME=\"%s\"" % self.data) + _file.write('export CMS_HOME="%s"' % self.data) _file.write(os.linesep) _file.flush() except IOError as ex: @@ -2658,19 +2982,25 @@ class Installer: # Avoid create database failed by the value of CTSQL_SSL_KEY_PASSWD self.clean_ssl_env() - os.environ['PATH'] = (os.path.join(self.installPath, "bin") - + ":" + os.environ['PATH']) + os.environ["PATH"] = ( + os.path.join(self.installPath, "bin") + ":" + os.environ["PATH"] + ) # in some system LD_LIBRARY_PATH is not set, # so must check it, or excetion will be raise - if 'LD_LIBRARY_PATH' in os.environ: - os.environ['LD_LIBRARY_PATH'] = ("%s:%s:%s" % ( - os.path.join(self.installPath, "lib"), os.path.join( - self.installPath, "add-ons",), - os.environ['LD_LIBRARY_PATH'])) + if "LD_LIBRARY_PATH" in os.environ: + os.environ["LD_LIBRARY_PATH"] = "%s:%s:%s" % ( + os.path.join(self.installPath, "lib"), + os.path.join( + self.installPath, + "add-ons", + ), + os.environ["LD_LIBRARY_PATH"], + ) else: - os.environ['LD_LIBRARY_PATH'] = ("%s:%s" % ( + os.environ["LD_LIBRARY_PATH"] = "%s:%s" % ( os.path.join(self.installPath, "lib"), - os.path.join(self.installPath, "add-ons"),)) + os.path.join(self.installPath, "add-ons"), + ) os.environ["CTDB_HOME"] = self.installPath os.environ["CTDB_DATA"] = self.data os.environ["CMS_HOME"] = self.data @@ -2700,8 +3030,10 @@ class Installer: cmd = cmd.strip(";") ret_code, _, stderr = _exec_popen(cmd) if ret_code: - raise Exception("Can not write the %s, command: %s," - " output: %s" % (conf_file, cmd, stderr)) + raise Exception( + "Can not write the %s, command: %s," + " output: %s" % (conf_file, cmd, stderr) + ) def clean_old_conf(self, param_list, conf_file): """ @@ -2712,28 +3044,38 @@ class Installer: cmd = "" # make the command of delete the parameter for parameter in param_list: - cmd += "sed -i '/^%s/d' %s;" % (parameter.replace('[', '\[').replace(']', '\]'), conf_file) + cmd += "sed -i '/^%s/d' %s;" % ( + parameter.replace("[", "\[").replace("]", "\]"), + conf_file, + ) if cmd: cmd = cmd.strip(";") ret_code, _, stderr = _exec_popen(cmd) if ret_code: - raise Exception("Can not write the %s, command: %s," - " output: %s" - % (conf_file, cmd, stderr)) + raise Exception( + "Can not write the %s, command: %s," + " output: %s" % (conf_file, cmd, stderr) + ) def generate_nomount_passwd(self, plain_passwd=""): if g_opts.install_user_privilege == "withoutroot": cmd = "%s/bin/ctencrypt -e PBKDF2" % self.installPath else: - cmd = (""" su - '%s' -c "%s/bin/ctencrypt -e PBKDF2" """ - % (self.user, self.installPath)) - g_opts.db_passwd = g_opts.db_passwd if len(plain_passwd.strip()) == 0 else plain_passwd.strip() + cmd = """ su - '%s' -c "%s/bin/ctencrypt -e PBKDF2" """ % ( + self.user, + self.installPath, + ) + g_opts.db_passwd = ( + g_opts.db_passwd if len(plain_passwd.strip()) == 0 else plain_passwd.strip() + ) values = [g_opts.db_passwd, g_opts.db_passwd] ret_code, stdout, stderr = _exec_popen(cmd, values) if ret_code: - raise OSError("Failed to encrypt password of user [sys]." - " Error: %s" % (stderr+os.linesep+stderr)) + raise OSError( + "Failed to encrypt password of user [sys]." + " Error: %s" % (stderr + os.linesep + stderr) + ) # Example of output: # Please enter password to encrypt: @@ -2758,17 +3100,25 @@ class Installer: cmd = "echo >> %s" % conf_file ret_code, _, stderr = _exec_popen(cmd) if ret_code: - raise Exception("Can not write the %s, command: %s," - " output: %s" % (file, cmd, stderr)) + raise Exception( + "Can not write the %s, command: %s," + " output: %s" % (file, cmd, stderr) + ) # Generate new kernel parameters common_parameters = copy.deepcopy(config) # Set password of NOMOUNT mode before create database. if encrypt_passwd: - common_parameters["_SYS_PASSWORD"] = self.generate_nomount_passwd(common_parameters["_SYS_PASSWORD"]) - if "GCC_TYPE" in common_parameters and\ - (common_parameters["GCC_TYPE"] == "FILE" or common_parameters["GCC_TYPE"] == "NFS"): - common_parameters["GCC_HOME"] = os.path.join(common_parameters["GCC_HOME"], "gcc_file") + common_parameters["_SYS_PASSWORD"] = self.generate_nomount_passwd( + common_parameters["_SYS_PASSWORD"] + ) + if "GCC_TYPE" in common_parameters and ( + common_parameters["GCC_TYPE"] == "FILE" + or common_parameters["GCC_TYPE"] == "NFS" + ): + common_parameters["GCC_HOME"] = os.path.join( + common_parameters["GCC_HOME"], "gcc_file" + ) # 1.clean old conf self.clean_old_conf(list(common_parameters.keys()), conf_file) @@ -2785,25 +3135,34 @@ class Installer: cmd = "echo >> %s" % conf_file ret_code, _, stderr = _exec_popen(cmd) if ret_code: - raise Exception("Can not write the %s, command: %s," - " output: %s" % (conf_file, cmd, stderr)) + raise Exception( + "Can not write the %s, command: %s," + " output: %s" % (conf_file, cmd, stderr) + ) size = CLUSTER_SIZE - if g_opts.running_mode in [CANTIAND, CANTIAND_WITH_MYSQL, CANTIAND_WITH_MYSQL_ST]: + if g_opts.running_mode in [ + CANTIAND, + CANTIAND_WITH_MYSQL, + CANTIAND_WITH_MYSQL_ST, + ]: size = 1 node_ip = re.split(r"[;,]", self.cantiandConfigs["INTERCONNECT_ADDR"]) if len(node_ip) == 1: node_ip.append("127.0.0.1") - + gcc_home = self.cmsConfigs["GCC_HOME"] - if self.cmsConfigs["GCC_TYPE"] == "FILE" or self.cmsConfigs["GCC_TYPE"] == "NFS": + if ( + self.cmsConfigs["GCC_TYPE"] == "FILE" + or self.cmsConfigs["GCC_TYPE"] == "NFS" + ): gcc_home = os.path.join(gcc_home, "gcc_file") - + # Generate new kernel parameters common_parameters = { "GCC_HOME": gcc_home, "REPORT_FILE": g_opts.log_file, "STATUS_LOG": os.path.join(self.data, "log", "cantianstatus.log"), - "LD_LIBRARY_PATH": os.environ['LD_LIBRARY_PATH'], + "LD_LIBRARY_PATH": os.environ["LD_LIBRARY_PATH"], "USER_HOME": self.userHomePath, "USE_GSS": g_opts.use_gss, "USE_DBSTOR": g_opts.use_dbstor, @@ -2844,7 +3203,7 @@ class Installer: addr_list = [_.strip() for _ in g_opts.white_list.split(",")] for item in re.split(r"[;,]", self.cantiandConfigs["INTERCONNECT_ADDR"]): if item not in addr_list: - addr_list.append(item) + addr_list.append(item) if "127.0.0.1" in addr_list: addr_list.remove("127.0.0.1") if "::1" in addr_list: @@ -2861,8 +3220,11 @@ class Installer: cmd = cmd.strip(";") ret_code, _, stderr = _exec_popen(cmd) if ret_code: - raise Exception("Can not write the %s, command: %s," - " output: %s" % (self.CANTIAND_HBA_FILE, cmd, stderr)) + raise Exception( + "Can not write the %s, command: %s," + " output: %s" % (self.CANTIAND_HBA_FILE, cmd, stderr) + ) + def update_ssl_conf(self): """Update ssl config.""" if self.close_ssl: @@ -2883,8 +3245,9 @@ class Installer: key_, work_key = self.get_ctencrypt_keys_and_file() # Get the value of SSL_KEY_PASSWORD from ini and # CTSQL_SSL_KEY_PASSWD from env and set SSL_KEY_PASSWORD into ini - cipher = self.encrypt_ssl_key_passwd(key_, work_key, - ssl_constructor.passwd, skip_execute_sql) + cipher = self.encrypt_ssl_key_passwd( + key_, work_key, ssl_constructor.passwd, skip_execute_sql + ) # 3. Modify cantiand.ini by write # Set the ssl config in cantiand.ini for server self.set_ssl_conf(cipher, key_, work_key) @@ -2926,9 +3289,9 @@ class Installer: log("Get password of REPL_AUTH keys.") if sys.stdin.isatty(): # If not pipe content, get passwd by interactive input - passwd = self.get_new_passwd(pw_prompt="password", - user_prompt="REPL_AUTH keys", - shortest_len=16) + passwd = self.get_new_passwd( + pw_prompt="password", user_prompt="REPL_AUTH keys", shortest_len=16 + ) else: try: # Get passwd from pipe @@ -2942,15 +3305,19 @@ class Installer: log("Generate REPL_AUTH keys.", True) if g_opts.install_user_privilege == "withoutroot": - cmd = "%s/bin/ctencrypt -r -d '%s'" % (self.installPath, - self.data) + cmd = "%s/bin/ctencrypt -r -d '%s'" % (self.installPath, self.data) else: - cmd = (""" su - '%s' -c "%s/bin/ctencrypt -r -d '%s'" """ - % (self.user, self.installPath, self.data)) + cmd = """ su - '%s' -c "%s/bin/ctencrypt -r -d '%s'" """ % ( + self.user, + self.installPath, + self.data, + ) ret_code, stdout, stderr = _exec_popen(cmd, [passwd, passwd]) if ret_code: - raise OSError("Failed to generate REPL_AUTH keys." - " Error: %s" % (stdout + os.linesep + stderr)) + raise OSError( + "Failed to generate REPL_AUTH keys." + " Error: %s" % (stdout + os.linesep + stderr) + ) log("Successfully generate REPL_AUTH keys") except Exception as err: self.rollBack() @@ -2970,7 +3337,7 @@ class Installer: insList = insStr.split(os.sep) regString = "" for i in insList: - if(i == ""): + if i == "": continue else: regString += r"\/" + i @@ -2984,8 +3351,7 @@ class Installer: log("Clean environment variables cmd: %s" % cmd) ret_code, _, stderr = _exec_popen(cmd) if ret_code: - log("Failed to clean environment variables." - " Error: %s" % stderr) + log("Failed to clean environment variables." " Error: %s" % stderr) logExit("Failed to clean environment variables.") def clean_ssl_env(self): @@ -3016,13 +3382,15 @@ class Installer: # Clear environment variable CTDB_DATA data_cmd = r"/^\s*export\s*CTDB_DATA=.*$/d" # Clear environment variable PATH about database - path_cmd = (r"/^\s*export\s*PATH=.*%s\/bin.*:\$PATH$/d" - % self.genregstring(self.installPath)) + path_cmd = r"/^\s*export\s*PATH=.*%s\/bin.*:\$PATH$/d" % self.genregstring( + self.installPath + ) # Clear environment variable LD_LIBRARY_PATH about database - lib_cmd = (r"/^\s*export\s*LD_LIBRARY_PATH=.*%s\/lib.*" - r":.*%s\/add-ons.*:\$LD_LIBRARY_PATH$/d" - % (self.genregstring(self.installPath), - self.genregstring(self.installPath))) + lib_cmd = ( + r"/^\s*export\s*LD_LIBRARY_PATH=.*%s\/lib.*" + r":.*%s\/add-ons.*:\$LD_LIBRARY_PATH$/d" + % (self.genregstring(self.installPath), self.genregstring(self.installPath)) + ) # Clear environment variable CTDB_HOME home_cmd = r"/^\s*export\s*CTDB_HOME=.*$/d" # Clear environment variable CMS_HOME @@ -3035,8 +3403,17 @@ class Installer: mode_cmd = r"/^\s*export\s*CTSQL_SSL_MODE=.*$/d" cipher_cmd = r"/^\s*export\s*CTSQL_SSL_KEY_PASSWD=.*$/d" - cmds = [path_cmd, lib_cmd, home_cmd, cms_cmd, - ca_cmd, cert_cmd, key_cmd, mode_cmd, cipher_cmd] + cmds = [ + path_cmd, + lib_cmd, + home_cmd, + cms_cmd, + ca_cmd, + cert_cmd, + key_cmd, + mode_cmd, + cipher_cmd, + ] if self.option == self.INS_ALL: cmds.insert(0, data_cmd) @@ -3063,9 +3440,11 @@ class Installer: # Delete program. self.rollbackFromDecompress() # rollback from set user env - elif(self.FAILED_POS == self.SET_ENV_FAILED - or self.FAILED_POS == self.PRE_DATA_DIR_FAILED - or self.FAILED_POS == self.INIT_DB_FAILED): + elif ( + self.FAILED_POS == self.SET_ENV_FAILED + or self.FAILED_POS == self.PRE_DATA_DIR_FAILED + or self.FAILED_POS == self.INIT_DB_FAILED + ): self.rollbackFromSetUserEnv() # rollback from create database @@ -3077,7 +3456,9 @@ class Installer: log("Roll back: init ") else: # should not be here. - logExit("Roll back can not recognize this operation: " + str(self.FAILED_POS)) + logExit( + "Roll back can not recognize this operation: " + str(self.FAILED_POS) + ) log("End roll back") def rollbackDataDirs(self): @@ -3117,19 +3498,27 @@ class Installer: def __kill_process(self, process_name): # root do install, need su - user kill process - cmd = ("proc_pid_list=\`ps ux | grep %s | grep -v grep " - "| awk '{print \$2}'\`" % process_name) - cmd += (" && (if [ X\\\"\$proc_pid_list\\\" != X\\\"\\\" ]; " - "then echo \\\"\$proc_pid_list\\\" | xargs kill -9 ; " - "exit 0; fi)") + cmd = ( + "proc_pid_list=\`ps ux | grep %s | grep -v grep " + "| awk '{print \$2}'\`" % process_name + ) + cmd += ( + ' && (if [ X\\"\$proc_pid_list\\" != X\\"\\" ]; ' + 'then echo \\"\$proc_pid_list\\" | xargs kill -9 ; ' + "exit 0; fi)" + ) kill_cmd = "su - '%s' -c \"%s\" " % (self.user, cmd) if g_opts.install_user_privilege == "withoutroot": # user do install, kill process - kill_cmd = (r"proc_pid_list=`ps ux | grep %s | grep -v grep" - r"|awk '{print $2}'` && " % process_name) - kill_cmd += (r"(if [ X\"$proc_pid_list\" != X\"\" ];then echo " - r"$proc_pid_list | xargs kill -9; exit 0; fi)") + kill_cmd = ( + r"proc_pid_list=`ps ux | grep %s | grep -v grep" + r"|awk '{print $2}'` && " % process_name + ) + kill_cmd += ( + r"(if [ X\"$proc_pid_list\" != X\"\" ];then echo " + r"$proc_pid_list | xargs kill -9; exit 0; fi)" + ) log("kill process cmd: %s" % kill_cmd) ret_code, _, _ = _exec_popen(kill_cmd) if ret_code: @@ -3143,7 +3532,9 @@ class Installer: log("Begin to backup log cmd: " + strCmd) ret_code, _, stderr = _exec_popen(strCmd) if ret_code: - logExit("Can not backup cantiand log command: %s, output: %s" % (strCmd, stderr)) + logExit( + "Can not backup cantiand log command: %s, output: %s" % (strCmd, stderr) + ) log("Error:The detail log for CREATE_DB_FAILED: %s" % self.backup_log_dir) # backup cantiand cfg before rm data @@ -3151,7 +3542,9 @@ class Installer: log("Begin to backup cfg cmd: " + strCmd) ret_code, _, stderr = _exec_popen(strCmd) if ret_code: - logExit("Can not backup cantiand cfg command: %s, output: %s" % (strCmd, stderr)) + logExit( + "Can not backup cantiand cfg command: %s, output: %s" % (strCmd, stderr) + ) log("Error:The detail log for CREATE_DB_FAILED: %s" % self.backup_log_dir) # kill database process @@ -3194,31 +3587,47 @@ class Installer: log("Change privilege cmd: %s" % strCmd) ret_code, _, stderr = _exec_popen(strCmd) if ret_code: - raise Exception("chmod %s return: " % CommonValue.KEY_DIRECTORY_MODE + str(ret_code) + os.linesep + stderr) + raise Exception( + "chmod %s return: " % CommonValue.KEY_DIRECTORY_MODE + + str(ret_code) + + os.linesep + + stderr + ) # create data, cfg, log dir, trc data_dir = "%s/data" % self.data if g_opts.in_container: # Do not create the data dir in slave cluster. if not g_opts.slave_cluster: - create_dir_if_needed(skip_execute_in_node_1(), CommonValue.DOCKER_DATA_DIR) + create_dir_if_needed( + skip_execute_in_node_1(), CommonValue.DOCKER_DATA_DIR + ) cmd = "ln -s %s %s;" % (CommonValue.DOCKER_DATA_DIR, data_dir) ret_code, _, stderr = _exec_popen(cmd) if ret_code: - raise Exception("Can not link data dir, command: %s, output: %s" % (cmd, stderr)) + raise Exception( + "Can not link data dir, command: %s, output: %s" % (cmd, stderr) + ) else: os.makedirs(data_dir, CommonValue.KEY_DIRECTORY_PERMISSION) os.makedirs("%s/log" % self.data, CommonValue.KEY_DIRECTORY_PERMISSION) - os.makedirs("%s/archive_log" % self.data, CommonValue.KEY_DIRECTORY_PERMISSION) + os.makedirs( + "%s/archive_log" % self.data, CommonValue.KEY_DIRECTORY_PERMISSION + ) os.makedirs("%s/trc" % self.data, CommonValue.KEY_DIRECTORY_PERMISSION) os.makedirs("%s/tmp" % self.data, CommonValue.KEY_DIRECTORY_PERMISSION) if not g_opts.use_gss and not g_opts.use_dbstor: if g_opts.in_container: - create_dir_if_needed(skip_execute_in_node_1(), CommonValue.DOCKER_GCC_DIR) + create_dir_if_needed( + skip_execute_in_node_1(), CommonValue.DOCKER_GCC_DIR + ) cmd = "ln -s %s %s" % (CommonValue.DOCKER_GCC_DIR, self.gcc_home) ret_code, _, stderr = _exec_popen(cmd) if ret_code: - raise Exception("Can not link gcc home dir, command: %s, output: %s" % (cmd, stderr)) + raise Exception( + "Can not link gcc home dir, command: %s, output: %s" + % (cmd, stderr) + ) else: os.makedirs(self.gcc_home, CommonValue.KEY_DIRECTORY_PERMISSION) @@ -3229,15 +3638,27 @@ class Installer: cmd = "mv -i %s/cfg %s" % (self.installPath, self.data) ret_code, _, stderr = _exec_popen(cmd) if ret_code: - raise Exception("Can not create prepare data dir, command: %s, output: %s" % (cmd, stderr)) + raise Exception( + "Can not create prepare data dir, command: %s, output: %s" + % (cmd, stderr) + ) # Change the mode of config files to 600 cmd = "chmod {0} {1}/cfg/{2} {1}/cfg/{3} {1}/cfg/{4}".format( - CommonValue.KEY_FILE_MODE, self.data, self.CANTIAND_CONF_FILE, - self.CMS_CONF_FILE, self.CANTIAND_HBA_FILE) + CommonValue.KEY_FILE_MODE, + self.data, + self.CANTIAND_CONF_FILE, + self.CMS_CONF_FILE, + self.CANTIAND_HBA_FILE, + ) ret_code, _, stderr = _exec_popen(cmd) if ret_code: - raise Exception("chmod %s return: " % CommonValue.KEY_FILE_MODE + str(ret_code) + os.linesep + stderr) + raise Exception( + "chmod %s return: " % CommonValue.KEY_FILE_MODE + + str(ret_code) + + os.linesep + + stderr + ) # Change the owner of config files self.chownDataDir() @@ -3253,14 +3674,14 @@ class Installer: """ if not g_opts.use_gss and not g_opts.use_dbstor: return - + cmd = "sh %s -P setcap >> %s 2>&1" % (INSTALL_SCRIPT, g_opts.log_file) ret_code, stdout, stderr = _exec_popen(cmd) if ret_code: output = stdout + stderr raise Exception("Failed to setcap.\ncmd: %s.\nOutput: %s" % (cmd, output)) log("setcap sucessed", True) - + def prepareMysqlForSingle(self): if g_opts.running_mode.lower() not in VALID_SINGLE_MYSQL_RUNNING_MODE: return @@ -3269,7 +3690,9 @@ class Installer: self.prepare_mysql_data_dir() self.prepare_mysql_bin_dir() self.set_mysql_env() + pass + def start_gss(self): if not g_opts.use_gss: return @@ -3280,7 +3703,9 @@ class Installer: ret_code, stdout, stderr = _exec_popen(cmd) if ret_code: output = stdout + stderr - raise Exception("Can not start gss.\nStart cmd: %s.\nOutput: %s" % (cmd, output)) + raise Exception( + "Can not start gss.\nStart cmd: %s.\nOutput: %s" % (cmd, output) + ) log("gss has started", True) def start_cms(self): @@ -3291,9 +3716,11 @@ class Installer: ret_code, stdout, stderr = _exec_popen(cmd) if ret_code: output = stdout + stderr - raise Exception("Can not start cms.\nStart cmd: %s.\nOutput: %s" % (cmd, output)) + raise Exception( + "Can not start cms.\nStart cmd: %s.\nOutput: %s" % (cmd, output) + ) log("cms has started", True) - + ################################################################## # start cantian instance ################################################################## @@ -3313,23 +3740,33 @@ class Installer: start_mode = self.OPEN_MODE if g_opts.node_id == 1: start_mode = self.OPEN_MODE - + # Start instance, according to running mode can point to cantiand or cantiand with mysql cmd = "sh %s -P cantiand -M %s -T %s -C %s >> %s 2>&1" % ( - INSTALL_SCRIPT, start_mode, g_opts.running_mode.lower(), g_opts.mysql_config_file_path, g_opts.log_file) + INSTALL_SCRIPT, + start_mode, + g_opts.running_mode.lower(), + g_opts.mysql_config_file_path, + g_opts.log_file, + ) if os.getuid() == 0: cmd = "su - %s -c '" % self.user + cmd + "'" - log("start cantiand with cmd:%s"%cmd) + log("start cantiand with cmd:%s" % cmd) status, stdout, stderr = _exec_popen(cmd) if status != 0: output = stdout + stderr - raise Exception("Can not start instance %s.\nStart cmd: %s.\nOutput: %s" % (self.data, cmd, output)) + raise Exception( + "Can not start instance %s.\nStart cmd: %s.\nOutput: %s" + % (self.data, cmd, output) + ) # In some condition cantian will take some time to start, so wait # it by checking the process cyclically after the start command # returned. If the cantiand process can't be found within the # expected time, it is considered that the startup failed. - tem_log_info, status_success = self.initSomeCondition(status_success, self.status_log) + tem_log_info, status_success = self.initSomeCondition( + status_success, self.status_log + ) # the log file's permission is 600, change it if os.path.exists(self.status_log): @@ -3339,8 +3776,10 @@ class Installer: os.chmod(self.status_log, CommonValue.KEY_FILE_PERMISSION) if not status_success: - raise Exception("Can not get instance '%s' process pid," - "The detailed information: '%s' " % (self.data, tem_log_info)) + raise Exception( + "Can not get instance '%s' process pid," + "The detailed information: '%s' " % (self.data, tem_log_info) + ) log("cantiand has started", True) def get_invalid_parameter(self): @@ -3350,7 +3789,10 @@ class Installer: ret_code, stdout, stderr = _exec_popen(cmd) output = stdout + stderr if ret_code: - log("Failed to get the error message from '%s'. Output: %s" % (run_log, output)) + log( + "Failed to get the error message from '%s'. Output: %s" + % (run_log, output) + ) return "" else: return output @@ -3361,25 +3803,38 @@ class Installer: for i in range(0, start_time): time.sleep(3) if g_opts.install_user_privilege == "withoutroot": - cmd = ("ps ux | grep -v grep | grep %s | grep $ " - "|awk '{print $2}'" % (self.data)) + cmd = "ps ux | grep -v grep | grep %s | grep $ " "|awk '{print $2}'" % ( + self.data + ) else: - cmd = ("su - '%s' -c \"ps ux | grep -v grep | grep cantiand | " - "grep %s$ |awk '{print \$2}'\" " - % (self.user, self.data)) - if g_opts.running_mode.lower() in [CANTIAND_WITH_MYSQL, CANTIAND_WITH_MYSQL_ST, CANTIAND_WITH_MYSQL_IN_CLUSTER]: - cmd = ("su - '%s' -c \"ps -ef | grep -v grep | grep mysqld\""%(self.user)) + cmd = ( + "su - '%s' -c \"ps ux | grep -v grep | grep cantiand | " + "grep %s$ |awk '{print \$2}'\" " % (self.user, self.data) + ) + if g_opts.running_mode.lower() in [ + CANTIAND_WITH_MYSQL, + CANTIAND_WITH_MYSQL_ST, + CANTIAND_WITH_MYSQL_IN_CLUSTER, + ]: + cmd = "su - '%s' -c \"ps -ef | grep -v grep | grep mysqld\"" % ( + self.user + ) status_log = MYSQL_LOG_FILE pass ret_code, stdout, stderr = _exec_popen(cmd) if ret_code: status_success = False - tem_log_info = ("Failed to execute cmd: %s.output:%s" - % (str(cmd), str(stderr))) + tem_log_info = "Failed to execute cmd: %s.output:%s" % ( + str(cmd), + str(stderr), + ) break else: all_the_text = open(status_log).read() - log("Instance start log output:%s.cmd:%s" %(str(all_the_text),str(cmd))) + log( + "Instance start log output:%s.cmd:%s" + % (str(all_the_text), str(cmd)) + ) if all_the_text.find("instance started") > 0: if stdout: status_success = True @@ -3395,7 +3850,10 @@ class Installer: run_log_info = self.get_invalid_parameter() if run_log_info: tem_log_info += os.linesep - tem_log_info += ("The run log error: %s%s" % (os.linesep, run_log_info)) + tem_log_info += "The run log error: %s%s" % ( + os.linesep, + run_log_info, + ) break if (i + 1) == start_time: status_success = False @@ -3427,53 +3885,66 @@ class Installer: return if g_opts.install_user_privilege == "withoutroot": if self.enableSysdbaLogin: - cmd = "%s/bin/ctsql / as sysdba -q -D %s -f \"%s\" " % ( + cmd = '%s/bin/ctsql / as sysdba -q -D %s -f "%s" ' % ( self.installPath, self.data, - sql_file) + sql_file, + ) return_code, stdout_data, stderr_data = _exec_popen(cmd) else: - cmd = "%s/bin/ctsql %s@%s:%s -q -f \"%s\" " % ( + cmd = '%s/bin/ctsql %s@%s:%s -q -f "%s" ' % ( self.installPath, g_opts.db_user, self.LOGIN_IP, self.lsnr_port, - sql_file) + sql_file, + ) return_code, stdout_data, stderr_data = _exec_popen( - cmd, [g_opts.db_passwd]) + cmd, [g_opts.db_passwd] + ) else: if self.enableSysdbaLogin: - cmd = ("su - '%s' -c \"%s/bin/ctsql / as sysdba " - "-q -D %s -f \"%s\" \"" - % (self.user, self.installPath, self.data, sql_file)) + cmd = ( + "su - '%s' -c \"%s/bin/ctsql / as sysdba " + '-q -D %s -f "%s" "' + % (self.user, self.installPath, self.data, sql_file) + ) return_code, stdout_data, stderr_data = _exec_popen(cmd) else: - cmd = ("su - '%s' -c \"%s/bin/ctsql %s@%s:%s -q -f \"%s\"\"" % ( + cmd = 'su - \'%s\' -c "%s/bin/ctsql %s@%s:%s -q -f "%s""' % ( self.user, self.installPath, g_opts.db_user, self.LOGIN_IP, self.lsnr_port, - sql_file)) + sql_file, + ) return_code, stdout_data, stderr_data = _exec_popen( - cmd, [g_opts.db_passwd]) + cmd, [g_opts.db_passwd] + ) output = "%s%s" % (str(stdout_data), str(stderr_data)) log("Execute sql file %s output: %s" % (sql_file, output)) if return_code: - raise Exception("Failed to execute sql file %s, output:%s" % (sql_file, output)) + raise Exception( + "Failed to execute sql file %s, output:%s" % (sql_file, output) + ) # return code is 0, but output has error info, CT-xxx, ZS-xxx result = output.replace("\n", "") if re.match(".*CT-\d{5}.*", result) or re.match(".*ZS-\d{5}.*", result): - raise Exception("Failed to execute sql file %s, output:%s" % (sql_file, output)) + raise Exception( + "Failed to execute sql file %s, output:%s" % (sql_file, output) + ) def execute_mysql_update(self, sql_file): is_need_update = False if is_need_update: print("update cantian sys table and views...") action_parse = argparse.ArgumentParser() - action_parse.add_argument("--mysql_cmd", dest="mysql_cmd", required=True) # /usr/local/mysql/bin/mysql + action_parse.add_argument( + "--mysql_cmd", dest="mysql_cmd", required=True + ) # /usr/local/mysql/bin/mysql action_parse.add_argument("--mysql_user", dest="mysql_user", required=True) args = action_parse.parse_args() mysql_cmd = args.mysql_cmd @@ -3484,12 +3955,13 @@ class Installer: mysql_pwd = getpass.getpass() cmd = "%s -u%s -p%s < %s" % (mysql_cmd, mysql_user, mysql_pwd, sql_file) - return_code, stdout_data, stderr_data = _exec_popen(cmd) if return_code and MYSQL_VERSION != VERSION_DOCKER_META: output = "%s%s" % (str(stdout_data), str(stderr_data)) - raise Exception("Failed to execute mysql file %s, output:%s, return_code:%s" - % (sql_file, output, return_code)) + raise Exception( + "Failed to execute mysql file %s, output:%s, return_code:%s" + % (sql_file, output, return_code) + ) def execute_sql(self, sql, message): """ @@ -3505,41 +3977,47 @@ class Installer: # 4 normal user execute install.py and disable sysdba if g_opts.install_user_privilege == "withoutroot": if self.enableSysdbaLogin: - cmd = "%s/bin/ctsql / as sysdba -q -D %s -c \"%s\" " % ( + cmd = '%s/bin/ctsql / as sysdba -q -D %s -c "%s" ' % ( self.installPath, self.data, - sql) + sql, + ) return_code, stdout_data, stderr_data = _exec_popen(cmd) else: - cmd = "%s/bin/ctsql %s@%s:%s -q -c \"%s\" " % ( + cmd = '%s/bin/ctsql %s@%s:%s -q -c "%s" ' % ( self.installPath, g_opts.db_user, self.LOGIN_IP, self.lsnr_port, - sql) + sql, + ) return_code, stdout_data, stderr_data = _exec_popen( - cmd, [g_opts.db_passwd]) + cmd, [g_opts.db_passwd] + ) else: if self.enableSysdbaLogin: - cmd = ("su - '%s' -c \"%s/bin/ctsql / as sysdba " - "-q -D %s -c \\\"%s\\\" \"" - % (self.user, self.installPath, self.data, sql)) + cmd = ( + "su - '%s' -c \"%s/bin/ctsql / as sysdba " + '-q -D %s -c \\"%s\\" "' + % (self.user, self.installPath, self.data, sql) + ) return_code, stdout_data, stderr_data = _exec_popen(cmd) else: - cmd = ("su - '%s' -c \"%s/bin/ctsql %s@%s:%s -q" - " -c \\\"%s\\\"\"" % (self.user, - self.installPath, - g_opts.db_user, - self.LOGIN_IP, - self.lsnr_port, - sql)) + cmd = "su - '%s' -c \"%s/bin/ctsql %s@%s:%s -q" ' -c \\"%s\\""' % ( + self.user, + self.installPath, + g_opts.db_user, + self.LOGIN_IP, + self.lsnr_port, + sql, + ) return_code, stdout_data, stderr_data = _exec_popen( - cmd, [g_opts.db_passwd]) + cmd, [g_opts.db_passwd] + ) output = "%s%s" % (str(stdout_data), str(stderr_data)) if return_code: - raise Exception("Failed to %s by sql, output:%s" - % (message, output)) + raise Exception("Failed to %s by sql, output:%s" % (message, output)) # return code is 0, but output has error info, CT-xxx, ZS-xxx result = output.replace("\n", "") @@ -3567,19 +4045,26 @@ class Installer: self.status_log = "%s/log/cantianstatus.log" % self.data if os.path.exists(self.status_log): os.remove(self.status_log) - + if os.getuid() == 0: cmd = "chown %s:%s %s;" % (self.user, self.group, INSTALL_SCRIPT) ret_code, _, stderr = _exec_popen(cmd) if ret_code: - raise Exception("chown to %s:%s return: %s%s%s" - % (self.user, self.group, str(ret_code), - os.linesep, stderr)) - - persist_environment_variable("RUN_MODE", g_opts.running_mode.lower(),self.userProfile) - - # start mysql and cantian in single process mode with metadata in cantian - if g_opts.running_mode.lower() in [CANTIAND_WITH_MYSQL, CANTIAND_WITH_MYSQL_ST, CANTIAND_WITH_MYSQL_IN_CLUSTER]: + raise Exception( + "chown to %s:%s return: %s%s%s" + % (self.user, self.group, str(ret_code), os.linesep, stderr) + ) + + persist_environment_variable( + "RUN_MODE", g_opts.running_mode.lower(), self.userProfile + ) + + # start mysql and cantian in single process mode with metadata in cantian + if g_opts.running_mode.lower() in [ + CANTIAND_WITH_MYSQL, + CANTIAND_WITH_MYSQL_ST, + CANTIAND_WITH_MYSQL_IN_CLUSTER, + ]: try: # start MySQL in single process mode self.FAILED_POS = self.CREATE_DB_FAILED @@ -3589,13 +4074,15 @@ class Installer: self.start_gss() # create and start threads to start mysql and create db cantiand_thread = threading.Thread(target=self.start_cantiand) - create_db_thread = threading.Thread(target=self.create_db_for_cantian_in_single) + create_db_thread = threading.Thread( + target=self.create_db_for_cantian_in_single + ) cantiand_thread.start() create_db_thread.start() # wait for completion of threads cantiand_thread.join() create_db_thread.join() - log("Both cantiand and database creation processes are complete.",True) + log("Both cantiand and database creation processes are complete.", True) except Exception as err: self.rollBack() logExit(str(err)) @@ -3622,17 +4109,17 @@ class Installer: # wait for mysql process to start log_home = self.cantiandConfigs["LOG_HOME"] run_log = os.path.join(log_home, "run", "cantiand.rlog") - search_string = 'start waiting for db to be open' + search_string = "start waiting for db to be open" cmd = "cat %s | grep '%s'" % (run_log, search_string) is_instance_up = False while not is_instance_up: ret_code, stdout, stderr = _exec_popen(cmd) output = stdout + stderr if ret_code: - log("still waiting for cantian to start",True) + log("still waiting for cantian to start", True) else: is_instance_up = True - log("cantian instance started",True) + log("cantian instance started", True) # wait for 5 seconds time.sleep(5) log("mysqld is running. Proceeding with database creation...", True) @@ -3666,18 +4153,21 @@ class Installer: self._sed_file("dbfiles3", "+vg3", create_database_sql) elif g_opts.use_dbstor: file_name = "create_dbstor_database.sample.sql" - if g_opts.running_mode in [CANTIAND_IN_CLUSTER, CANTIAND_WITH_MYSQL_IN_CLUSTER]: + if g_opts.running_mode in [ + CANTIAND_IN_CLUSTER, + CANTIAND_WITH_MYSQL_IN_CLUSTER, + ]: file_name = "create_dbstor_cluster_database.sample.sql" create_database_sql = os.path.join(sql_file_path, file_name) else: - dbDataPath = os.path.join(self.data, "data").replace('/', '\/') + dbDataPath = os.path.join(self.data, "data").replace("/", "\/") self._sed_file("dbfiles1", dbDataPath, create_database_sql) self._sed_file("dbfiles2", dbDataPath, create_database_sql) self._sed_file("dbfiles3", dbDataPath, create_database_sql) return create_database_sql def _sed_file(self, prefix, replace, file_name): - fixSqlFileCmd = ("sed -i 's/%s/%s/g' %s" % (prefix, replace, file_name)) + fixSqlFileCmd = "sed -i 's/%s/%s/g' %s" % (prefix, replace, file_name) ret_code, _, _ = _exec_popen(fixSqlFileCmd) if ret_code: raise Exception("sed %s failed, replace %s" % (file_name, replace)) @@ -3689,21 +4179,28 @@ class Installer: cmd = "chown -R %s:%s %s; " % (self.user, self.group, self.ssl_path) ret_code, _, stderr = _exec_popen(cmd) if ret_code: - raise Exception("chown to %s:%s return: %s%s%s" - % (self.user, self.group, str(ret_code), os.linesep, stderr)) + raise Exception( + "chown to %s:%s return: %s%s%s" + % (self.user, self.group, str(ret_code), os.linesep, stderr) + ) - def get_ctencrypt_keys(self, skip_execute_sql = False): + def get_ctencrypt_keys(self, skip_execute_sql=False): """Set the config about _FACTOR_KEY and LOCAL_KEY.""" # Generate Key and WorkKey log("Generate encrypted keys.") if g_opts.install_user_privilege == "withoutroot": cmd = "%s/bin/ctencrypt -g" % self.installPath else: - cmd = "su - '%s' -c \"%s/bin/ctencrypt -g \"" % (self.user, self.installPath) + cmd = "su - '%s' -c \"%s/bin/ctencrypt -g \"" % ( + self.user, + self.installPath, + ) ret_code, stdout, stderr = _exec_popen(cmd) if ret_code: - raise OSError("Failed to generate encrypted keys. Error: %s" - % (stderr+os.linesep+stderr)) + raise OSError( + "Failed to generate encrypted keys. Error: %s" + % (stderr + os.linesep + stderr) + ) # Example of output: # eg'Key: XXXXXXXXXXXXXXXXXXXXXXX' @@ -3734,12 +4231,17 @@ class Installer: if g_opts.install_user_privilege == "withoutroot": cmd = "%s/bin/ctencrypt -g -o '%s' " % (self.installPath, f_factor1) else: - cmd = ("su - '%s' -c \"%s/bin/ctencrypt -g -o '%s' \"" - % (self.user, self.installPath, f_factor1)) + cmd = "su - '%s' -c \"%s/bin/ctencrypt -g -o '%s' \"" % ( + self.user, + self.installPath, + f_factor1, + ) ret_code, stdout, stderr = _exec_popen(cmd) if ret_code: - raise OSError("Failed to generate encrypted keys. Error: %s" - % (stderr+os.linesep+stderr)) + raise OSError( + "Failed to generate encrypted keys. Error: %s" + % (stderr + os.linesep + stderr) + ) # Example of output: # eg'Key: XXXXXXXXXXXXXXXXXXXXXXX' @@ -3762,21 +4264,31 @@ class Installer: log("Generate encrypted keys successfully.") return key_, work_key - def encrypt_ssl_key_passwd(self, key_, work_key, ssl_passwd, skip_execute_sql = False): + def encrypt_ssl_key_passwd( + self, key_, work_key, ssl_passwd, skip_execute_sql=False + ): """Encrypt ssl key password with _FACTOR_KEY and LOCAL_KEY.""" log("Encrypt ssl key password.") if g_opts.install_user_privilege == "withoutroot": - cmd = ("""%s/bin/ctencrypt -e AES256 -f %s -k %s """ - % (self.installPath, key_, work_key)) + cmd = """%s/bin/ctencrypt -e AES256 -f %s -k %s """ % ( + self.installPath, + key_, + work_key, + ) else: - cmd = ("su - '%s' -c \"%s/bin/ctencrypt -e AES256" - " -f '%s' -k '%s' \"" - % (self.user, self.installPath, key_, work_key)) + cmd = "su - '%s' -c \"%s/bin/ctencrypt -e AES256" " -f '%s' -k '%s' \"" % ( + self.user, + self.installPath, + key_, + work_key, + ) values = [ssl_passwd, ssl_passwd] ret_code, stdout, stderr = _exec_popen(cmd, values) if ret_code: - raise OSError("Failed to encrypt ssl key password. Error: %s" - % (stderr+os.linesep+stderr)) + raise OSError( + "Failed to encrypt ssl key password. Error: %s" + % (stderr + os.linesep + stderr) + ) # Example of output: # Please enter password to encrypt: @@ -3798,8 +4310,7 @@ class Installer: # Don't set SSL_CA and CTSQL_SSL_CA. # Avoid the need to copy files, env and kernel parameter # from the primary dn when installing the backup dn. - cantian_conf_file = os.path.join(self.data, "cfg", - self.CANTIAND_CONF_FILE) + cantian_conf_file = os.path.join(self.data, "cfg", self.CANTIAND_CONF_FILE) ssl_map = { "SSL_CERT": os.path.join(self.ssl_path, "server.crt"), "SSL_KEY": os.path.join(self.ssl_path, "server.key"), @@ -3828,20 +4339,25 @@ class Installer: log("Set user environment variables about ssl.") try: with open(self.userProfile, "a") as _file: - _file.write("export CTSQL_SSL_CERT=\"%s\"" - % os.path.join(self.ssl_path, "client.crt")) + _file.write( + 'export CTSQL_SSL_CERT="%s"' + % os.path.join(self.ssl_path, "client.crt") + ) _file.write(os.linesep) - _file.write("export CTSQL_SSL_KEY=\"%s\"" - % os.path.join(self.ssl_path, "client.key")) + _file.write( + 'export CTSQL_SSL_KEY="%s"' + % os.path.join(self.ssl_path, "client.key") + ) _file.write(os.linesep) - _file.write("export CTSQL_SSL_MODE=\"required\"") + _file.write('export CTSQL_SSL_MODE="required"') _file.write(os.linesep) - _file.write("export CTSQL_SSL_KEY_PASSWD=\"%s\"" % cipher) + _file.write('export CTSQL_SSL_KEY_PASSWD="%s"' % cipher) _file.write(os.linesep) _file.flush() except IOError as ex: - raise IOError("Failed Set user environment variables about ssl: %s" - % str(ex)) + raise IOError( + "Failed Set user environment variables about ssl: %s" % str(ex) + ) os.environ["CTSQL_SSL_CERT"] = os.path.join(self.ssl_path, "client.crt") os.environ["CTSQL_SSL_KEY"] = os.path.join(self.ssl_path, "client.key") @@ -3858,38 +4374,70 @@ class Installer: if g_opts.install_user_privilege == "withoutroot": if not g_opts.db_passwd: # connect database by sysdba - cmd = ("%s/bin/shutdowndb.sh -h %s -p %s -w -m %s -D %s -T %d" - % (self.installPath, host_ip, self.lsnr_port, - "immediate", self.data, timeout)) + cmd = "%s/bin/shutdowndb.sh -h %s -p %s -w -m %s -D %s -T %d" % ( + self.installPath, + host_ip, + self.lsnr_port, + "immediate", + self.data, + timeout, + ) ret_code, _, stderr = _exec_popen(cmd) else: # connect database by username and password - cmd = ("%s/bin/shutdowndb.sh -h" - " %s -p %s -U %s -m %s -W -D %s -T %d" % - (self.installPath, host_ip, - self.lsnr_port, g_opts.db_user, "immediate", - self.data, timeout)) + cmd = ( + "%s/bin/shutdowndb.sh -h" + " %s -p %s -U %s -m %s -W -D %s -T %d" + % ( + self.installPath, + host_ip, + self.lsnr_port, + g_opts.db_user, + "immediate", + self.data, + timeout, + ) + ) ret_code, _, stderr = _exec_popen(cmd, [g_opts.db_passwd]) else: if not g_opts.db_passwd: # connect database by sysdba - cmd = ("su - '%s' -c \"%s/bin/shutdowndb.sh -h %s" - " -p %s -w -m %s -D %s -T %d \" " - % (self.user, self.installPath, host_ip, self.lsnr_port, - "immediate", self.data, timeout)) + cmd = ( + "su - '%s' -c \"%s/bin/shutdowndb.sh -h %s" + ' -p %s -w -m %s -D %s -T %d " ' + % ( + self.user, + self.installPath, + host_ip, + self.lsnr_port, + "immediate", + self.data, + timeout, + ) + ) ret_code, _, stderr = _exec_popen(cmd) else: # connect database by username and password - cmd = ("su - '%s' -c \" %s/bin/shutdowndb.sh -h" - " %s -p %s -U %s -m %s -W -D %s -T %d \" " % - (self.user, self.installPath, - host_ip, self.lsnr_port, g_opts.db_user, "immediate", - self.data, timeout)) + cmd = ( + "su - '%s' -c \" %s/bin/shutdowndb.sh -h" + ' %s -p %s -U %s -m %s -W -D %s -T %d " ' + % ( + self.user, + self.installPath, + host_ip, + self.lsnr_port, + g_opts.db_user, + "immediate", + self.data, + timeout, + ) + ) ret_code, _, stderr = _exec_popen(cmd, [g_opts.db_passwd]) if ret_code: - raise Exception("Failed to stop database. Error: %s" - % (stderr+os.linesep+stderr)) + raise Exception( + "Failed to stop database. Error: %s" % (stderr + os.linesep + stderr) + ) log("stop cantian instance successfully.") def setSslCert(self): @@ -3913,8 +4461,7 @@ class Installer: key_, work_key = self.get_ctencrypt_keys_and_file() # Get the value of SSL_KEY_PASSWORD from ini and # CTSQL_SSL_KEY_PASSWD from env and set SSL_KEY_PASSWORD into ini - cipher = self.encrypt_ssl_key_passwd(key_, work_key, - ssl_constructor.passwd) + cipher = self.encrypt_ssl_key_passwd(key_, work_key, ssl_constructor.passwd) # 3. Modify cantiand.ini by write # Set the ssl config in cantiand.ini for server @@ -3953,14 +4500,21 @@ class Installer: output: NA """ try: - strCmd = ("find '%s'/admin -type f | xargs chmod %s " - % (self.installPath, CommonValue.MIN_FILE_MODE)) + strCmd = "find '%s'/admin -type f | xargs chmod %s " % ( + self.installPath, + CommonValue.MIN_FILE_MODE, + ) ret_code, _, _ = _exec_popen(strCmd) if ret_code: - print("Change file permission to %s failed." - " Please chmod %s filein directory %s/admin manually." - % (CommonValue.MIN_FILE_MODE, - CommonValue.MIN_FILE_MODE, self.installPath)) + print( + "Change file permission to %s failed." + " Please chmod %s filein directory %s/admin manually." + % ( + CommonValue.MIN_FILE_MODE, + CommonValue.MIN_FILE_MODE, + self.installPath, + ) + ) except Exception as err: logExit(str(err)) @@ -3977,7 +4531,7 @@ class Installer: log("Changing file permission due to security audit.", True) # 1. chmod sql file permission self.chmodInstallSqlfile() - + def set_core_dump_filter(self): """ function: set_core_dump_filter, modify num to support core dump shared memory @@ -4003,12 +4557,22 @@ class Installer: for cantiand_pid in cantiand_pids.split(): cantiand_pid = cantiand_pid.strip() if cantiand_pid is not None and len(cantiand_pid) > 0: - cmd = "echo 0x6f > " + sep_mark + "proc" + sep_mark + str(cantiand_pid) + \ - sep_mark + "coredump_filter" + cmd = ( + "echo 0x6f > " + + sep_mark + + "proc" + + sep_mark + + str(cantiand_pid) + + sep_mark + + "coredump_filter" + ) ret_code, _, stderr = _exec_popen(cmd) if ret_code: - logExit("can not set coredump_filter, command: %s, err: %s" % (cmd, stderr)) - + logExit( + "can not set coredump_filter, command: %s, err: %s" + % (cmd, stderr) + ) + def set_core_dump_filter_mysql(self): """ function: set_core_dump_filter_mysql, modify num to support core dump shared memory @@ -4034,11 +4598,21 @@ class Installer: for mysqld_pid in mysqld_pids.split(): mysqld_pid = mysqld_pid.strip() if mysqld_pid is not None and len(mysqld_pid) > 0: - cmd = "echo 0x6f > " + sep_mark + "proc" + sep_mark + str(mysqld_pid) + \ - sep_mark + "coredump_filter" + cmd = ( + "echo 0x6f > " + + sep_mark + + "proc" + + sep_mark + + str(mysqld_pid) + + sep_mark + + "coredump_filter" + ) ret_code, _, stderr = _exec_popen(cmd) if ret_code: - logExit("can not set coredump_filter, command: %s, err: %s" % (cmd, stderr)) + logExit( + "can not set coredump_filter, command: %s, err: %s" + % (cmd, stderr) + ) ###################################################################### # The main process of installation. @@ -4064,7 +4638,7 @@ class Installer: pass else: self.checkRunner() - self.checkParameter() # add cantiand, cms, gss config parameter check logic in this method + self.checkParameter() # add cantiand, cms, gss config parameter check logic in this method # user and group is right, change owner of the log file to user self.chownLogFile() self.checkOldInstall() @@ -4073,11 +4647,11 @@ class Installer: self.checkDIR() self.checkSHA256() self.generateSslCert() - self.decompressBin() #TODO shiyi check CMS, GSS bin when compile success + self.decompressBin() # TODO shiyi check CMS, GSS bin when compile success self.setUserEnv() self.prepareDataDir() self.prepareMysqlForSingle() - self.InitDbInstance() # init db config, including cantiand, cms, gss, ssl + self.InitDbInstance() # init db config, including cantiand, cms, gss, ssl self.generateReplAuthKeys() log("Successfully Initialize %s instance." % self.instance_name) if self.option == self.INS_ALL: @@ -4095,19 +4669,33 @@ class Installer: def check_parameter_mysql(self): if g_opts.mysql_config_file_path == "unset": - g_opts.mysql_config_file_path = os.path.join(MYSQL_CODE_DIR, "scripts/my.cnf") - log("no mysql config file assigned, set to %s" % g_opts.mysql_config_file_path, True) + g_opts.mysql_config_file_path = os.path.join( + MYSQL_CODE_DIR, "scripts/my.cnf" + ) + log( + "no mysql config file assigned, set to %s" + % g_opts.mysql_config_file_path, + True, + ) real_path = os.path.realpath(g_opts.mysql_config_file_path) if not os.path.isfile(real_path): - logExit("mysql config file {} not existed or it is not a file".format(real_path)) + logExit( + "mysql config file {} not existed or it is not a file".format(real_path) + ) if not self.is_readable(real_path, self.user): - logExit("mysql config file {} is not readable by {}".format(real_path, self.user)) + logExit( + "mysql config file {} is not readable by {}".format( + real_path, self.user + ) + ) g_opts.mysql_config_file_path = real_path cmd = "chown {}:{} {};".format(self.user, self.group, real_path) cmd += "chmod {} {}".format(CommonValue.MAX_FILE_MODE, real_path) ret_code, _, stderr = _exec_popen(cmd) if ret_code: - logExit("Can not set mysql config mode, command: %s, output: %s" % (cmd, stderr)) + logExit( + "Can not set mysql config mode, command: %s, output: %s" % (cmd, stderr) + ) def prepare_mysql_data_dir(self): log("Preparing mysql data dir...", True) @@ -4122,7 +4710,9 @@ class Installer: log("Preparing mysql bin dir...", True) self.clean_dir(MYSQL_BIN_DIR) self.prepareGivenPath(MYSQL_BIN_DIR, True) - cmd = "cp -arf {} {};".format(os.path.join(MYSQL_CODE_DIR, "mysql_bin/mysql/*"), MYSQL_BIN_DIR) + cmd = "cp -arf {} {};".format( + os.path.join(MYSQL_CODE_DIR, "mysql_bin/mysql/*"), MYSQL_BIN_DIR + ) cmd += "cp %s/cfg/osd.cfg %s/bin/osd.cfg;" % (self.data, MYSQL_BIN_DIR) cmd += "chown -R {}:{} {};".format(self.user, self.group, MYSQL_BIN_DIR) @@ -4133,25 +4723,32 @@ class Installer: def set_mysql_env(self): log("Preparing mysql running env...", True) - if 'LD_LIBRARY_PATH' in os.environ: - os.environ['LD_LIBRARY_PATH'] = ("%s:%s:%s:%s:%s" % ( + if "LD_LIBRARY_PATH" in os.environ: + os.environ["LD_LIBRARY_PATH"] = "%s:%s:%s:%s:%s" % ( os.path.join(MYSQL_BIN_DIR, "lib"), os.path.join(MYSQL_CODE_DIR, "cantian_lib"), os.path.join(CTC_LIB_DIR, "cantian_lib"), os.path.join(MYSQL_BIN_DIR, "lib/private"), - os.environ['LD_LIBRARY_PATH'])) + os.environ["LD_LIBRARY_PATH"], + ) else: - os.environ['LD_LIBRARY_PATH'] = ("%s:%s:%s:%s" % ( - os.path.join(MYSQL_BIN_DIR, "lib"), os.path.join(MYSQL_BIN_DIR, "lib/private"), os.path.join( - MYSQL_CODE_DIR, "cantian_lib"), os.path.join(CTC_LIB_DIR, "cantian_lib"))) - cmd = "ldconfig -N %s %s" % (os.path.join(MYSQL_CODE_DIR, "cantian_lib"), os.path.join(MYSQL_BIN_DIR, "lib")) + os.environ["LD_LIBRARY_PATH"] = "%s:%s:%s:%s" % ( + os.path.join(MYSQL_BIN_DIR, "lib"), + os.path.join(MYSQL_BIN_DIR, "lib/private"), + os.path.join(MYSQL_CODE_DIR, "cantian_lib"), + os.path.join(CTC_LIB_DIR, "cantian_lib"), + ) + cmd = "ldconfig -N %s %s" % ( + os.path.join(MYSQL_CODE_DIR, "cantian_lib"), + os.path.join(MYSQL_BIN_DIR, "lib"), + ) ret_code, _, stderr = _exec_popen(cmd) if ret_code: logExit("Can not link mysql lib, command: %s, output: %s" % (cmd, stderr)) def get_jemalloc_path(self): for path in ["/usr/lib", "/usr/lib64", "/usr/local/lib", "/usr/local/lib64"]: - je_path = os.path.join(path, 'libjemalloc.so') + je_path = os.path.join(path, "libjemalloc.so") if os.path.exists(je_path): return je_path return "" @@ -4159,26 +4756,37 @@ class Installer: def start_mysql(self, is_slave_cluster): log("Starting mysqld...", True) if os.path.exists(MYSQL_LOG_FILE) and os.path.isfile(MYSQL_LOG_FILE): - log("Warning: the mysql log file %s should empty for mysqld start" % MYSQL_LOG_FILE, True) + log( + "Warning: the mysql log file %s should empty for mysqld start" + % MYSQL_LOG_FILE, + True, + ) # mysql init # Do not init mysql in slave cluster. je_path = self.get_jemalloc_path() if not is_slave_cluster: # mysql_view_file = self.get_cantian_defs_file() - cmd = "LD_PRELOAD=%s %s --defaults-file=%s --initialize-insecure --datadir=%s \ - --early-plugin-load=\"ctc_ddl_rewriter=ha_ctc.so;ctc=ha_ctc.so;\" \ - --core-file --log-error=%s" % ( - je_path, - os.path.join(MYSQL_BIN_DIR, "bin/mysqld"), - g_opts.mysql_config_file_path, - MYSQL_DATA_DIR, - MYSQL_LOG_FILE) + cmd = ( + 'LD_PRELOAD=%s %s --defaults-file=%s --initialize-insecure --datadir=%s \ + --early-plugin-load="ctc_ddl_rewriter=ha_ctc.so;ctc=ha_ctc.so;" \ + --core-file --log-error=%s' + % ( + je_path, + os.path.join(MYSQL_BIN_DIR, "bin/mysqld"), + g_opts.mysql_config_file_path, + MYSQL_DATA_DIR, + MYSQL_LOG_FILE, + ) + ) if os.getuid() == 0: cmd = "su %s -c '" % self.user + cmd + "'" status, stdout, stderr = _exec_popen(cmd) if status != 0: output = stdout + stderr - raise Exception("Can not init mysqld %s.\nStart cmd: %s.\nOutput: %s" % (self.data, cmd, output)) + raise Exception( + "Can not init mysqld %s.\nStart cmd: %s.\nOutput: %s" + % (self.data, cmd, output) + ) # start mysqld cmd = """LD_PRELOAD=%s %s %s --defaults-file=%s --datadir=%s --plugin-dir=%s \ @@ -4186,43 +4794,78 @@ class Installer: --check_proxy_users=ON --mysql_native_password_proxy_users=ON \ --default-storage-engine=CTC \ --core-file > %s 2>&1 & - """ % (je_path, self.numactl_str, os.path.join(MYSQL_BIN_DIR, "bin/mysqld"), g_opts.mysql_config_file_path, - MYSQL_DATA_DIR, os.path.join(MYSQL_BIN_DIR, "lib/plugin"), MYSQL_LOG_FILE) + """ % ( + je_path, + self.numactl_str, + os.path.join(MYSQL_BIN_DIR, "bin/mysqld"), + g_opts.mysql_config_file_path, + MYSQL_DATA_DIR, + os.path.join(MYSQL_BIN_DIR, "lib/plugin"), + MYSQL_LOG_FILE, + ) if os.getuid() == 0: cmd = "su %s -c '" % self.user + cmd + "'" status, stdout, stderr = _exec_popen(cmd) if status != 0: output = stdout + stderr - raise Exception("Can not start mysqld %s.\nStart cmd: %s.\nOutput: %s" % (self.data, cmd, output)) + raise Exception( + "Can not start mysqld %s.\nStart cmd: %s.\nOutput: %s" + % (self.data, cmd, output) + ) if self.check_has_start_mysqld() == False: - raise Exception("failed or exceed time to start mysqld\nPlease see the log of %s" % MYSQL_LOG_FILE) - + raise Exception( + "failed or exceed time to start mysqld\nPlease see the log of %s" + % MYSQL_LOG_FILE + ) + def start_mysql_with_metadata_in_cantian(self): log("Starting mysqld...", True) if os.path.exists(MYSQL_LOG_FILE) and os.path.isfile(MYSQL_LOG_FILE): - log("Warning: the mysql log file %s should empty for mysqld start" % MYSQL_LOG_FILE, True) + log( + "Warning: the mysql log file %s should empty for mysqld start" + % MYSQL_LOG_FILE, + True, + ) # mysql_view_file = self.get_cantian_defs_file() je_path = self.get_jemalloc_path() - cmd_init_metadata_in_cantian = "LD_PRELOAD=%s %s --defaults-file=%s --initialize-insecure --datadir=%s \ - --early-plugin-load=\"ha_ctc.so\" --core-file --log-error=%s" % ( - je_path, - os.path.join(MYSQL_BIN_DIR, "bin/mysqld"), - g_opts.mysql_config_file_path, - MYSQL_DATA_DIR, - MYSQL_LOG_FILE) + cmd_init_metadata_in_cantian = ( + 'LD_PRELOAD=%s %s --defaults-file=%s --initialize-insecure --datadir=%s \ + --early-plugin-load="ha_ctc.so" --core-file --log-error=%s' + % ( + je_path, + os.path.join(MYSQL_BIN_DIR, "bin/mysqld"), + g_opts.mysql_config_file_path, + MYSQL_DATA_DIR, + MYSQL_LOG_FILE, + ) + ) if os.path.exists("/.dockerenv") and (g_opts.cantian_in_container != "1"): cmd_start_mysqld = """LD_PRELOAD=%s %s --defaults-file=%s --datadir=%s --user=root --skip-innodb \ --early-plugin-load="ha_ctc.so" --core-file >> %s 2>&1 & - """ % (je_path, os.path.join(MYSQL_BIN_DIR, "bin/mysqld"), g_opts.mysql_config_file_path, MYSQL_DATA_DIR, - MYSQL_LOG_FILE) + """ % ( + je_path, + os.path.join(MYSQL_BIN_DIR, "bin/mysqld"), + g_opts.mysql_config_file_path, + MYSQL_DATA_DIR, + MYSQL_LOG_FILE, + ) else: cmd_start_mysqld = """LD_PRELOAD=%s %s %s --defaults-file=%s --datadir=%s --plugin-dir=%s \ --early-plugin-load="ha_ctc.so" \ --core-file >> %s 2>&1 & - """ % (je_path, self.numactl_str, os.path.join(MYSQL_BIN_DIR, "bin/mysqld"), g_opts.mysql_config_file_path, - MYSQL_DATA_DIR, os.path.join(MYSQL_BIN_DIR, "lib/plugin"), MYSQL_LOG_FILE) + """ % ( + je_path, + self.numactl_str, + os.path.join(MYSQL_BIN_DIR, "bin/mysqld"), + g_opts.mysql_config_file_path, + MYSQL_DATA_DIR, + os.path.join(MYSQL_BIN_DIR, "lib/plugin"), + MYSQL_LOG_FILE, + ) if os.getuid() == 0: - cmd_init_metadata_in_cantian = "su %s -c '" % self.user + cmd_init_metadata_in_cantian + "'" + cmd_init_metadata_in_cantian = ( + "su %s -c '" % self.user + cmd_init_metadata_in_cantian + "'" + ) cmd_start_mysqld = "su %s -c '" % self.user + cmd_start_mysqld + "'" # we donot check initialize metadata dir failure since metadata init failed can also succeed to start mysqld. # metadata maybe initialized by other node @@ -4237,7 +4880,10 @@ class Installer: time.sleep(2) log("retry initilize and start mysqld again...", True) if retry_time == 0: - raise Exception("failed or exceed time to start mysqld\nPlease see the log of %s" % MYSQL_LOG_FILE) + raise Exception( + "failed or exceed time to start mysqld\nPlease see the log of %s" + % MYSQL_LOG_FILE + ) def check_has_start_mysqld(self): log_data = "" @@ -4246,7 +4892,10 @@ class Installer: try: with open(MYSQL_LOG_FILE) as file_hanlde: log_data = "".join(file_hanlde.readlines()) - if log_data.find("ready for connections") != -1 and log_data.find("ctc reg instance success") != -1: + if ( + log_data.find("ready for connections") != -1 + and log_data.find("ctc reg instance success") != -1 + ): log("succeed to start mysqld", True) return True except Exception as exerr: @@ -4256,7 +4905,6 @@ class Installer: if retry_time == 0: return False - def install_mysql(self): self.check_parameter_mysql() # If it is not normalized, do not prepare_mysql_data_dir in slave cluster. @@ -4267,20 +4915,26 @@ class Installer: if MYSQL_VERSION == VERSION_DOCKER_META: print("mysql_meta: going to start docker mysql in meta.") self.start_mysql_with_metadata_in_cantian() - #self.execute_mysql_update(self.get_cantian_defs_file()) + # self.execute_mysql_update(self.get_cantian_defs_file()) elif MYSQL_VERSION == VERSION_DOCKER_NOMETA: print("mysql_nometa: developer docker deploy.") self.start_mysql(g_opts.slave_cluster) - #self.execute_mysql_update(self.get_cantian_defs_file()) + # self.execute_mysql_update(self.get_cantian_defs_file()) elif MYSQL_VERSION == VERSION_ENV_META: mysql_plugin_path = os.path.join(MYSQL_BIN_DIR, "lib/plugin") - print("mysql_meta: going to start mysql in meta. bin_dir:%s" % mysql_plugin_path) + print( + "mysql_meta: going to start mysql in meta. bin_dir:%s" + % mysql_plugin_path + ) self.start_mysql_with_metadata_in_cantian() - #if g_opts.node_id == 0: - #self.execute_mysql_update(self.get_cantian_defs_file()) + # if g_opts.node_id == 0: + # self.execute_mysql_update(self.get_cantian_defs_file()) elif MYSQL_VERSION == VERSION_ENV_NOMETA: mysql_plugin_path = os.path.join(MYSQL_BIN_DIR, "lib/plugin") - print("mysql_nometa: going to start mysql in nometa. bin_dir:%s" % mysql_plugin_path) + print( + "mysql_nometa: going to start mysql in nometa. bin_dir:%s" + % mysql_plugin_path + ) self.start_mysql(g_opts.slave_cluster) # self.execute_mysql_update(self.get_cantian_defs_file()) @@ -4289,31 +4943,34 @@ class Installer: # self.set_core_dump_filter_mysql() def checkCreatecantiandefsFile(self): - ''' + """ check it is a file; user has read permission, :return: - ''' + """ # check it is a file if not os.path.isfile(self.create_cantian_defs_file): - raise Exception("Error: %s does not exists or is not a file" - " or permission is not right." - % self.create_cantian_defs_file) + raise Exception( + "Error: %s does not exists or is not a file" + " or permission is not right." % self.create_cantian_defs_file + ) if not checkPath(self.create_cantian_defs_file): - raise Exception("Error: %s file path invalid: " - % self.create_cantian_defs_file) + raise Exception( + "Error: %s file path invalid: " % self.create_cantian_defs_file + ) # if execute user is root, check common user has read permission file_path = os.path.dirname(self.create_cantian_defs_file) # check path of cantian defs sql file that user can cd permission_ok, _ = self.checkPermission(file_path, True) if not permission_ok: - raise Exception("Error: %s can not access %s" - % (self.user, file_path)) + raise Exception("Error: %s can not access %s" % (self.user, file_path)) # check cantian defs file is readable for user if not self.is_readable(self.create_cantian_defs_file, self.user): - raise Exception("Error: %s is not readable for user %s" - % (self.create_cantian_defs_file, self.user)) + raise Exception( + "Error: %s is not readable for user %s" + % (self.create_cantian_defs_file, self.user) + ) # change file to a realpath file self.create_cantian_defs_file = os.path.realpath(self.create_cantian_defs_file) @@ -4329,10 +4986,10 @@ class Installer: file_name = "cantian_defs.sql" create_cantian_defs_file = os.path.join(sql_file_path, file_name) return create_cantian_defs_file - + def get_user_profile(self): strCmd = "" - if(g_opts.install_user_privilege == "withoutroot"): + if g_opts.install_user_privilege == "withoutroot": strCmd = "echo ~" else: strCmd = "su - '%s' -c \"echo ~\"" % self.user @@ -4341,32 +4998,42 @@ class Installer: logExit("Can not get user home.") # Get the profile of user. output = os.path.realpath(os.path.normpath(stdout)) - if (not checkPath(output)): + if not checkPath(output): logExit("The user home directory is invalid.") self.userProfile = os.path.join(output, ".bashrc") print("user profile path:%s" % self.userProfile) - + def set_relaunch_env_for_single(self): # set necessary environment variables self.get_user_profile() self.parseKeyAndValue() - mysql_library_path = ("%s:%s:%s:%s:%s:%s" % ( - os.path.join(self.installPath, "lib"), - os.path.join(self.installPath, "add-ons"), - os.path.join(MYSQL_BIN_DIR, "lib"), - os.path.join(MYSQL_BIN_DIR, "lib/private"), - os.path.join(MYSQL_CODE_DIR, "cantian_lib"), - MYSQL_LIB_OUTPUT_DIR)) - persist_environment_variable("RUN_MODE", g_opts.running_mode.lower(), self.userProfile) - persist_environment_variable("LD_LIBRARY_PATH", mysql_library_path, self.userProfile) + mysql_library_path = "%s:%s:%s:%s:%s:%s" % ( + os.path.join(self.installPath, "lib"), + os.path.join(self.installPath, "add-ons"), + os.path.join(MYSQL_BIN_DIR, "lib"), + os.path.join(MYSQL_BIN_DIR, "lib/private"), + os.path.join(MYSQL_CODE_DIR, "cantian_lib"), + MYSQL_LIB_OUTPUT_DIR, + ) + persist_environment_variable( + "RUN_MODE", g_opts.running_mode.lower(), self.userProfile + ) + persist_environment_variable( + "LD_LIBRARY_PATH", mysql_library_path, self.userProfile + ) persist_environment_variable("CANTIAND_MODE", self.OPEN_MODE, self.userProfile) - persist_environment_variable("CANTIAND_HOME_DIR", self.data.lower(), self.userProfile) - + persist_environment_variable( + "CANTIAND_HOME_DIR", self.data.lower(), self.userProfile + ) + def kill_process(self, process_name): cmd = "pidof %s" % (process_name) ret_code, pids, stderr = _exec_popen(cmd) if ret_code: - print("can not get pid of %s, command: %s, err: %s" % (process_name, cmd, stderr)) + print( + "can not get pid of %s, command: %s, err: %s" + % (process_name, cmd, stderr) + ) return for pid in pids.split(): pid = pid.strip() @@ -4375,31 +5042,41 @@ class Installer: if ret_code: print("Failed to kill process %s. Error: %s" % (cmd, stderr)) else: - print("Process %s with PID %s killed successfully." % (process_name, pid)) - + print( + "Process %s with PID %s killed successfully." % (process_name, pid) + ) + def relaunch_single_process(self): - self.kill_process('mysqld') - self.kill_process('cantiand') + self.kill_process("mysqld") + self.kill_process("cantiand") self.set_relaunch_env_for_single() - relaunch_command = ("""%s --defaults-file=%s --datadir=%s --plugin-dir=%s --early-plugin-load=%s \ + relaunch_command = """%s --defaults-file=%s --datadir=%s --plugin-dir=%s --early-plugin-load=%s \ --check_proxy_users=ON --mysql_native_password_proxy_users=ON \ --default-storage-engine=CTC --core-file > %s 2>&1 &""" % ( - os.path.join(MYSQL_BIN_DIR, "bin/mysqld"), - g_opts.mysql_config_file_path, - MYSQL_DATA_DIR, - os.path.join(MYSQL_BIN_DIR, "lib/plugin"), - CTC_PLUGIN, - MYSQL_LOG_FILE)) + os.path.join(MYSQL_BIN_DIR, "bin/mysqld"), + g_opts.mysql_config_file_path, + MYSQL_DATA_DIR, + os.path.join(MYSQL_BIN_DIR, "lib/plugin"), + CTC_PLUGIN, + MYSQL_LOG_FILE, + ) if os.getuid() == 0: relaunch_command = "su - %s -c '" % self.user + relaunch_command + "'" - print('relaunch mysql using command %s' % relaunch_command) + print("relaunch mysql using command %s" % relaunch_command) status, stdout, stderr = _exec_popen(relaunch_command) if status != 0: output = stdout + stderr - raise Exception("Can not start mysqld %s.\nStart cmd: %s.\nOutput: %s" % (self.data, relaunch_command, output)) + raise Exception( + "Can not start mysqld %s.\nStart cmd: %s.\nOutput: %s" + % (self.data, relaunch_command, output) + ) if not self.check_has_start_mysqld(): - raise Exception("failed or exceed time to start mysqld\nPlease see the log of %s" % MYSQL_LOG_FILE) - + raise Exception( + "failed or exceed time to start mysqld\nPlease see the log of %s" + % MYSQL_LOG_FILE + ) + + def main(): """ main entry @@ -4414,11 +5091,18 @@ def main(): installer = Installer(g_opts.os_user, g_opts.os_group) if g_opts.running_mode == MYSQLD: installer.install_mysql() - elif g_opts.running_mode.lower() in VALID_SINGLE_MYSQL_RUNNING_MODE and g_opts.is_relaunch: + elif ( + g_opts.running_mode.lower() in VALID_SINGLE_MYSQL_RUNNING_MODE + and g_opts.is_relaunch + ): installer.relaunch_single_process() else: installer.install() - log("Install successfully, for more detail information see %s." % g_opts.log_file, True) + log( + "Install successfully, for more detail information see %s." + % g_opts.log_file, + True, + ) except Exception as err: logExit("Install failed: " + str(err)) diff --git a/pkg/install/sql_process.py b/pkg/install/sql_process.py index c8203d05..c90f4ff0 100644 --- a/pkg/install/sql_process.py +++ b/pkg/install/sql_process.py @@ -5,6 +5,7 @@ import sys + sys.dont_write_bytecode = True try: import os @@ -15,12 +16,7 @@ try: except ImportError as e: sys.exit("Unable to import module: %s." % str(e)) -old_core_tables = [ - "COLUMN$", - "INDEX$", - "TABLE$", - "USER$" - ] +old_core_tables = ["COLUMN$", "INDEX$", "TABLE$", "USER$"] old_systables = [ "BACKUP_SET$", @@ -72,12 +68,7 @@ old_systables = [ "SQL_MAP$", ] -new_core_tables = [ - "SYS_COLUMNS", - "SYS_INDEXES", - "SYS_TABLES", - "SYS_USERS" - ] +new_core_tables = ["SYS_COLUMNS", "SYS_INDEXES", "SYS_TABLES", "SYS_USERS"] new_systables = [ "SYS_BACKUP_SETS", @@ -131,10 +122,10 @@ new_systables = [ def isSameSqlList(list1, list2): - ''' + """ two lists of string compare(ignore cases) if they are same - ''' + """ if len(list1) != len(list2): return False @@ -146,19 +137,20 @@ def isSameSqlList(list1, list2): class CreateTableSql(object): - ''' + """ this class process the SQL: CREATE TABLE items - ''' + """ + def __init__(self, sql): - ''' + """ init a create table sql - ''' + """ - sqls = str(sql).split('\n') - if sqls[0][0:2] == '--': + sqls = str(sql).split("\n") + if sqls[0][0:2] == "--": sqls.pop(0) - self.__sql = '\n'.join(sqls) + self.__sql = "\n".join(sqls) self.__items = [] # divide create table sql to 3 parts # 1. before the 'relational_properties' @@ -166,44 +158,45 @@ class CreateTableSql(object): # 3. after the 'relational_properties' # the relational_properties embraced by a pair of parentheses # find the first left parenthesis and it's peer right one - self.__indexOfLeftParenthesis = self.__sql.index('(') - self.__indexOfRightParenthesis =\ - self.find_right_parenthesis(self.__sql, - self.__indexOfLeftParenthesis) - self.__preSql = self.__sql[0:self.__indexOfLeftParenthesis] - self.__postSql = self.__sql[self.__indexOfRightParenthesis+1:] + self.__indexOfLeftParenthesis = self.__sql.index("(") + self.__indexOfRightParenthesis = self.find_right_parenthesis( + self.__sql, self.__indexOfLeftParenthesis + ) + self.__preSql = self.__sql[0 : self.__indexOfLeftParenthesis] + self.__postSql = self.__sql[self.__indexOfRightParenthesis + 1 :] - contentSql = self.__sql[self.__indexOfLeftParenthesis+1: - self.__indexOfRightParenthesis] + contentSql = self.__sql[ + self.__indexOfLeftParenthesis + 1 : self.__indexOfRightParenthesis + ] - for i in contentSql.split('\n'): - if i.strip().strip(','): - self.__items.append(i.strip().strip(',')) + for i in contentSql.split("\n"): + if i.strip().strip(","): + self.__items.append(i.strip().strip(",")) if not self.__items: raise Exception("Syntax Error:\n%s" % self.__sql) def find_right_parenthesis(self, content, index_left_p): - ''' + """ find the corresponding right parenthesis if left parenthesis encountered , level plus 1 : means enter if right parenthesis encountered, if level is zero , found else level minus 1 unexpected : retuen -1 - ''' + """ # verify left index is correct - if content[index_left_p] != '(': + if content[index_left_p] != "(": return -1 level = 0 count = 0 # start from left plus 1 - for i in content[index_left_p+1:]: - if i == '(': + for i in content[index_left_p + 1 :]: + if i == "(": level += 1 - elif i == ')': + elif i == ")": if level == 0: return index_left_p + 1 + count else: @@ -216,20 +209,20 @@ class CreateTableSql(object): return -1 def tableSpace(self): - ''' + """ return tablespace of this table - ''' + """ contents = self.__postSql.split() try: i = contents.index("TABLESPACE") - ts = contents[i+1] + ts = contents[i + 1] return ts except ValueError as e: print(str(e)) return "UNKNOWN" def isSamePreContent(self, other): - ''' + """ self should from new initdb other should from old initdb we split this sql to 3 parts: @@ -237,49 +230,49 @@ class CreateTableSql(object): 2. colum items 3. ) xxx this function compare part1 - ''' + """ pre1 = self.__preSql.split() pre2 = other.__preSql.split() return isSameSqlList(pre1, pre2) def ignore_key_int(self, sql_str, key_str): - ''' + """ ignore the 'key word' + 'int' - ''' + """ content = sql_str.split() for i in range(len(content)): if content[i] == key_str: - if content[i+1].isdigit(): + if content[i + 1].isdigit(): content.pop(i) content.pop(i) - return ' '.join(content) + return " ".join(content) return sql_str def ignore_pctfree(self, sql_str): - ''' + """ ignore the pctfree key word - ''' + """ if sql_str.find("PCTFREE") < 0: return sql_str return self.ignore_key_int(sql_str, "PCTFREE") def ignore_storage(self, sql_str): - ''' + """ ignore the storage key word - ''' + """ index_of_storage = sql_str.find("STORAGE") if index_of_storage < 0: return sql_str - index_of_end = sql_str.find(')', index_of_storage) + index_of_end = sql_str.find(")", index_of_storage) - return sql_str[0:index_of_storage] + sql_str[index_of_end+1:] + return sql_str[0:index_of_storage] + sql_str[index_of_end + 1 :] def isSamePostContent(self, other): - ''' + """ self should from new initdb other should from old initdb we split this sql to 3 parts: @@ -289,16 +282,15 @@ class CreateTableSql(object): this function compare part3 ignore PCTFREE xx (xx is a integer number) ignore STORAGE (INITIAL xxxK) (xxx is a integer number) - ''' + """ storage_info = other.ignore_storage(other.__postSql) - post1 =\ - self.ignore_pctfree(self.ignore_storage(self.__postSql)).split() + post1 = self.ignore_pctfree(self.ignore_storage(self.__postSql)).split() post2 = other.ignore_pctfree(storage_info).split() return isSameSqlList(post1, post2) def incrementItems(self, other): - ''' + """ self should from new initdb other should from old initdb we split this sql to 3 parts: @@ -307,45 +299,47 @@ class CreateTableSql(object): 3. ) xxx this function process part2 output new col items in create sql - ''' + """ increment = [] length1 = len(self.__items) length2 = len(other.__items) if length1 == length2: - ''' + """ __isSame of SqlItem may not correct when sql has ',' use strip(',') to pass the right sql to isSameSqlList - ''' + """ for i in range(length1): - li = self.__items[i].strip().strip(',') - ll = other.__items[i].strip().strip(',') + li = self.__items[i].strip().strip(",") + ll = other.__items[i].strip().strip(",") if not isSameSqlList(li.split(), ll.split()): # output entire sqls for easy debug - raise Exception("Decrement items:\n%s\n%s\n" - % (self.__sql, other.__sql)) + raise Exception( + "Decrement items:\n%s\n%s\n" % (self.__sql, other.__sql) + ) return increment if length1 < length2: # output entire sqls for easy debug - raise Exception("Decrement items:\n%s\n%s\n" - % (self.__sql, other.__sql)) + raise Exception("Decrement items:\n%s\n%s\n" % (self.__sql, other.__sql)) for i in range(length2): - if not isSameSqlList(self.__items[i].strip().strip(',').split(), - other.__items[i].strip().strip(',').split()): + if not isSameSqlList( + self.__items[i].strip().strip(",").split(), + other.__items[i].strip().strip(",").split(), + ): # output entire sqls for easy debug - raise Exception("Decrement items:\n%s\n%s\n" - % (self.__sql, other.__sql)) + raise Exception( + "Decrement items:\n%s\n%s\n" % (self.__sql, other.__sql) + ) for i in range(length2, length1): - increment.append(self.__items[i].strip().strip(',')) + increment.append(self.__items[i].strip().strip(",")) return increment class SqlItem(object): - """ this class manage sql commands items compare 2 sql @@ -353,20 +347,20 @@ class SqlItem(object): """ def __init__(self, sql, flag=False, is_target=False, ignore=False): - ''' + """ init from raw sql some sql use SYS.tablename will strip 'SYS.' prefix - ''' + """ self.__sql = str(sql) self.__flag = flag self.__sql_type = 0 - self.__table_name = '' - self.__index_name = '' - self.__role_name = '' - self.__sequence_name = '' - self.__privilege_name = '' - self.__grantee = '' + self.__table_name = "" + self.__index_name = "" + self.__role_name = "" + self.__sequence_name = "" + self.__privilege_name = "" + self.__grantee = "" self.__is_target = is_target self.__is_ignore = ignore self.__sql_version = 0 @@ -374,52 +368,51 @@ class SqlItem(object): self.__diffs = [] self.__add_table_items = [] - self.nameStyle = '' + self.nameStyle = "" if self.__table_name: table_name = self.__table_name - if table_name[0:4] == 'SYS.': + if table_name[0:4] == "SYS.": table_name = table_name[4:] if table_name in old_systables: - self.nameStyle = 'old' + self.nameStyle = "old" elif table_name in new_systables: - self.nameStyle = 'new' + self.nameStyle = "new" else: - self.nameStyle = '' + self.nameStyle = "" def _replace(self, s, t): - ''' + """ replace sql contents - ''' + """ self.__sql = self.__sql.replace(s, t) def rename2old(self): - ''' + """ rename table name to old style - ''' + """ self.__rename() def rename2new(self): - ''' + """ rename table name to new style - ''' + """ self.__rename(to_new=True) def version(self): - ''' + """ get the version - ''' + """ return self.__sql_version def unique2normal(self): - ''' - ''' + """ """ self.__sql = self.__sql.replace("CREATE UNIQUE", "CREATE") self.__analyse() def originSql(self): - ''' + """ return the origin sql - ''' + """ return self.__sql def index_name(self): @@ -430,17 +423,17 @@ class SqlItem(object): return "" def tableName(self): - ''' + """ return the related table name of this sql - ''' + """ return self.__table_name def name(self): - ''' + """ if create role return role name elif create sequence return sequence name else return table name - ''' + """ if self.isCreateRoleSql(): return self.__role_name elif self.isCreateSeqenceSql(): @@ -449,141 +442,143 @@ class SqlItem(object): return self.__table_name def roleName(self): - ''' + """ return the related role name of this sql - ''' + """ return self.__role_name def genDrop(self): - ''' + """ generate the Drop sql for index sql - ''' + """ if self.isCreateIndexSql(): drop = "DROP INDEX %s on %s" % (self.__index_name, self.__table_name) return drop - return '' + return "" def isIndexSpecialCondition1(self, other): - ''' + """ CREATE UNIQUE INDEX IX_PROCARGU_001 ON SYS_PROC_ARGS(USER#, OBJECT_NAME, PACKAGE, SEQUENCE, OVERLOAD) TABLESPACE SYSTEM CREATE UNIQUE INDEX IX_PROCARGU_001 ON PROC_ARGS $(USER#, OBJECT_NAME, SEQUENCE) TABLESPACE SYSTEM - ''' - sqls1 = ['CREATE UNIQUE INDEX IX_PROCARGU_001 ON SYS_PROC_ARGS' - '(USER#, OBJECT_NAME, PACKAGE, SEQUENCE, OVERLOAD)' - ' TABLESPACE SYSTEM', - 'CREATE UNIQUE INDEX IX_PROCARGU_001 ON PROC_ARGS' - '$(USER#, OBJECT_NAME, PACKAGE, SEQUENCE, OVERLOAD)' - ' TABLESPACE SYSTEM'] - sqls2 = ['CREATE UNIQUE INDEX IX_PROCARGU_001 ON SYS_PROC_ARGS' - '(USER#, OBJECT_NAME, SEQUENCE) TABLESPACE SYSTEM', - 'CREATE UNIQUE INDEX IX_PROCARGU_001 ON PROC_ARGS' - '$(USER#, OBJECT_NAME, SEQUENCE) TABLESPACE SYSTEM'] + """ + sqls1 = [ + "CREATE UNIQUE INDEX IX_PROCARGU_001 ON SYS_PROC_ARGS" + "(USER#, OBJECT_NAME, PACKAGE, SEQUENCE, OVERLOAD)" + " TABLESPACE SYSTEM", + "CREATE UNIQUE INDEX IX_PROCARGU_001 ON PROC_ARGS" + "$(USER#, OBJECT_NAME, PACKAGE, SEQUENCE, OVERLOAD)" + " TABLESPACE SYSTEM", + ] + sqls2 = [ + "CREATE UNIQUE INDEX IX_PROCARGU_001 ON SYS_PROC_ARGS" + "(USER#, OBJECT_NAME, SEQUENCE) TABLESPACE SYSTEM", + "CREATE UNIQUE INDEX IX_PROCARGU_001 ON PROC_ARGS" + "$(USER#, OBJECT_NAME, SEQUENCE) TABLESPACE SYSTEM", + ] if not self.isCreateIndexSql(): return False if not other.isCreateIndexSql(): return False - if self.__index_name != 'IX_PROCARGU_001': + if self.__index_name != "IX_PROCARGU_001": return False if self.originSql() in sqls2 and other.originSql() in sqls1: return True return False def setFlag(self, flag): - ''' + """ this flag indicate whether this item has been fetched or not - ''' + """ self.__flag = flag def isFlagTrue(self): - ''' + """ retrun the internal flag - ''' + """ return self.__flag def isTableSql(self): - ''' + """ return if this sql is a table related sql - ''' + """ if self.__sql_type in [1, 2, 3, 4, 5, 6, 10, 11]: return True return False def isCreateRoleSql(self): - ''' + """ return if this sql is a create role sql - ''' + """ if self.__sql_type in [7]: return True return False def isCreateSeqenceSql(self): - ''' + """ return if this sql is a create sequence sql - ''' + """ if self.__sql_type in [8]: return True return False def isGrantSql(self): - ''' + """ return if this sql is a grant xxx|ALL to role sql - ''' + """ if self.__sql_type in [9]: return True return False def isCreateTableSql(self): - ''' + """ return if this sql is a create table sql - ''' + """ if self.__sql_type in [1, 2, 3]: return True return False def isCreateIndexSql(self): - ''' + """ return if this sql is a create index sql - ''' + """ if self.__sql_type in [4, 5]: return True return False def isCreateNormalIndexSql(self): - ''' + """ return if this sql is a create index sql - ''' + """ if self.__sql_type in [4]: return True return False def isCreateUniqueIndexSql(self): - ''' + """ return if this sql is a create index sql - ''' + """ if self.__sql_type in [5]: return True return False def isAlterSystemSql(self): - ''' + """ return if this sql is a create index sql - ''' + """ if self.__sql_type in [6]: return True return False def isViewDropableSql(self): - ''' - ''' + """ """ if self.__sql_type in [1, 2, 3, 4, 5, 8, 12, 13, 14, 15, 16, 17, 18]: return True return False def isCreateOrReplaceView(self): - ''' - ''' + """ """ if self.__sql_type in [12]: return True return False @@ -593,8 +588,8 @@ class SqlItem(object): if self.__sql_type != 15: return "" - end_symbol = '\n/\n\n' - drop_sql = 'DROP PROCEDURE IF EXISTS %s' % p_name + end_symbol = "\n/\n\n" + drop_sql = "DROP PROCEDURE IF EXISTS %s" % p_name drop_sql += end_symbol drop_sql += "BEGIN\n" drop_sql += " FOR ITEM IN " @@ -607,48 +602,50 @@ class SqlItem(object): return drop_sql def generateDropSql(self): - ''' - ''' + """ """ if not self.isViewDropableSql(): - return '' + return "" if self.__sql_type == 8: - return 'DROP SEQUENCE IF EXISTS %s' % self.__sequence_name + return "DROP SEQUENCE IF EXISTS %s" % self.__sequence_name if self.__sql_type == 12: - return 'DROP VIEW IF EXISTS %s' % self.__table_name + return "DROP VIEW IF EXISTS %s" % self.__table_name if self.__sql_type == 13: - return 'DROP PUBLIC SYNONYM IF EXISTS %s' % self.__table_name + return "DROP PUBLIC SYNONYM IF EXISTS %s" % self.__table_name if self.__sql_type == 14: - return 'DROP SYNONYM IF EXISTS %s' % self.__table_name + return "DROP SYNONYM IF EXISTS %s" % self.__table_name if self.__sql_type == 15: return self.generate_drop_procedure(self.__table_name) if self.__sql_type in [1, 2, 3, 16]: - return 'DROP TABLE IF EXISTS %s' % self.__table_name + return "DROP TABLE IF EXISTS %s" % self.__table_name if self.__sql_type in [4, 5, 17, 18]: - return 'DROP INDEX IF EXISTS %s ON %s' % (self.__index_name, self.__table_name) + return "DROP INDEX IF EXISTS %s ON %s" % ( + self.__index_name, + self.__table_name, + ) def __isTableMatched(self, other): - ''' + """ return if the 2 sqls operate the same table - ''' + """ if self.__table_name == other.__table_name: return True return False def __isIndexMatched(self, other): - ''' + """ return if the 2 sqls operate the same index - ''' + """ if self.__index_name == other.__index_name: return True return False def __isSame(self, other): - ''' + """ return if the 2 sqls are identical - ''' + """ sql1 = self.__sql.split() sql2 = other.__sql.split() @@ -656,18 +653,18 @@ class SqlItem(object): return isSameSqlList(sql1, sql2) def __isAllMatched(self, other): - ''' + """ return if the 2 sqls are identical - ''' + """ return self.__isSame(other) def isMatched(self, other): - ''' + """ return if the 2 sqls create the same table or are identical - ''' + """ if not isinstance(other, SqlItem): return False @@ -693,9 +690,9 @@ class SqlItem(object): return False def __incrementOfCreateTable(self, other): - ''' + """ find increment items of Create Table - ''' + """ createTable1 = CreateTableSql(self.__sql) createTable2 = CreateTableSql(other.__sql) @@ -711,58 +708,61 @@ class SqlItem(object): return createTable1.incrementItems(createTable2) def generateDegradeSql(self, other): - ''' + """ this is generate interface self is from old initdb file other is from new initdb file old and new is for upgrade script actually the old initdb file is newer one other may be None if self is totally new - ''' + """ up = [] self.__diffs = [] if other: if not isinstance(other, SqlItem): - raise Exception('unrecognized object %s' % str(other)) + raise Exception("unrecognized object %s" % str(other)) if self.__isSame(other): - ''' + """ process for: the same sqls - ''' + """ return [] else: - ''' + """ process for: DECREMENT of CREATE TABLE - ''' + """ # CREATE TABLE decrement decrementItems = other.__incrementOfCreateTable(self) for item in decrementItems: - if ' '.join(item.split()).upper().find("NOT NULL") >= 0: + if " ".join(item.split()).upper().find("NOT NULL") >= 0: if item.upper().find("DEFAULT") < 0: - raise Exception("Can not handle" - " decrement sql: %s" % item) + raise Exception( + "Can not handle" " decrement sql: %s" % item + ) else: - ''' + """ process for: new sqls - ''' + """ if self.isCreateTableSql(): up.append("DROP TABLE %s" % self.__table_name) elif self.isCreateIndexSql(): - up.append("DROP INDEX IF EXISTS %s on %s" % (self.__index_name, self.__table_name)) + up.append( + "DROP INDEX IF EXISTS %s on %s" + % (self.__index_name, self.__table_name) + ) elif self.__sql_type in [7]: up.append("DROP ROLE %s" % self.__role_name) elif self.__sql_type in [8]: up.append("DROP SEQUENCE IF EXISTS %s" % self.__sequence_name) elif self.__sql_type in [9]: - up.append("REVOKE %s FROM %s" - % (self.__privilege_name, self.__grantee)) + up.append("REVOKE %s FROM %s" % (self.__privilege_name, self.__grantee)) elif self.__sql_type == 6: pass else: @@ -772,12 +772,12 @@ class SqlItem(object): return up def generateUpgradeSql(self, other): - ''' + """ this is generate interface self is from new initdb file other is from old initdb file other may be None if self is totally new - ''' + """ up = [] extra_sqls = [] @@ -785,51 +785,58 @@ class SqlItem(object): if other: if not isinstance(other, SqlItem): - raise Exception('unrecognized object %s' % str(other)) + raise Exception("unrecognized object %s" % str(other)) if self.__isSame(other): - ''' + """ process for: the same sqls - ''' + """ return [], [] elif self.isCreateIndexSql(): - up.append("DROP INDEX %s on %s" % (other.__index_name, other.__table_name)) + up.append( + "DROP INDEX %s on %s" % (other.__index_name, other.__table_name) + ) up.append(self.__sql) else: - ''' + """ process for: INCREMENT of CREATE TABLE - ''' + """ # CREATE TABLE INCREMENT incrementItems = self.__incrementOfCreateTable(other) for item in incrementItems: - if ' '.join(item.split()).upper().find("NOT NULL") >= 0: + if " ".join(item.split()).upper().find("NOT NULL") >= 0: if item.upper().find("DEFAULT") < 0: - raise Exception("Can not handle" - " increment sql: %s" % item) + raise Exception( + "Can not handle" " increment sql: %s" % item + ) else: sql_content = [i.upper() for i in item.split()] - default_value =\ - sql_content[sql_content.index("DEFAULT")+1] - update_sql = "UPDATE %s SET %s=%s"\ - % (self.__table_name, - sql_content[0], default_value) + default_value = sql_content[ + sql_content.index("DEFAULT") + 1 + ] + update_sql = "UPDATE %s SET %s=%s" % ( + self.__table_name, + sql_content[0], + default_value, + ) extra_sqls.append(update_sql) extra_sqls.append("COMMIT") - upgrade_sql = "ALTER TABLE %s ADD %s"\ - % (self.__table_name, - ' '.join(item.strip(',').split())) - self.__add_table_items.append(item.strip(',')) + upgrade_sql = "ALTER TABLE %s ADD %s" % ( + self.__table_name, + " ".join(item.strip(",").split()), + ) + self.__add_table_items.append(item.strip(",")) up.append(upgrade_sql) else: - ''' + """ process for: new sqls - ''' + """ up.append(self.__sql) self.__diffs.extend(up) @@ -837,15 +844,15 @@ class SqlItem(object): return up, extra_sqls def __str__(self): - ''' + """ for easy output an object - ''' + """ return "\nSQL:\n%s\nFetched:%s\n" % (self.__sql, str(self.__flag)) def __analyse(self): - ''' + """ analyse the syntax of a single sql string - ''' + """ sql = self.__sql @@ -864,75 +871,81 @@ class SqlItem(object): # GRANT -- 9 ########################################### tokens = sql.split() - if tokens[0][0:2] == '--': + if tokens[0][0:2] == "--": if tokens[0][2:].isdigit(): self.__sql_version = int(tokens[0][2:]) tokens.pop(0) - self.__sql = sql[sql.find(tokens[0]):] + self.__sql = sql[sql.find(tokens[0]) :] - if (tokens[0].upper() == 'CREATE' and tokens[1].upper() == 'TABLE'): - if (tokens[2] == 'IF' - and tokens[3] == 'NOT' - and tokens[4] == 'EXISTS'): + if tokens[0].upper() == "CREATE" and tokens[1].upper() == "TABLE": + if tokens[2] == "IF" and tokens[3] == "NOT" and tokens[4] == "EXISTS": self.__sql_type = 16 self.__table_name = tokens[5] else: self.__sql_type = 1 self.__table_name = tokens[2] - elif (tokens[0].upper() == 'CREATE' - and tokens[1].upper() == 'TEMPORARY' - and tokens[2].upper() == 'TABLE'): + elif ( + tokens[0].upper() == "CREATE" + and tokens[1].upper() == "TEMPORARY" + and tokens[2].upper() == "TABLE" + ): self.__sql_type = 2 self.__table_name = tokens[3] - elif (tokens[0].upper() == 'CREATE' - and tokens[1].upper() == 'GLOBAL' - and tokens[2].upper() == 'TEMPORARY' - and tokens[3].upper() == 'TABLE'): + elif ( + tokens[0].upper() == "CREATE" + and tokens[1].upper() == "GLOBAL" + and tokens[2].upper() == "TEMPORARY" + and tokens[3].upper() == "TABLE" + ): self.__sql_type = 3 self.__table_name = tokens[4] - elif (tokens[0].upper() == 'CREATE' - and tokens[1].upper() == 'INDEX' - and tokens[3].upper() == 'ON'): + elif ( + tokens[0].upper() == "CREATE" + and tokens[1].upper() == "INDEX" + and tokens[3].upper() == "ON" + ): self.__sql_type = 4 self.__index_name = tokens[2] - self.__table_name = tokens[4].split('(')[0] - - elif (tokens[0].upper() == 'CREATE' - and tokens[1].upper() == 'UNIQUE' - and tokens[2].upper() == 'INDEX' - and tokens[4].upper() == 'ON'): + self.__table_name = tokens[4].split("(")[0] + + elif ( + tokens[0].upper() == "CREATE" + and tokens[1].upper() == "UNIQUE" + and tokens[2].upper() == "INDEX" + and tokens[4].upper() == "ON" + ): self.__sql_type = 5 self.__index_name = tokens[3] - self.__table_name = tokens[5].split('(')[0] + self.__table_name = tokens[5].split("(")[0] - elif (tokens[0].upper() == 'ALTER' - and tokens[1].upper() == 'SYSTEM' - and tokens[2].upper() == 'LOAD' - and tokens[3].upper() == 'DICTIONARY' - and tokens[4].upper() == 'FOR'): + elif ( + tokens[0].upper() == "ALTER" + and tokens[1].upper() == "SYSTEM" + and tokens[2].upper() == "LOAD" + and tokens[3].upper() == "DICTIONARY" + and tokens[4].upper() == "FOR" + ): self.__sql_type = 6 self.__table_name = tokens[5].strip() - elif (tokens[0].upper() == 'CREATE' - and tokens[1].upper() == 'ROLE'): + elif tokens[0].upper() == "CREATE" and tokens[1].upper() == "ROLE": self.__sql_type = 7 self.__role_name = tokens[2].strip() - elif (tokens[0].upper() == 'CREATE' - and tokens[1].upper() == 'SEQUENCE'): + elif tokens[0].upper() == "CREATE" and tokens[1].upper() == "SEQUENCE": self.__sql_type = 8 self.__sequence_name = tokens[2].strip() - elif (tokens[0].upper() == 'GRANT'): + elif tokens[0].upper() == "GRANT": indexOfTO = 0 endOfIndex = len(tokens) for tok in tokens: - if tok.upper() == 'TO': + if tok.upper() == "TO": indexOfTO = tokens.index(tok) break else: @@ -940,89 +953,104 @@ class SqlItem(object): privileges = tokens[1:indexOfTO] self.__sql_type = 9 - self.__privilege_name = ' '.join(privileges) - self.__role_name = tokens[indexOfTO+1] + self.__privilege_name = " ".join(privileges) + self.__role_name = tokens[indexOfTO + 1] - if tokens[-1].upper() == 'OPTION': - if tokens[-2].upper() == 'ADMIN'\ - and tokens[-3].upper() == 'WITH': + if tokens[-1].upper() == "OPTION": + if tokens[-2].upper() == "ADMIN" and tokens[-3].upper() == "WITH": endOfIndex -= 3 - self.__grantee = ' '.join(tokens[indexOfTO+1:endOfIndex]) + self.__grantee = " ".join(tokens[indexOfTO + 1 : endOfIndex]) - elif (tokens[0].upper() == 'ALTER' - and tokens[1].upper() == 'TABLE' - and self.__is_target): + elif ( + tokens[0].upper() == "ALTER" + and tokens[1].upper() == "TABLE" + and self.__is_target + ): self.__sql_type = 10 self.__table_name = tokens[2].strip() - elif (tokens[0].upper() == 'DROP' - and tokens[1].upper() == 'INDEX' - and self.__is_target): + elif ( + tokens[0].upper() == "DROP" + and tokens[1].upper() == "INDEX" + and self.__is_target + ): self.__sql_type = 11 self.__index_name = tokens[2].strip() - elif (tokens[0].upper() == 'CREATE' - and tokens[1].upper() == 'OR' - and tokens[2].upper() == 'REPLACE' - and tokens[3].upper() == 'VIEW' - and self.__is_target): + elif ( + tokens[0].upper() == "CREATE" + and tokens[1].upper() == "OR" + and tokens[2].upper() == "REPLACE" + and tokens[3].upper() == "VIEW" + and self.__is_target + ): self.__sql_type = 12 self.__table_name = tokens[4].strip() - elif (tokens[0].upper() == 'CREATE' - and tokens[1].upper() == 'OR' - and tokens[2].upper() == 'REPLACE' - and tokens[3].upper() == 'PUBLIC' - and tokens[4].upper() == 'SYNONYM' - and self.__is_target): + elif ( + tokens[0].upper() == "CREATE" + and tokens[1].upper() == "OR" + and tokens[2].upper() == "REPLACE" + and tokens[3].upper() == "PUBLIC" + and tokens[4].upper() == "SYNONYM" + and self.__is_target + ): self.__sql_type = 13 self.__table_name = tokens[5].strip() - elif (tokens[0].upper() == 'CREATE' - and tokens[1].upper() == 'OR' - and tokens[2].upper() == 'REPLACE' - and tokens[3].upper() == 'SYNONYM' - and self.__is_target): + elif ( + tokens[0].upper() == "CREATE" + and tokens[1].upper() == "OR" + and tokens[2].upper() == "REPLACE" + and tokens[3].upper() == "SYNONYM" + and self.__is_target + ): self.__sql_type = 14 self.__table_name = tokens[4].strip() - elif (tokens[0].upper() == 'CREATE' - and tokens[1].upper() == 'OR' - and tokens[2].upper() == 'REPLACE' - and tokens[3].upper() == 'PROCEDURE' - and self.__is_target): + elif ( + tokens[0].upper() == "CREATE" + and tokens[1].upper() == "OR" + and tokens[2].upper() == "REPLACE" + and tokens[3].upper() == "PROCEDURE" + and self.__is_target + ): self.__sql_type = 15 # procedure may have paramters after the name # like : create or replace procedure_name(parameter list) - self.__table_name = tokens[4].strip().split('(')[0] - - elif (tokens[0].upper() == 'CREATE' - and tokens[1].upper() == 'INDEX' - and tokens[2].upper() == 'IF' - and tokens[3].upper() == 'NOT' - and tokens[4].upper() == 'EXISTS' - and tokens[6].upper() == 'ON'): + self.__table_name = tokens[4].strip().split("(")[0] + + elif ( + tokens[0].upper() == "CREATE" + and tokens[1].upper() == "INDEX" + and tokens[2].upper() == "IF" + and tokens[3].upper() == "NOT" + and tokens[4].upper() == "EXISTS" + and tokens[6].upper() == "ON" + ): self.__sql_type = 17 self.__index_name = tokens[5] - self.__table_name = tokens[7].split('(')[0] - - elif (tokens[0].upper() == 'CREATE' - and tokens[1].upper() == 'UNIQUE' - and tokens[2].upper() == 'INDEX' - and tokens[3].upper() == 'IF' - and tokens[4].upper() == 'NOT' - and tokens[5].upper() == 'EXISTS' - and tokens[7].upper() == 'ON'): + self.__table_name = tokens[7].split("(")[0] + + elif ( + tokens[0].upper() == "CREATE" + and tokens[1].upper() == "UNIQUE" + and tokens[2].upper() == "INDEX" + and tokens[3].upper() == "IF" + and tokens[4].upper() == "NOT" + and tokens[5].upper() == "EXISTS" + and tokens[7].upper() == "ON" + ): self.__sql_type = 18 self.__index_name = tokens[6] - self.__table_name = tokens[8].split('(')[0] + self.__table_name = tokens[8].split("(")[0] else: if not self.__is_ignore: @@ -1039,10 +1067,10 @@ class SqlItem(object): return self.__diffs def __rename(self, to_new=False): - ''' + """ rename all table_name with old name - ''' - if self.__table_name.find('SYS.') == 0: + """ + if self.__table_name.find("SYS.") == 0: self.__table_name = self.__table_name[4:] source_tables = new_systables @@ -1054,12 +1082,13 @@ class SqlItem(object): if self.__table_name: if self.__table_name in source_tables: - target_name =\ - target_tables[source_tables.index(self.__table_name)] - self.__sql = self.__sql.replace(' '+self.__table_name, - ' '+target_name) - self.__sql = self.__sql.replace(' SYS.'+self.__table_name, - ' SYS.'+target_name) + target_name = target_tables[source_tables.index(self.__table_name)] + self.__sql = self.__sql.replace( + " " + self.__table_name, " " + target_name + ) + self.__sql = self.__sql.replace( + " SYS." + self.__table_name, " SYS." + target_name + ) self.__table_name = target_name @@ -1067,19 +1096,19 @@ g_later_delete_flag = False class TableGroup(object): - ''' + """ a group include all table sqls: create table create index create SEQUENCE - ''' + """ def __init__(self, create_sql): - ''' + """ init a TableGroup object from create table sql - ''' + """ - self.__table_name = '' + self.__table_name = "" self.__sqls = [] self.__sqls_ct = [] self.__sqls_ci = [] @@ -1087,17 +1116,25 @@ class TableGroup(object): self.__sqls_other = [] self.__unmatched = [] self.__sql_version = create_sql.version() - self.__special_tables = ["SYS_HISTGRAM_ABSTR", "HIST_HEAD$", - "MON_MODS_ALL$", "SYS_DML_STATS"] - self.__sysaux_tables = ["SYS_HISTGRAM_ABSTR", "HIST_HEAD$", - "SYS_HISTGRAM", "HISTGRAM$"] + self.__special_tables = [ + "SYS_HISTGRAM_ABSTR", + "HIST_HEAD$", + "MON_MODS_ALL$", + "SYS_DML_STATS", + ] + self.__sysaux_tables = [ + "SYS_HISTGRAM_ABSTR", + "HIST_HEAD$", + "SYS_HISTGRAM", + "HISTGRAM$", + ] if not isinstance(create_sql, SqlItem): - raise Exception("Unexpected type:%s," - " SqlItem is expected" % type(create_sql)) + raise Exception( + "Unexpected type:%s," " SqlItem is expected" % type(create_sql) + ) if not create_sql.isCreateTableSql(): - raise Exception("Need 'Create table' sql :\n%s" - % create_sql.originSql()) + raise Exception("Need 'Create table' sql :\n%s" % create_sql.originSql()) self.__table_name = create_sql.tableName() self.__sqls.append(create_sql) @@ -1131,45 +1168,45 @@ class TableGroup(object): return self.__modified_unique_index def __str__(self): - ''' + """ this function for easy debug - ''' - s = '' + """ + s = "" for item in self.__sqls: s += str(item) return s def version(self): - ''' + """ get the version - ''' + """ return self.__sql_version def tableName(self): - ''' + """ return the table name - ''' + """ return self.__table_name def name(self): - ''' + """ return the table name - ''' + """ return self.__table_name def append(self, sql): - ''' + """ append to internal lists - ''' + """ - if sql.tableName() not in [self.__table_name, - 'SYS.'+self.__table_name]: - raise Exception("Cannot append %s to table:%s" - % (sql.originSql(), self.__table_name)) + if sql.tableName() not in [self.__table_name, "SYS." + self.__table_name]: + raise Exception( + "Cannot append %s to table:%s" % (sql.originSql(), self.__table_name) + ) self.__sqls.append(sql) @@ -1183,23 +1220,23 @@ class TableGroup(object): self.__sqls_other.append(sql) def setFlag(self, flag): - ''' + """ set flag for fetch - ''' + """ self.__flag = flag def isFlagTrue(self): - ''' + """ getreturn the flag - ''' + """ return self.__flag def rename2old(self): - ''' + """ rename table name to old - ''' + """ for i in self.__sqls: i.rename2old() @@ -1207,14 +1244,13 @@ class TableGroup(object): # rename table group table name if self.__table_name: if self.__table_name in new_systables: - old_name =\ - old_systables[new_systables.index(self.__table_name)] + old_name = old_systables[new_systables.index(self.__table_name)] self.__table_name = old_name def rename2new(self): - ''' + """ rename table name to new - ''' + """ for i in self.__sqls: i.rename2new() @@ -1225,9 +1261,9 @@ class TableGroup(object): self.__table_name = _name def isMatched(self, other): - ''' + """ if table name is the same - ''' + """ if not isinstance(other, TableGroup): return False @@ -1238,9 +1274,9 @@ class TableGroup(object): return False def __fetch(self, sql, update_flag=True): - ''' + """ fetch matched item - ''' + """ matched = None for i in self.__sqls: if i.isMatched(sql): @@ -1251,9 +1287,9 @@ class TableGroup(object): return matched def __isAllMatched(self, other): - ''' + """ check if all items matched - ''' + """ for i in self.__sqls: if not i.isCreateIndexSql(): @@ -1266,9 +1302,9 @@ class TableGroup(object): return True def __isDropIndex(self, other): - ''' + """ shall we drop all index? - ''' + """ if not other: return False @@ -1285,12 +1321,10 @@ class TableGroup(object): return True if self.__table_name in self.__special_tables: - if i.isCreateUniqueIndexSql()\ - and matched.isCreateNormalIndexSql(): + if i.isCreateUniqueIndexSql() and matched.isCreateNormalIndexSql(): i.unique2normal() - if not isSameSqlList(i.originSql().split(), - matched.originSql().split()): + if not isSameSqlList(i.originSql().split(), matched.originSql().split()): if i.isCreateUniqueIndexSql(): self.__modified_unique_index.append(i.index_name()) if i.isIndexSpecialCondition1(matched): @@ -1304,9 +1338,9 @@ class TableGroup(object): return False def __geneDropAllIndexSqls(self): - ''' + """ generate drop index sqls - ''' + """ drops = [] @@ -1317,68 +1351,68 @@ class TableGroup(object): return drops def tableSpace(self): - ''' + """ return the tablespace name - ''' + """ return CreateTableSql(self.__sqls_ct[0].originSql()).tableSpace() def __checkTableSpaceChange(self, other): - ''' + """ check if tablespace changes - ''' + """ tablespace1 = self.tableSpace() tablespace2 = other.tableSpace() - if tablespace1 == 'SYSTEM' and tablespace2 == 'SYSAUX': + if tablespace1 == "SYSTEM" and tablespace2 == "SYSAUX": return True - if tablespace1 == 'SYSAUX' and tablespace2 == 'SYSTEM': + if tablespace1 == "SYSAUX" and tablespace2 == "SYSTEM": return True return False def generateDegradeSql(self, other): - ''' + """ generate degrade sqls - ''' + """ self.__table_diff = [] upgrade_sqls = [] - load_sql = 'ALTER SYSTEM LOAD DICTIONARY FOR %s' % self.__table_name + load_sql = "ALTER SYSTEM LOAD DICTIONARY FOR %s" % self.__table_name if not other: - ''' + """ other is empty or none means all sqls are new - ''' + """ upgrade_sqls.append(load_sql) upgrade_sqls.append("DROP TABLE %s" % self.__table_name) self.__table_diff.append("DROP TABLE %s" % self.__table_name) else: - ''' + """ other is not none check is all matched then check if need drop all index then generate the sqls - ''' + """ if not isinstance(other, TableGroup): raise Exception("Unexpected type %s" % type(other)) - if self.__checkTableSpaceChange(other)\ - and self.__table_name in self.__sysaux_tables: + if ( + self.__checkTableSpaceChange(other) + and self.__table_name in self.__sysaux_tables + ): for i in self.__sqls_ct: i._replace("TABLESPACE SYSAUX", "TABLESPACE SYSTEM") for i in other.__sqls_ct: - i._replace("TABLESPACE SYSAUX", - "TABLESPACE SYSTEM") + i._replace("TABLESPACE SYSAUX", "TABLESPACE SYSTEM") if not self.__isAllMatched(other): - raise Exception("Unmatched item %s" - % self.__unmatched[0].originSql()) + raise Exception("Unmatched item %s" % self.__unmatched[0].originSql()) if other.__isDropIndex(self): @@ -1387,39 +1421,43 @@ class TableGroup(object): global g_later_delete_flag if self.__table_name in self.__special_tables: - self.__table_diff.append("DELETE FROM %s" - % self.__table_name) + self.__table_diff.append("DELETE FROM %s" % self.__table_name) self.__table_diff.append("COMMIT") - if self.__table_name == 'SYS_PROC_ARGS'\ - and other.__special_case1: + if self.__table_name == "SYS_PROC_ARGS" and other.__special_case1: g_later_delete_flag = True - self.__table_diff.append("DELETE FROM SYS_PROCS" - " WHERE TYPE IN ('S','B')") - self.__table_diff.append("DELETE FROM SYS_PROC_ARGS" - " WHERE LENGTH(PACKAGE)>0") + self.__table_diff.append( + "DELETE FROM SYS_PROCS" " WHERE TYPE IN ('S','B')" + ) + self.__table_diff.append( + "DELETE FROM SYS_PROC_ARGS" " WHERE LENGTH(PACKAGE)>0" + ) self.__table_diff.append("COMMIT") - if self.__table_name == 'PROC_ARGS$'\ - and other.__special_case1: + if self.__table_name == "PROC_ARGS$" and other.__special_case1: g_later_delete_flag = True - self.__table_diff.append("DELETE FROM PROC$" - " WHERE TYPE IN ('S','B')") - self.__table_diff.append("DELETE FROM PROC_ARGS$" - " WHERE LENGTH(PACKAGE)>0") + self.__table_diff.append( + "DELETE FROM PROC$" " WHERE TYPE IN ('S','B')" + ) + self.__table_diff.append( + "DELETE FROM PROC_ARGS$" " WHERE LENGTH(PACKAGE)>0" + ) self.__table_diff.append("COMMIT") - if self.__table_name == 'SYS_DEPENDENCIES'\ - and g_later_delete_flag: - self.__table_diff.append("DELETE FROM SYS_DEPENDENCIES" - " WHERE D_TYPE# IN (15, 16)" - " OR P_TYPE# IN (15, 16)") + if self.__table_name == "SYS_DEPENDENCIES" and g_later_delete_flag: + self.__table_diff.append( + "DELETE FROM SYS_DEPENDENCIES" + " WHERE D_TYPE# IN (15, 16)" + " OR P_TYPE# IN (15, 16)" + ) self.__table_diff.append("COMMIT") - if self.__table_name == 'DEPENDENCY$' and g_later_delete_flag: - self.__table_diff.append("DELETE FROM DEPENDENCY$" - " WHERE D_TYPE# IN (15, 16)" - " OR P_TYPE# IN (15, 16)") + if self.__table_name == "DEPENDENCY$" and g_later_delete_flag: + self.__table_diff.append( + "DELETE FROM DEPENDENCY$" + " WHERE D_TYPE# IN (15, 16)" + " OR P_TYPE# IN (15, 16)" + ) self.__table_diff.append("COMMIT") up = self.__geneDropAllIndexSqls() @@ -1458,21 +1496,21 @@ class TableGroup(object): return upgrade_sqls def generateUpgradeSql(self, other): - ''' + """ generate upgrade sqls - ''' + """ self.__table_diff = [] upgrade_sqls = [] extra_sqls = [] - load_sql = 'ALTER SYSTEM LOAD DICTIONARY FOR %s' % self.__table_name + load_sql = "ALTER SYSTEM LOAD DICTIONARY FOR %s" % self.__table_name if not other: - ''' + """ other is empty or none means all sqls are new - ''' + """ self.__new_table.append(self.__table_name) @@ -1484,26 +1522,27 @@ class TableGroup(object): self.__table_diff.extend(up) else: - ''' + """ other is not none check is all matched then check if need drop all index then generate the sqls - ''' + """ if not isinstance(other, TableGroup): raise Exception("Unexpected type %s" % type(other)) - if self.__checkTableSpaceChange(other)\ - and self.__table_name in self.__sysaux_tables: + if ( + self.__checkTableSpaceChange(other) + and self.__table_name in self.__sysaux_tables + ): for i in self.__sqls_ct: i._replace("TABLESPACE SYSAUX", "TABLESPACE SYSTEM") for i in other.__sqls_ct: i._replace("TABLESPACE SYSAUX", "TABLESPACE SYSTEM") if not self.__isAllMatched(other): - raise Exception("Unmatched item %s" - % self.__unmatched[0].originSql()) + raise Exception("Unmatched item %s" % self.__unmatched[0].originSql()) if self.__isDropIndex(other): @@ -1557,13 +1596,14 @@ class TableGroup(object): class ViewGroup(object): - ''' + """ ex: __01(No.) is a view group The No. is unique - ''' + """ + def __init__(self): self.__number = 0 - self.__sqls = '' + self.__sqls = "" def init(self, number): self.__number = number @@ -1582,7 +1622,7 @@ class ViewGroup(object): sql2 = sql2.strip() if len(sql1) != len(sql2): return False - return (sql1 == sql2) + return sql1 == sql2 def is_same(self, other): @@ -1594,16 +1634,16 @@ class ViewGroup(object): class RoleGroup(object): - ''' + """ a role group include sqls: create role grant xxx,xxx|ALL to role - ''' + """ def __init__(self, role_sql): - ''' + """ init a TableGroup object from greate table sql - ''' + """ self.__sqls = [] self.__sqls.append(role_sql) @@ -1613,52 +1653,52 @@ class RoleGroup(object): self.__flag = False def setFlag(self, flag): - ''' + """ set flag for fetch - ''' + """ self.__flag = flag def tableName(self): - ''' + """ empty - ''' - return '' + """ + return "" def roleName(self): - ''' + """ return the role name - ''' + """ return self.__role_name def name(self): - ''' + """ empty - ''' + """ return self.__role_name def rename2old(self): - ''' + """ do nothing (this group do not need rename) - ''' + """ return def rename2new(self): - ''' + """ do nothing (this group do not need rename) - ''' + """ return def append(self, sql): - ''' + """ append to internal lists - ''' + """ self.__sqls.append(sql) def isMatched(self, other): - ''' + """ if table name is the same - ''' + """ if not isinstance(other, RoleGroup): return False @@ -1669,9 +1709,9 @@ class RoleGroup(object): return False def create_role_sql(self): - ''' + """ return the create role sql in this group - ''' + """ for sql in self.__sqls: if sql.isCreateRoleSql(): @@ -1679,9 +1719,9 @@ class RoleGroup(object): return None def grant_sql(self): - ''' + """ get the grant sql in this group - ''' + """ for sql in self.__sqls: if sql.isGrantSql(): @@ -1689,9 +1729,9 @@ class RoleGroup(object): return None def generateDegradeSql(self, other): - ''' + """ generate degrade sqls - ''' + """ self.__diffs = [] if not other: @@ -1710,9 +1750,9 @@ class RoleGroup(object): return self.__diffs def generateUpgradeSql(self, other): - ''' + """ generate upgrade sqls - ''' + """ self.__diffs = [] if not other: @@ -1732,9 +1772,9 @@ class RoleGroup(object): return self.__diffs, [] def isFlagTrue(self): - ''' + """ retrun the internal flag - ''' + """ return self.__flag def last_generated_diff(self): @@ -1742,9 +1782,9 @@ class RoleGroup(object): return self.__diffs def version(self): - ''' + """ get the version - ''' + """ return self.__sql_version @@ -1752,9 +1792,9 @@ class RoleGroup(object): class InitDbSqls(object): def __init__(self): - ''' + """ init item list - ''' + """ self.__all_items = [] self.__style = set() self.__fast_ref = {} @@ -1762,53 +1802,53 @@ class InitDbSqls(object): self.__last_version = None def __str__(self): - ''' + """ this function for easy debug - ''' - s = '' + """ + s = "" for item in self.__all_items: s += str(item) return s def get_last_version(self): - ''' + """ get last version - ''' + """ return self.__last_version def getStyle(self): - ''' + """ return table name style - ''' - if 'old' in self.__style: - return 'old' + """ + if "old" in self.__style: + return "old" else: - return 'new' + return "new" def rename2old(self): - ''' + """ rename all item to old style - ''' + """ for i in self.__all_items: i.rename2old() def rename2new(self): - ''' + """ rename all item to new style - ''' + """ for i in self.__all_items: i.rename2new() def init(self, sql_file): - ''' + """ init from a initdb.sql like file - ''' - with open(sql_file, 'r') as fp: + """ + with open(sql_file, "r") as fp: content = fp.read() # translate to unix format - content = content.replace('\r\n', '\n') - cmds = content.split('/') + content = content.replace("\r\n", "\n") + cmds = content.split("/") for cmd in cmds: if cmd.strip(): self.__append(cmd.strip()) @@ -1817,18 +1857,20 @@ class InitDbSqls(object): raise Exception("Error: mixed table name") def __iter__(self): - ''' + """ iterator of item list - ''' + """ + def item_iter(): for item in self.__all_items: yield item + return item_iter() def __check_version(self, sql_item): - ''' + """ check table, role, sequence 's version - ''' + """ if self.__last_version is None: self.__last_version = sql_item.version() @@ -1840,9 +1882,9 @@ class InitDbSqls(object): raise Exception("Unsupported versions! \n%s" % str(sql_item)) def __append(self, sql): - ''' + """ append sql item to item list - ''' + """ item = SqlItem(sql) @@ -1851,7 +1893,7 @@ class InitDbSqls(object): group = TableGroup(item) self.__all_items.append(group) self.__fast_ref[item.tableName()] = group - self.__fast_ref['SYS.'+item.tableName()] = group + self.__fast_ref["SYS." + item.tableName()] = group self.__check_version(item) elif item.isTableSql(): @@ -1860,8 +1902,9 @@ class InitDbSqls(object): self.__fast_ref[item.tableName()].append(item) except Exception as e: - raise Exception("%s before create table %s" - % (item.originSql(), str(e))) + raise Exception( + "%s before create table %s" % (item.originSql(), str(e)) + ) elif item.isCreateRoleSql(): @@ -1876,8 +1919,7 @@ class InitDbSqls(object): self.__fast_ref_role[item.roleName()].append(item) except Exception as e: - raise Exception("%s before create role %s" - % (item.originSql(), str(e))) + raise Exception("%s before create role %s" % (item.originSql(), str(e))) else: self.__all_items.append(item) @@ -1887,9 +1929,9 @@ class InitDbSqls(object): self.__style.add(item.nameStyle) def fetch(self, sql): - ''' + """ fetch a sql from item list - ''' + """ for item in self.__all_items: if item.isMatched(sql): @@ -1898,9 +1940,9 @@ class InitDbSqls(object): return None def checkUnMatchedItem(self): - ''' + """ check if some item have not been fetched - ''' + """ unmatched = [] for item in self.__all_items: @@ -1909,51 +1951,53 @@ class InitDbSqls(object): if unmatched: for item in unmatched: - print('Error unmatched: %s' % str(item)) + print("Error unmatched: %s" % str(item)) raise Exception("Some item(s) unmatched!") class InitViewSqls(object): def __init__(self): - ''' + """ init item list - ''' + """ self.__all_items = [] def init(self, sql_file): - ''' + """ init from a initdb.sql like file - ''' - with open(sql_file, 'r') as fp: + """ + with open(sql_file, "r") as fp: content = fp.read() # translate to unix format - content = content.replace('\r\n', '\n') - cmds = content.split('\n/') + content = content.replace("\r\n", "\n") + cmds = content.split("\n/") for cmd in cmds: if cmd.strip(): self.__append(cmd.strip()) def __iter__(self): - ''' + """ iterator of item list - ''' + """ + def item_iter(): for item in self.__all_items: yield item + return item_iter() def __append(self, sql): - ''' + """ append sql item to item list - ''' + """ item = SqlItem(sql, is_target=True, ignore=True) self.__all_items.append(item) def fetch(self, sql): - ''' + """ fetch a sql from item list - ''' + """ for item in self.__all_items: if item.isMatched(sql): @@ -1969,36 +2013,33 @@ class InitIncreaseView(object): self.__path = "" def add_group(self, viewGroup): - if viewGroup.all_sqls().rstrip() != '': + if viewGroup.all_sqls().rstrip() != "": self.__numberList.append(viewGroup.number()) self.__viewGroups.append(viewGroup) elif viewGroup.number() != 0: gnum = viewGroup.number() - raise Exception("In %s, the %d module is empty!" % (self.__path, - gnum)) + raise Exception("In %s, the %d module is empty!" % (self.__path, gnum)) def init(self, sqlfile): self.__path = sqlfile - with open(sqlfile, 'r') as fp: + with open(sqlfile, "r") as fp: line = fp.readline() if line.rstrip() != "--01": raise Exception("The %s should start with --01 !" % sqlfile) viewgroup = ViewGroup() - while(line): - if line.rstrip()[0:2] == '--' and line.rstrip()[2:].isdigit(): + while line: + if line.rstrip()[0:2] == "--" and line.rstrip()[2:].isdigit(): self.add_group(viewgroup) number = int(line[2:]) viewgroup = ViewGroup() viewgroup.init(number) else: - line = line.replace('\r\n', '\n') + line = line.replace("\r\n", "\n") viewgroup.add_sql(line) line = fp.readline() self.add_group(viewgroup) - - def get_all_group(self): return self.__viewGroups @@ -2038,12 +2079,13 @@ def writeFile(filename, contents): write file """ - with open(filename, 'w') as fp: + with open(filename, "w") as fp: fp.write(contents) - cmd = 'chmod %s %s' % (DefaultValue.KEY_FILE_MODE, filename) - p = subprocess.Popen(['bash', '-c', cmd], - stdout=subprocess.PIPE, stderr=subprocess.PIPE) + cmd = "chmod %s %s" % (DefaultValue.KEY_FILE_MODE, filename) + p = subprocess.Popen( + ["bash", "-c", cmd], stdout=subprocess.PIPE, stderr=subprocess.PIPE + ) (stdoutdata, stderrdata) = p.communicate() status = p.returncode output = stdoutdata + stderrdata @@ -2053,8 +2095,7 @@ def writeFile(filename, contents): output = output.decode() if status != 0: - raise Exception("Can not change %s mode.\nError:%s" - % (filename, output)) + raise Exception("Can not change %s mode.\nError:%s" % (filename, output)) def generateRenameSqlFile(outputFilePath, appeared, is_rename2new=True): @@ -2067,7 +2108,7 @@ def generateRenameSqlFile(outputFilePath, appeared, is_rename2new=True): upgradeRename = os.path.join(outputFilePath, "upgradeRename.sql") - tables_name = 'SYS_TABLES' + tables_name = "SYS_TABLES" if is_rename2new: s_core_tables = old_core_tables t_core_tables = new_core_tables @@ -2082,21 +2123,25 @@ def generateRenameSqlFile(outputFilePath, appeared, is_rename2new=True): updateList = [] for item in s_core_tables: - updateNameSql = 'UPDATE %s SET NAME=\'%s\' WHERE NAME=\'%s\''\ - % (tables_name, - t_core_tables[s_core_tables.index(item)], item) + updateNameSql = "UPDATE %s SET NAME='%s' WHERE NAME='%s'" % ( + tables_name, + t_core_tables[s_core_tables.index(item)], + item, + ) updateList.append(updateNameSql) - updateList.append('ALTER SYSTEM LOAD DICTIONARY FOR %s' - % tables_name) + updateList.append("ALTER SYSTEM LOAD DICTIONARY FOR %s" % tables_name) for item in s_systables: if item not in s_core_tables: if item in appeared: - updateNameSql = 'ALTER TABLE %s rename to %s'\ - % (item, t_systables[s_systables.index(item)]) + updateNameSql = "ALTER TABLE %s rename to %s" % ( + item, + t_systables[s_systables.index(item)], + ) updateList.append(updateNameSql) - updateNameSql = 'ALTER SYSTEM LOAD DICTIONARY FOR %s'\ - % (t_systables[s_systables.index(item)]) + updateNameSql = "ALTER SYSTEM LOAD DICTIONARY FOR %s" % ( + t_systables[s_systables.index(item)] + ) updateList.append(updateNameSql) if updateList: @@ -2108,6 +2153,7 @@ class CmdOption(object): """ :define global parameters """ + def __init__(self): """ initial assignment @@ -2131,37 +2177,37 @@ g_opts = CmdOption() def usage(): """ -sql_process.py is a utility to upgrade. - -Usage: - python sql_process.py -? | --help - python sql_process.py -t generate --new-initdb=NEW_INITDB_SQL_FILE - --old-initdb=OLD_INITDB_SQL_FILE --outdir=OUT_DIR - --sqls-path=SQLS_DIR [--degrade] - python sql_process.py -t gen-view --new=NEW_SQL_FILE --old=OLD_SQL_FILE - --outfile=OUT_FILE - python sql_process.py -t gen-dict --initdb=INITDB_SQL_FILE - --outdir=OUT_DIR - python sql_process.py -t check-initdb --old-initdb=LOW_INITDB_SQL_FILE - --new-initdb=HIGH_INITDB_SQL_FILE [--adjacent] - python sql_process.py -t check-whitelist --sqls-path=SQLS_DIR - python sql_process.py -t gen-increase-view --new=NEW_SQL_FILE - --old=OLD_SQL_FILE --outfile=OUT_FILE - -General options: - -t Specified the action. - --new-initdb Specified new initdb file. - --old-initdb Specified old initdb file. - --initdb Specified initdb file. - --outdir Specified output directory. - --new Specified new sql file. - --old Specified old sql file. - --outfile Specified output file. - --sqls-path Specified upgrade and degrade sqls' path - --degrade From high version to low version. - --adjacent The two initdb.sql are adjacent versions. - -?, --help Show help information for this utility, - and exit the command line mode. + sql_process.py is a utility to upgrade. + + Usage: + python sql_process.py -? | --help + python sql_process.py -t generate --new-initdb=NEW_INITDB_SQL_FILE + --old-initdb=OLD_INITDB_SQL_FILE --outdir=OUT_DIR + --sqls-path=SQLS_DIR [--degrade] + python sql_process.py -t gen-view --new=NEW_SQL_FILE --old=OLD_SQL_FILE + --outfile=OUT_FILE + python sql_process.py -t gen-dict --initdb=INITDB_SQL_FILE + --outdir=OUT_DIR + python sql_process.py -t check-initdb --old-initdb=LOW_INITDB_SQL_FILE + --new-initdb=HIGH_INITDB_SQL_FILE [--adjacent] + python sql_process.py -t check-whitelist --sqls-path=SQLS_DIR + python sql_process.py -t gen-increase-view --new=NEW_SQL_FILE + --old=OLD_SQL_FILE --outfile=OUT_FILE + + General options: + -t Specified the action. + --new-initdb Specified new initdb file. + --old-initdb Specified old initdb file. + --initdb Specified initdb file. + --outdir Specified output directory. + --new Specified new sql file. + --old Specified old sql file. + --outfile Specified output file. + --sqls-path Specified upgrade and degrade sqls' path + --degrade From high version to low version. + --adjacent The two initdb.sql are adjacent versions. + -?, --help Show help information for this utility, + and exit the command line mode. """ print(usage.__doc__) @@ -2174,68 +2220,96 @@ def parseCommandLine(): """ try: - (opts, args) = getopt.getopt(sys.argv[1:], "t:?", ["new-initdb=", - "old-initdb=", - "initdb=", - "outdir=", - "new=", - "old=", - "outfile=", - "sqls-path=", - "degrade", - "adjacent", - "help"]) + (opts, args) = getopt.getopt( + sys.argv[1:], + "t:?", + [ + "new-initdb=", + "old-initdb=", + "initdb=", + "outdir=", + "new=", + "old=", + "outfile=", + "sqls-path=", + "degrade", + "adjacent", + "help", + ], + ) except Exception as e: print(str(e)) sys.exit(2) # The error is exited if an illegal parameter appears - if (len(args) > 0): + if len(args) > 0: raise Exception("Unknown parameter [%s]." % args[0]) # the command must contains parameters - if (len(opts) == 0): + if len(opts) == 0: raise Exception("Missing required parameters.") # the list of invalid characters - VALUE_CHECK_LIST = ["|", ";", "&", "$", "<", ">", - "`", "\\", "'", "\"", "{", "}", - "(", ")", "[", "]", "~", "*", "?", "!", "\n"] + VALUE_CHECK_LIST = [ + "|", + ";", + "&", + "$", + "<", + ">", + "`", + "\\", + "'", + '"', + "{", + "}", + "(", + ")", + "[", + "]", + "~", + "*", + "?", + "!", + "\n", + ] # start to parse the parameter value - for (key, value) in opts: + for key, value in opts: for role in VALUE_CHECK_LIST: if value.find(role) >= 0: - raise Exception("The value of parameter [%s]" - " contains invalid characters:" - " '%s'" % (key, role)) + raise Exception( + "The value of parameter [%s]" + " contains invalid characters:" + " '%s'" % (key, role) + ) # output version information and exit - if (key == "--help" or key == "-?"): + if key == "--help" or key == "-?": usage() sys.exit(0) # get parameter value - elif (key == "-t"): + elif key == "-t": g_opts.action = value - elif (key == "--new-initdb"): + elif key == "--new-initdb": g_opts.new_init_db = os.path.realpath(value) - elif (key == "--old-initdb"): + elif key == "--old-initdb": g_opts.old_init_db = os.path.realpath(value) - elif (key == "--initdb"): + elif key == "--initdb": g_opts.init_db = os.path.realpath(value) - elif (key == "--outdir"): + elif key == "--outdir": if value: g_opts.generate_path = os.path.realpath(value) - elif (key == "--new"): + elif key == "--new": g_opts.new = os.path.realpath(value) - elif (key == "--old"): + elif key == "--old": g_opts.old = os.path.realpath(value) - elif (key == "--outfile"): + elif key == "--outfile": if value: g_opts.generate_file = os.path.realpath(value) - elif (key == "--sqls-path"): + elif key == "--sqls-path": if value: g_opts.sqls_path = os.path.realpath(value) - elif (key == "--degrade"): + elif key == "--degrade": g_opts.isDegrade = True - elif (key == "--adjacent"): + elif key == "--adjacent": g_opts.isAdjacent = True else: raise Exception("unknown paramter: [-%s]." % key) @@ -2254,12 +2328,12 @@ def checkFile(parame, filename): raise Exception("less necessary parameter [%s]." % parame) # file does not exist if not os.path.exists(filename): - raise Exception("The value of necessary parameter" - " [%s] is not exists." % parame) + raise Exception( + "The value of necessary parameter" " [%s] is not exists." % parame + ) # not a file type elif not os.path.isfile(filename): - raise Exception("The value of parameter [%s]" - " is not file type." % parame) + raise Exception("The value of parameter [%s]" " is not file type." % parame) def parseParams(): @@ -2273,24 +2347,23 @@ def parseParams(): checkFile("--old-initdb", g_opts.old_init_db) if g_opts.generate_path: if not os.path.exists(g_opts.generate_path): - raise Exception("The value of parameter" - " [--outdir] is not exists.") + raise Exception("The value of parameter" " [--outdir] is not exists.") if os.path.isfile(g_opts.generate_path): - raise Exception("The value of parameter" - " [--outdir] is not dirctory.") + raise Exception("The value of parameter" " [--outdir] is not dirctory.") else: raise Exception("The value of parameter [--outdir] is necessary.") if g_opts.sqls_path: if not os.path.exists(g_opts.sqls_path): - raise Exception("The value of parameter" - " [--sqls-path] is not exists.") + raise Exception( + "The value of parameter" " [--sqls-path] is not exists." + ) if os.path.isfile(g_opts.sqls_path): - raise Exception("The value of parameter" - " [--sqls-path] is not dirctory.") + raise Exception( + "The value of parameter" " [--sqls-path] is not dirctory." + ) else: - raise Exception("The value of parameter" - " [--sqls-path] is necessary.") + raise Exception("The value of parameter" " [--sqls-path] is necessary.") elif g_opts.action == "gen-view": if g_opts.new: @@ -2300,11 +2373,11 @@ def parseParams(): if g_opts.generate_file: if os.path.exists(g_opts.generate_path): if not os.path.isfile(g_opts.generate_path): - raise Exception("The value of parameter" - " [--outfile] is not a file.") + raise Exception( + "The value of parameter" " [--outfile] is not a file." + ) else: - raise Exception("The value of parameter" - " [--outfile] is necessary.") + raise Exception("The value of parameter" " [--outfile] is necessary.") elif g_opts.action == "gen-dict": @@ -2312,11 +2385,9 @@ def parseParams(): if g_opts.generate_path: if not os.path.exists(g_opts.generate_path): - raise Exception("The value of parameter" - " [--outdir] is not exists.") + raise Exception("The value of parameter" " [--outdir] is not exists.") if os.path.isfile(g_opts.generate_path): - raise Exception("The value of parameter" - " [--outdir] is not dirctory.") + raise Exception("The value of parameter" " [--outdir] is not dirctory.") else: raise Exception("The value of parameter [--outdir] is necessary.") @@ -2329,34 +2400,33 @@ def parseParams(): if g_opts.sqls_path: if not os.path.exists(g_opts.sqls_path): - raise Exception("The value of parameter" - " [--sqls-path] is not exists.") + raise Exception( + "The value of parameter" " [--sqls-path] is not exists." + ) if os.path.isfile(g_opts.sqls_path): - raise Exception("The value of parameter" - " [--sqls-path] is not dirctory.") + raise Exception( + "The value of parameter" " [--sqls-path] is not dirctory." + ) else: - raise Exception("The value of parameter" - " [--sqls-path] is necessary.") + raise Exception("The value of parameter" " [--sqls-path] is necessary.") elif g_opts.action == "gen-increase-view": if g_opts.new: checkFile("--new", g_opts.new) else: - raise Exception("The value of parameter" - " [--new] is necessary.") + raise Exception("The value of parameter" " [--new] is necessary.") if g_opts.old: checkFile("--old", g_opts.old) else: - raise Exception("The value of parameter" - " [--old] is necessary.") + raise Exception("The value of parameter" " [--old] is necessary.") if g_opts.generate_file: if os.path.exists(g_opts.generate_path): if not os.path.isfile(g_opts.generate_path): - raise Exception("The value of parameter" - " [--outfile] is not a file.") + raise Exception( + "The value of parameter" " [--outfile] is not a file." + ) else: - raise Exception("The value of parameter" - " [--outfile] is necessary.") + raise Exception("The value of parameter" " [--outfile] is necessary.") else: raise Exception("The value of parameter [-t] is illegal.") @@ -2383,7 +2453,7 @@ def generate_1_0_degrade(zero_init_db, ver1_init_db): it = iter(ver1_init_db) new_init_db = zero_init_db - output_path = os.path.join(g_opts.generate_path, '01') + output_path = os.path.join(g_opts.generate_path, "01") if os.path.exists(output_path): for _root, _dirs, _files in os.walk(output_path, topdown=False): for fname in _files: @@ -2407,8 +2477,10 @@ def generate_1_0_degrade(zero_init_db, ver1_init_db): table_update = i.last_generated_diff() if table_update: outputSqlString = "\n/\n\n".join(table_update) + "\n/\n" - writeFile(os.path.join(g_opts.generate_path, '01', - name+'_degrade_1.sql'), outputSqlString) + writeFile( + os.path.join(g_opts.generate_path, "01", name + "_degrade_1.sql"), + outputSqlString, + ) def generate_0_1_upgrade(zero_init_db, ver1_init_db): @@ -2418,7 +2490,7 @@ def generate_0_1_upgrade(zero_init_db, ver1_init_db): it = iter(ver1_init_db) old_init_db = zero_init_db - output_path = os.path.join(g_opts.generate_path, '01') + output_path = os.path.join(g_opts.generate_path, "01") if os.path.exists(output_path): for _root, _dirs, _files in os.walk(output_path, topdown=False): for fname in _files: @@ -2443,8 +2515,10 @@ def generate_0_1_upgrade(zero_init_db, ver1_init_db): table_update = i.last_generated_diff() if table_update: outputSqlString = "\n/\n\n".join(table_update) + "\n/\n" - writeFile(os.path.join(g_opts.generate_path, '01', - name+'_upgrade_1.sql'), outputSqlString) + writeFile( + os.path.join(g_opts.generate_path, "01", name + "_upgrade_1.sql"), + outputSqlString, + ) def generate_degrade(): @@ -2464,7 +2538,7 @@ def generate_degrade(): new_init_db.init(g_opts.new_init_db) styleB = new_init_db.getStyle() - if styleA == 'new' and styleB == 'old': + if styleA == "new" and styleB == "old": new_init_db.rename2new() old_last_version = old_init_db.get_last_version() @@ -2473,36 +2547,35 @@ def generate_degrade(): if new_last_version == 0 and old_last_version >= 1: generate_01_degrade = True - initdb_01_file = os.path.join(g_opts.sqls_path, 'initdb_01.sql') + initdb_01_file = os.path.join(g_opts.sqls_path, "initdb_01.sql") if not os.path.exists(initdb_01_file): raise Exception("Can not find file %s" % initdb_01_file) initdb_01 = InitDbSqls() initdb_01.init(initdb_01_file) - if styleA == 'new': + if styleA == "new": initdb_01.rename2new() if generate_01_degrade: generate_1_0_degrade(new_init_db, initdb_01) - if styleA == 'old' or styleB == 'new': + if styleA == "old" or styleB == "new": systable_new = [] else: systable_new = get_system_table_names(old_init_db) - generateRenameSqlFile(g_opts.generate_path, - systable_new, is_rename2new=False) + generateRenameSqlFile(g_opts.generate_path, systable_new, is_rename2new=False) it = iter(old_init_db) - output_content = '' - output_file_name = os.path.join(g_opts.generate_path, 'upgradeFile.sql') + output_content = "" + output_file_name = os.path.join(g_opts.generate_path, "upgradeFile.sql") for i in it: - output_content = generateDegreFiles(i, new_init_db, - initdb_01, old_systables, - new_systables, output_content) + output_content = generateDegreFiles( + i, new_init_db, initdb_01, old_systables, new_systables, output_content + ) sql = "ALTER SYSTEM INIT DICTIONARY\n/\n\n" output_content += sql writeFile(output_file_name, output_content) @@ -2513,25 +2586,27 @@ def generate_degrade(): def checkDegredFile(item_01, name): if item_01: - degrade_01_file = os.path.join(g_opts.generate_path, '01', - name+'_degrade_1.sql') + degrade_01_file = os.path.join( + g_opts.generate_path, "01", name + "_degrade_1.sql" + ) if not os.path.exists(degrade_01_file): - raise Exception("Cannot find degrade file for" - " version_1 to version_0 %s" - % degrade_01_file) + raise Exception( + "Cannot find degrade file for" + " version_1 to version_0 %s" % degrade_01_file + ) else: - degrade_01_file = os.path.join(g_opts.sqls_path, - name+'_degrade_1.sql') + degrade_01_file = os.path.join(g_opts.sqls_path, name + "_degrade_1.sql") if not os.path.exists(degrade_01_file): - raise Exception("Cannot find degrade file for" - " version_1 to version_0 %s" - % degrade_01_file) - upgrade_01_file = os.path.join(g_opts.sqls_path, - name+'_degrade_1.sql') + raise Exception( + "Cannot find degrade file for" + " version_1 to version_0 %s" % degrade_01_file + ) + upgrade_01_file = os.path.join(g_opts.sqls_path, name + "_degrade_1.sql") if not os.path.exists(upgrade_01_file): - raise Exception("Cannot find upgrade file for" - " version_0 to version_1 %s" - % upgrade_01_file) + raise Exception( + "Cannot find upgrade file for" + " version_0 to version_1 %s" % upgrade_01_file + ) return degrade_01_file @@ -2550,16 +2625,15 @@ def generateDegreFiles(i, ndb, i01, osystables, nsystables, output): if not low_item: degrade_01_file = checkDegredFile(item_01, name) else: - degrade_01_file = os.path.join(g_opts.generate_path, '01', - name+'_degrade_1.sql') + degrade_01_file = os.path.join( + g_opts.generate_path, "01", name + "_degrade_1.sql" + ) if os.path.exists(degrade_01_file): diff_files.append(degrade_01_file) - start_ver = max(2, low_ver+1) - for t in range(start_ver, high_ver+1): - de_file = os.path.join(g_opts.sqls_path, - name+'_degrade_'+str(t)+'.sql') - up_file = os.path.join(g_opts.sqls_path, - name+'_upgrade_'+str(t)+'.sql') + start_ver = max(2, low_ver + 1) + for t in range(start_ver, high_ver + 1): + de_file = os.path.join(g_opts.sqls_path, name + "_degrade_" + str(t) + ".sql") + up_file = os.path.join(g_opts.sqls_path, name + "_upgrade_" + str(t) + ".sql") if not os.path.exists(de_file): raise Exception("Cannot find file %s" % de_file) if not os.path.exists(up_file): @@ -2569,15 +2643,15 @@ def generateDegreFiles(i, ndb, i01, osystables, nsystables, output): sql = "ALTER SYSTEM LOAD DICTIONARY FOR %s\n/\n\n" % i.name() output += sql - white_list_file = os.path.join(g_opts.sqls_path, 'degrade_whitelist') + white_list_file = os.path.join(g_opts.sqls_path, "degrade_whitelist") wl_rules = get_whitelist_rules(white_list_file) for f in diff_files[::-1]: check_wl_on_file(f, wl_rules) - with open(f, 'r') as fp: + with open(f, "r") as fp: content = fp.read() output += content - output += '\n' + output += "\n" if low_item and diff_files and isinstance(i, TableGroup): sql = "ALTER SYSTEM LOAD DICTIONARY FOR %s\n/\n\n" % i.name() output += sql @@ -2585,21 +2659,21 @@ def generateDegreFiles(i, ndb, i01, osystables, nsystables, output): def remove_comment(text): - ''' + """ remove comment which begin with '--' - ''' + """ all_text = [] if not text: - return '' + return "" - lines = text.strip().split('\n') + lines = text.strip().split("\n") for line in lines: - if line.strip().find('--') != 0: + if line.strip().find("--") != 0: all_text.append(line) - return '\n'.join(all_text) + return "\n".join(all_text) def generate_upgrade(): @@ -2619,7 +2693,7 @@ def generate_upgrade(): new_init_db.init(g_opts.new_init_db) styleB = new_init_db.getStyle() - if styleA == 'old' and styleB == 'new': + if styleA == "old" and styleB == "new": new_init_db.rename2old() old_last_version = old_init_db.get_last_version() @@ -2628,20 +2702,20 @@ def generate_upgrade(): if old_last_version == 0 and new_last_version >= 1: generate_01_upgrade = True - initdb_01_file = os.path.join(g_opts.sqls_path, 'initdb_01.sql') + initdb_01_file = os.path.join(g_opts.sqls_path, "initdb_01.sql") if not os.path.exists(initdb_01_file): raise Exception("Can not find file %s" % initdb_01_file) initdb_01 = InitDbSqls() initdb_01.init(initdb_01_file) - if styleA == 'old': + if styleA == "old": initdb_01.rename2old() if generate_01_upgrade: generate_0_1_upgrade(old_init_db, initdb_01) - if styleA == 'new' or styleB == 'old': + if styleA == "new" or styleB == "old": systable_new = [] else: systable_new = get_system_table_names(new_init_db) @@ -2650,15 +2724,15 @@ def generate_upgrade(): it = iter(new_init_db) - output_content = '' - output_file_name = os.path.join(g_opts.generate_path, 'upgradeFile.sql') + output_content = "" + output_file_name = os.path.join(g_opts.generate_path, "upgradeFile.sql") for i in it: diff_files = [] - diff_files, old_item = generateFileName(diff_files, i, old_systables, - old_init_db, initdb_01, - new_systables) + diff_files, old_item = generateFileName( + diff_files, i, old_systables, old_init_db, initdb_01, new_systables + ) if old_item: if isinstance(i, TableGroup): sql = "ALTER SYSTEM LOAD DICTIONARY FOR %s\n/\n\n" % i.name() @@ -2684,16 +2758,15 @@ def generateFileName(diff_files, i, otables, old_db, initdb_01, ntables): if not old_item: upgrade_01_file = checkFileExist(item_01, name) else: - upgrade_01_file = os.path.join(g_opts.generate_path, '01', - name+'_upgrade_1.sql') + upgrade_01_file = os.path.join( + g_opts.generate_path, "01", name + "_upgrade_1.sql" + ) if os.path.exists(upgrade_01_file): diff_files.append(upgrade_01_file) - start_ver = max(2, old_ver+1) - for t in range(start_ver, new_ver+1): - up_file = os.path.join(g_opts.sqls_path, - name+'_upgrade_'+str(t)+'.sql') - de_file = os.path.join(g_opts.sqls_path, - name+'_degrade_'+str(t)+'.sql') + start_ver = max(2, old_ver + 1) + for t in range(start_ver, new_ver + 1): + up_file = os.path.join(g_opts.sqls_path, name + "_upgrade_" + str(t) + ".sql") + de_file = os.path.join(g_opts.sqls_path, name + "_degrade_" + str(t) + ".sql") if not os.path.exists(up_file): raise Exception("Cannot find file %s" % up_file) if not os.path.exists(de_file): @@ -2704,51 +2777,52 @@ def generateFileName(diff_files, i, otables, old_db, initdb_01, ntables): def checkFileExist(item_01, name): if item_01: - upgrade_01_file = os.path.join(g_opts.generate_path, '01', - name+'_upgrade_1.sql') + upgrade_01_file = os.path.join( + g_opts.generate_path, "01", name + "_upgrade_1.sql" + ) if not os.path.exists(upgrade_01_file): - raise Exception("Cannot find upgrade file for" - " version_0 to version_1 %s" - % upgrade_01_file) + raise Exception( + "Cannot find upgrade file for" + " version_0 to version_1 %s" % upgrade_01_file + ) else: - upgrade_01_file = os.path.join(g_opts.sqls_path, - name+'_upgrade_1.sql') + upgrade_01_file = os.path.join(g_opts.sqls_path, name + "_upgrade_1.sql") if not os.path.exists(upgrade_01_file): - raise Exception("Cannot find upgrade file for" - " version_0 to version_1 %s" - % upgrade_01_file) - degrade_01_file = os.path.join(g_opts.sqls_path, - name+'_degrade_1.sql') + raise Exception( + "Cannot find upgrade file for" + " version_0 to version_1 %s" % upgrade_01_file + ) + degrade_01_file = os.path.join(g_opts.sqls_path, name + "_degrade_1.sql") if not os.path.exists(degrade_01_file): - raise Exception("Cannot find degrade file for" - " version_1 to version_0 %s" - % degrade_01_file) + raise Exception( + "Cannot find degrade file for" + " version_1 to version_0 %s" % degrade_01_file + ) return upgrade_01_file def readDiffFiles(i, diff_files, output_content, styleA): for f in diff_files: - with open(f, 'r') as fp: + with open(f, "r") as fp: content = fp.read() - if styleA == 'old': - content = content.replace('\r\n', '\n') + if styleA == "old": + content = content.replace("\r\n", "\n") content = remove_comment(content) - sqls = content.split('\n/') + sqls = content.split("\n/") sql_items = [] for sql in sqls: if sql.strip(): - sql_item = SqlItem(sql, - is_target=True, - ignore=True) + sql_item = SqlItem(sql, is_target=True, ignore=True) sql_item.rename2old() sql_items.append(sql_item) - content = "\n/\n\n".join([sql.originSql().strip() - for sql in sql_items])\ - + "\n/\n\n" + content = ( + "\n/\n\n".join([sql.originSql().strip() for sql in sql_items]) + + "\n/\n\n" + ) output_content += content else: output_content += content - output_content += '\n' + output_content += "\n" if diff_files and isinstance(i, TableGroup): sql = "ALTER SYSTEM LOAD DICTIONARY FOR %s\n/\n\n" % i.name() @@ -2780,14 +2854,14 @@ def generateViewSqlFile(sqlList, outputFilename): def generate_view(): - ''' + """ generate view file's diff - ''' + """ drop_sqls = [] drop_all = False drop_jobs = [] - drop_user_jobs = '''BEGIN + drop_user_jobs = """BEGIN FOR ITEM IN (SELECT * FROM USER_JOBS WHERE WHAT IN ('WSR$CREATE_SNAPSHOT();', 'WSR$DROP_SNAPSHOT_TIME();', @@ -2796,8 +2870,8 @@ def generate_view(): DBE_TASK.CANCEL(ITEM.JOB); END LOOP; COMMIT; - END;''' - drop_my_jobs = '''BEGIN + END;""" + drop_my_jobs = """BEGIN FOR ITEM IN (SELECT * FROM MY_JOBS WHERE WHAT IN ('WSR$CREATE_SNAPSHOT();', 'WSR$DROP_SNAPSHOT_TIME();', @@ -2806,7 +2880,7 @@ def generate_view(): DBE_TASK.CANCEL(ITEM.JOB); END LOOP; COMMIT; - END;''' + END;""" if not g_opts.old: return @@ -2822,13 +2896,19 @@ def generate_view(): new_view_sql = InitViewSqls() new_view_sql.init(g_opts.new) - drop_sqls, drop_jobs =\ - init_jobsAndSqls(old_view_sql, new_view_sql, drop_jobs, drop_sqls, - drop_all, drop_my_jobs, drop_user_jobs) + drop_sqls, drop_jobs = init_jobsAndSqls( + old_view_sql, + new_view_sql, + drop_jobs, + drop_sqls, + drop_all, + drop_my_jobs, + drop_user_jobs, + ) if drop_sqls: generateViewSqlFile(drop_sqls, g_opts.generate_file) if drop_jobs: - generateViewSqlFile(drop_jobs, g_opts.generate_file+'_jobs') + generateViewSqlFile(drop_jobs, g_opts.generate_file + "_jobs") def init_jobsAndSqls(oldSql, newSql, dropJobs, dropSqls, is_all, mjob, uJob): @@ -2842,23 +2922,23 @@ def init_jobsAndSqls(oldSql, newSql, dropJobs, dropSqls, is_all, mjob, uJob): new_item = newSql.fetch(i) if not new_item: drop_sql = i.generateDropSql() - if (i.tableName() == 'USER_JOBS' - or i.tableName() == 'SYS.USER_JOBS')\ - and i.isCreateOrReplaceView(): + if ( + i.tableName() == "USER_JOBS" or i.tableName() == "SYS.USER_JOBS" + ) and i.isCreateOrReplaceView(): dropJobs.append(uJob) - if (i.tableName() == 'MY_JOBS' - or i.tableName() == 'SYS.MY_JOBS')\ - and i.isCreateOrReplaceView(): + if ( + i.tableName() == "MY_JOBS" or i.tableName() == "SYS.MY_JOBS" + ) and i.isCreateOrReplaceView(): dropJobs.append(mjob) dropSqls.append(drop_sql) return dropSqls, dropJobs def generate_increase_view(): - ''' + """ generate increase sql between old initview/initplsql/initwsr and new initview/initplsql/initwsr - ''' + """ oldview = InitIncreaseView() newview = InitIncreaseView() @@ -2868,10 +2948,11 @@ def generate_increase_view(): oldview.check_sequence() newview.check_sequence() if len(oldview.get_numberlist()) > len(newview.get_numberlist()): - raise Exception("Please check %s, that module is less than %s" - % (g_opts.new, g_opts.old)) + raise Exception( + "Please check %s, that module is less than %s" % (g_opts.new, g_opts.old) + ) viewgroups = newview.get_all_group() - sqls = '' + sqls = "" for viewgroup in viewgroups: matched = oldview.fetch(viewgroup) if not matched: @@ -2879,20 +2960,20 @@ def generate_increase_view(): number = "--%s\n" % str(viewgroup.number()) sql = number + sql sqls += sql - if sqls != '': + if sqls != "": writeFile(g_opts.generate_file, sqls) def generate_dictionary(): - ''' + """ generate alter system load dictionary for xxx sql file - ''' + """ init_db = InitDbSqls() init_db.init(g_opts.init_db) it = iter(init_db) - output_content = '' - output_file_name = os.path.join(g_opts.generate_path, 'upgradeFile.sql') + output_content = "" + output_file_name = os.path.join(g_opts.generate_path, "upgradeFile.sql") for i in it: if isinstance(i, TableGroup): @@ -2905,10 +2986,10 @@ def generate_dictionary(): def check_initdb(): - ''' + """ check initdb.sql low version -- old high version -- new if table has modified , version must change - ''' + """ # instance of the old initdb file content as an object low_init_db = InitDbSqls() low_init_db.init(g_opts.old_init_db) @@ -2925,8 +3006,7 @@ def check_initdb(): return if high_version == 0: - raise Exception("Versions in %s should larger than zero!" - % g_opts.new_init_db) + raise Exception("Versions in %s should larger than zero!" % g_opts.new_init_db) it = iter(high_init_db) for i in it: @@ -2961,8 +3041,7 @@ def check_initdb(): if diff: if low_item.version() == i.version(): - raise Exception("Version have to change" - " when %s changes." % i.name()) + raise Exception("Version have to change" " when %s changes." % i.name()) elif g_opts.isAdjacent: if low_item.version() + 1 != i.version(): raise Exception("Version of %s shoud add one." % i.name()) @@ -2971,8 +3050,7 @@ def check_initdb(): else: if g_opts.isAdjacent: if low_item.version() != i.version(): - raise Exception("Version of %s" - " shoud not change." % i.name()) + raise Exception("Version of %s" " shoud not change." % i.name()) if error_flag: sys.exit(1) @@ -2982,16 +3060,16 @@ def check_initdb(): def is_upgrade_file(file_name, up_type): - ''' + """ if file_name format as: xxx_upgrade_NN.sql or xxx_degrade_NN.sql where NN is digit numbers - ''' + """ - fname = file_name.rsplit('.', 1) - if fname[-1] != 'sql': + fname = file_name.rsplit(".", 1) + if fname[-1] != "sql": return False - table_up_ver = fname[0].split('_') + table_up_ver = fname[0].split("_") if len(table_up_ver) < 2: return False if table_up_ver[-2] == up_type: @@ -3001,13 +3079,13 @@ def is_upgrade_file(file_name, up_type): def check_wl_on_sql(sql, rules): - ''' + """ check one sql follow the white list - ''' + """ for rule in rules: for i in range(len(rule)): - if rule[i].lower() == 'xxx': + if rule[i].lower() == "xxx": continue try: if rule[i].upper() != sql[i].upper(): @@ -3020,20 +3098,20 @@ def check_wl_on_sql(sql, rules): def check_wl_on_file(file_name, rules): - ''' + """ check the sql file follow the white list - ''' + """ all_sqls = [] - with open(file_name, 'r') as fp: + with open(file_name, "r") as fp: content = fp.read() # translate to unix format - content = content.replace('\r\n', '\n') - all_sqls = content.split('\n/') + content = content.replace("\r\n", "\n") + all_sqls = content.split("\n/") for sql in all_sqls: sql = sql.strip() - if sql.find('--') == 0: + if sql.find("--") == 0: continue if sql: if not check_wl_on_sql(sql.split(), rules): @@ -3041,16 +3119,16 @@ def check_wl_on_file(file_name, rules): def get_whitelist_rules(file_name): - ''' + """ return the whitelist rules of list - ''' + """ rules = [] - with open(file_name, 'r') as fp: + with open(file_name, "r") as fp: content = fp.read() - content = content.replace('\r\n', '\n') - content = content.split('\n') + content = content.replace("\r\n", "\n") + content = content.split("\n") for i in content: if i: rules.append(i.split()) @@ -3059,20 +3137,19 @@ def get_whitelist_rules(file_name): def check_whitelist(): - ''' + """ check xxx_upgrade_xx.sql xxx_degrade_xx.sql if pass the whitelist - ''' - search_key = 'upgrade' + """ + search_key = "upgrade" if g_opts.isDegrade: - white_list_file = os.path.join(g_opts.sqls_path, 'degrade_whitelist') - search_key = 'degrade' + white_list_file = os.path.join(g_opts.sqls_path, "degrade_whitelist") + search_key = "degrade" else: - white_list_file = os.path.join(g_opts.sqls_path, 'upgrade_whitelist') + white_list_file = os.path.join(g_opts.sqls_path, "upgrade_whitelist") if not os.path.exists(white_list_file): - raise Exception("Can not find whitelist" - " in %s" % g_opts.sqls_path) + raise Exception("Can not find whitelist" " in %s" % g_opts.sqls_path) wl_rules = get_whitelist_rules(white_list_file) @@ -3083,7 +3160,6 @@ def check_whitelist(): check_wl_on_file(os.path.join(_root, fname), wl_rules) - def main(): """ according to different action, the corresponding method is called @@ -3108,9 +3184,9 @@ def main(): generate_increase_view() -if __name__ == '__main__': +if __name__ == "__main__": - if(os.getuid() == 0): + if os.getuid() == 0: print("Failed: Cannot use root user for this operation!") sys.exit(1) diff --git a/pkg/install/uninstall.py b/pkg/install/uninstall.py index ce2799ae..d80ac469 100644 --- a/pkg/install/uninstall.py +++ b/pkg/install/uninstall.py @@ -5,6 +5,7 @@ import sys + # If run by root, the import behavior will create folder '__pycache__' # whose owner will be root. The database owner has'nt permission to # remove the folder. So we can't create it. @@ -35,10 +36,12 @@ if os.path.exists("/.dockerenv"): else: MYSQL_BIN_DIR = "/opt/cantian/mysql/install/mysql" + class Options(object): """ class for command line options """ + def __init__(self): # user information self.user_info = pwd.getpwuid(os.getuid()) @@ -61,7 +64,7 @@ class Options(object): # The object of opened log file. self.fp = None - + self.use_gss = False self.in_container = False @@ -77,8 +80,13 @@ def _exec_popen(cmd): :return: status code, standard output, error output """ bash_cmd = ["bash"] - p = subprocess.Popen(bash_cmd, shell=False, stdin=subprocess.PIPE, - stdout=subprocess.PIPE, stderr=subprocess.PIPE) + p = subprocess.Popen( + bash_cmd, + shell=False, + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + ) if gPyVersion[0] == "3": stdout, stderr = p.communicate(cmd.encode()) @@ -111,22 +119,22 @@ def _get_input(msg): def usage(): """uninstall.py is a utility to uninstall cantiand server. -Usage: - python uninstall.py --help - python uninstall.py [-U user] [-F] [-D DATADIR] [-g withoutroot] [-d] [-s] - -Common options: - -U user who install the db - -F clean the database storage area - -D location of the database cluster storage area, - it will be available after -F - -g run uninstall script without root privilege, - but you must have permission of uninstallation folder - -d uninstall inside docker container - -P if sysdba login is disabled by configuration, - specify this option the end - -s uninstall with gss - --help show this help, then exit + Usage: + python uninstall.py --help + python uninstall.py [-U user] [-F] [-D DATADIR] [-g withoutroot] [-d] [-s] + + Common options: + -U user who install the db + -F clean the database storage area + -D location of the database cluster storage area, + it will be available after -F + -g run uninstall script without root privilege, + but you must have permission of uninstallation folder + -d uninstall inside docker container + -P if sysdba login is disabled by configuration, + specify this option the end + -s uninstall with gss + --help show this help, then exit """ print(usage.__doc__) @@ -161,16 +169,16 @@ def parse_parameter(): elif key == "-U": pass # Determine whether to delete the data directory - elif key == '-F': + elif key == "-F": g_opts.clean_data_dir_on = 0 # Get the specified data directory - elif key == '-D': + elif key == "-D": # If the file is checked directly here, the attacker can # query it through the error message here. A valid file # that is not found by it, so the correctness of the # parameter value will be checked in the following function. g_opts.clean_data_dir = value.strip() - elif key == '-P': + elif key == "-P": print("Need database connector's name and password:") g_opts.db_user = _get_input("Username:") check_invalid_symbol(g_opts.db_user) @@ -179,7 +187,7 @@ def parse_parameter(): if (not g_opts.db_user) or (not g_opts.db_passwd): print("Username and password can not be empty.") sys.exit(1) - elif key == '-d': + elif key == "-d": g_opts.in_container = True except getopt.GetoptError as err: @@ -201,8 +209,10 @@ def check_parameter(): sys.exit(1) if g_opts.install_user_privilege != "withoutroot": - print("Error: User has no root privilege, " - "do uninstall, need specify parameter '-g withoutroot'.") + print( + "Error: User has no root privilege, " + "do uninstall, need specify parameter '-g withoutroot'." + ) sys.exit(1) else: print("Error:Check os failed:current os is not linux") @@ -210,12 +220,14 @@ def check_parameter(): if g_opts.clean_data_dir_on == 1: if g_opts.clean_data_dir: - print("Error: Parameter input error: " - "you can not use -D without using -F") + print( + "Error: Parameter input error: " "you can not use -D without using -F" + ) sys.exit(1) if g_opts.clean_data_dir: g_opts.clean_data_dir = os.path.realpath( - os.path.normpath(g_opts.clean_data_dir)) + os.path.normpath(g_opts.clean_data_dir) + ) DefaultValue.checkInvalidPath(g_opts.clean_data_dir) @@ -226,8 +238,9 @@ def check_log(): """ # Get the log path home_path = g_opts.user_info.pw_dir - g_opts.log_file = os.path.join(os.path.realpath( - os.path.normpath(home_path)), "cantianduninstall.log") + g_opts.log_file = os.path.join( + os.path.realpath(os.path.normpath(home_path)), "cantianduninstall.log" + ) # Clean the old log file. if os.path.exists(g_opts.log_file): @@ -283,8 +296,10 @@ def logExit(msg): os.chmod(g_opts.log_file, stat.S_IRUSR) g_opts.fp = None - print("Please refer to uninstall log \"%s\" for more detailed information." - % g_opts.log_file) + print( + 'Please refer to uninstall log "%s" for more detailed information.' + % g_opts.log_file + ) sys.exit(1) @@ -313,7 +328,8 @@ def get_user_environment_file(): log("Getting user environment variables file path...", True) home_path = g_opts.user_info.pw_dir g_opts.user_env_path = os.path.realpath( - os.path.normpath(os.path.join(home_path, ".bashrc"))) + os.path.normpath(os.path.join(home_path, ".bashrc")) + ) if not os.path.isfile(os.path.realpath(g_opts.user_env_path)): logExit("Can't get the environment variables file.") log("End get user environment variables file path") @@ -327,7 +343,8 @@ def find_before_slice(slice_, str_): find '#' in the head of line """ place = str_.find(slice_) - return str_.find('#', 0, place) + return str_.find("#", 0, place) + #################################################################### # Check if there is an installation path in the environment variable @@ -340,26 +357,27 @@ def check_environment_install_path(): input: NA output: NA """ - log("Checking whether install path in the user environment variables...", - True) + log("Checking whether install path in the user environment variables...", True) f = None try: f = open(g_opts.user_env_path) except IOError: - logExit("Check environment variables failed:can not open " - "environment variables file,please check the user that " - "you offered is right") + logExit( + "Check environment variables failed:can not open " + "environment variables file,please check the user that " + "you offered is right" + ) LINE = f.readline() while LINE: # Obtain 'export CTDB_HOME' - if LINE.find('export CTDB_HOME') != -1: + if LINE.find("export CTDB_HOME") != -1: # Determine whether there is "#" before CTDB_HOME, the # function returns a value of -1, indicating that it is # not found, CTDB_HOME is valid. - if find_before_slice(LINE, 'CTDB_HOME') == -1: - INSTALL_ENV_DIC_L = LINE.split('=') + if find_before_slice(LINE, "CTDB_HOME") == -1: + INSTALL_ENV_DIC_L = LINE.split("=") INSTALL_ENV_TEMP_L = INSTALL_ENV_DIC_L[1].rstrip() INSTALL_ENV_L = os.path.normpath(INSTALL_ENV_TEMP_L) INSTALL_ENV_L = os.path.realpath(INSTALL_ENV_L[1:-1]) @@ -369,9 +387,11 @@ def check_environment_install_path(): return 0 LINE = f.readline() f.close() - logExit("Check install path in user environment variables failed:" - "can not find install path in user: %s environment variables" - % g_opts.user_info.pw_name) + logExit( + "Check install path in user environment variables failed:" + "can not find install path in user: %s environment variables" + % g_opts.user_info.pw_name + ) log("End check install path in user environment variables") @@ -404,36 +424,41 @@ def get_gsdata_path_env(): # Determine whether there is "#" before CTDB_DATA, the # function returns a value of -1, indicating that it is # not found, CTDB_DATA is valid. - if find_before_slice('export CTDB_DATA', LINE) == -1: - GSDATA_PATH_DIC_TEMP = LINE.split('=') + if find_before_slice("export CTDB_DATA", LINE) == -1: + GSDATA_PATH_DIC_TEMP = LINE.split("=") GSDATA_PATH_TEMP = GSDATA_PATH_DIC_TEMP[1].rstrip() GSDATA_PATH = os.path.normpath(GSDATA_PATH_TEMP) g_opts.gs_data_path = os.path.realpath(GSDATA_PATH[1:-1]) DefaultValue.checkInvalidPath(g_opts.gs_data_path) if not os.path.exists(g_opts.gs_data_path): f.close() - logExit("Get data directory in user environment variables" - " failed:data directory have been destroyed," - "can not uninstall") + logExit( + "Get data directory in user environment variables" + " failed:data directory have been destroyed," + "can not uninstall" + ) log("End find data directory in user environment variables") f.close() return 0 # deal with the CTDB_HOME with """ # Obtain 'export CTDB_DATA' - elif LINE.find('export CTDB_DATA') != -1: + elif LINE.find("export CTDB_DATA") != -1: # Determine whether there is "#" before CTDB_DATA, the # function returns a value of -1, indicating that it is # not found, CTDB_DATA is valid. - if find_before_slice('export CTDB_DATA', LINE) == -1: - GSDATA_PATH_DIC_TEMP = LINE.split('=') + if find_before_slice("export CTDB_DATA", LINE) == -1: + GSDATA_PATH_DIC_TEMP = LINE.split("=") GSDATA_PATH_TEMP = GSDATA_PATH_DIC_TEMP[1].rstrip() g_opts.gs_data_path = os.path.realpath( - os.path.normpath(GSDATA_PATH_TEMP)) + os.path.normpath(GSDATA_PATH_TEMP) + ) if not os.path.exists(g_opts.gs_data_path): f.close() - logExit("Get data directory in user environment variables " - "failed:data directory have been destroyed," - "can not uninstall") + logExit( + "Get data directory in user environment variables " + "failed:data directory have been destroyed," + "can not uninstall" + ) log("End find data directory in user environment variables") f.close() return 0 @@ -456,12 +481,14 @@ def check_data_dir(): """ log("Begin check data dir...", True) if g_opts.clean_data_dir: - if os.path.exists(g_opts.clean_data_dir) \ - and os.path.isdir(g_opts.clean_data_dir) \ - and g_opts.clean_data_dir == g_opts.gs_data_path: - log("path: \"%s\" is correct" % g_opts.clean_data_dir) + if ( + os.path.exists(g_opts.clean_data_dir) + and os.path.isdir(g_opts.clean_data_dir) + and g_opts.clean_data_dir == g_opts.gs_data_path + ): + log('path: "%s" is correct' % g_opts.clean_data_dir) else: - logExit("path: \"%s\" is incorrect" % g_opts.clean_data_dir) + logExit('path: "%s" is incorrect' % g_opts.clean_data_dir) log("end check,match") @@ -481,28 +508,37 @@ def clean_data_dir(): try: shutil.rmtree(DefaultValue.DOCKER_DATA_DIR) except OSError as err: - logExit("Clean share data path failed:can not delete share data path " - "%s\nPlease manually delete it." % str(err)) + logExit( + "Clean share data path failed:can not delete share data path " + "%s\nPlease manually delete it." % str(err) + ) try: shutil.rmtree(g_opts.gs_data_path) except OSError as err: - logExit("Clean data path failed:can not delete data path " - "%s\nPlease manually delete it." % str(err)) + logExit( + "Clean data path failed:can not delete data path " + "%s\nPlease manually delete it." % str(err) + ) else: - logExit("Clean data failed:can not find data directory path" - " in user environment variables," - "it might be destroyed or not exist") + logExit( + "Clean data failed:can not find data directory path" + " in user environment variables," + "it might be destroyed or not exist" + ) if not g_opts.use_gss: if g_opts.in_container and os.path.exists(DefaultValue.DOCKER_GCC_DIR): try: shutil.rmtree(DefaultValue.DOCKER_GCC_DIR) except OSError as err: - logExit("Clean gcc path failed:can not delete gcc path " - "%s\nPlease manually delete it." % str(err)) + logExit( + "Clean gcc path failed:can not delete gcc path " + "%s\nPlease manually delete it." % str(err) + ) else: log("Not clean data path") log("End clean data path") + ######################################################################### # Check the uninstall script location ######################################################################### @@ -515,38 +551,51 @@ def check_uninstall_pos(): output: NA """ log("Checking uninstall.py position...", True) - bin_path = g_opts.install_path_l + os.sep + 'bin' - addons_path = g_opts.install_path_l + os.sep + 'add-ons' - admin_path = g_opts.install_path_l + os.sep + 'admin' - lib_path = g_opts.install_path_l + os.sep + 'lib' - pkg_file = g_opts.install_path_l + os.sep + 'package.xml' + bin_path = g_opts.install_path_l + os.sep + "bin" + addons_path = g_opts.install_path_l + os.sep + "add-ons" + admin_path = g_opts.install_path_l + os.sep + "admin" + lib_path = g_opts.install_path_l + os.sep + "lib" + pkg_file = g_opts.install_path_l + os.sep + "package.xml" # Check if the install path exists if not os.path.exists(g_opts.install_path_l): - logExit("Check uninstall.py position failed:You have" - " changed uninstall.py position,install path not exist") + logExit( + "Check uninstall.py position failed:You have" + " changed uninstall.py position,install path not exist" + ) # Check if the bin path exists if not os.path.exists(bin_path): - logExit("Check uninstall.py position failed:You have" - " changed uninstall.py position,can not find path bin") + logExit( + "Check uninstall.py position failed:You have" + " changed uninstall.py position,can not find path bin" + ) # Check if the addons path exists if not os.path.exists(addons_path): - logExit("Check uninstall.py position failed:You have" - " changed uninstall.py position,can not find path add-ons") + logExit( + "Check uninstall.py position failed:You have" + " changed uninstall.py position,can not find path add-ons" + ) # Check if the admin path exists if not os.path.exists(admin_path): - logExit("Check uninstall.py position failed:You have" - " changed uninstall.py position,can not find path admin") + logExit( + "Check uninstall.py position failed:You have" + " changed uninstall.py position,can not find path admin" + ) # Check if the lib path exists if not os.path.exists(lib_path): - logExit("Check uninstall.py position failed:You have" - " changed uninstall.py position,can not find file lib") + logExit( + "Check uninstall.py position failed:You have" + " changed uninstall.py position,can not find file lib" + ) # Check if the package path exists if not os.path.isfile(pkg_file): - logExit("Check uninstall.py position failed:You have" - " changed uninstall.py position,can not find file package.xml") + logExit( + "Check uninstall.py position failed:You have" + " changed uninstall.py position,can not find file package.xml" + ) log("End check uninstall.py position") + ######################################################################### # Clear the installation path ######################################################################### @@ -568,8 +617,10 @@ def clean_install_path(): # Remove the install path shutil.rmtree(g_opts.install_path_l) except OSError as err: - logExit("Clean install path failed:can not delete install path " - "%s\nPlease manually delete it." % str(err)) + logExit( + "Clean install path failed:can not delete install path " + "%s\nPlease manually delete it." % str(err) + ) log("Clean install path success") log("End clean Install path") @@ -578,6 +629,7 @@ def clean_install_path(): # Clear environment variables ########################################################################### + # Resolution path def Genregstring(text): """ @@ -591,12 +643,13 @@ def Genregstring(text): insList = insStr.split(os.sep) regString = "" for i in insList: - if(i == ""): + if i == "": continue else: regString += r"\/" + i return regString + # Clear environment variables @@ -610,13 +663,15 @@ def clean_environment(): # Clear environment variable CTDB_DATA data_cmd = r"/^\s*export\s*CTDB_DATA=\".*\"$/d" # Clear environment variable PATH about database - path_cmd = (r"/^\s*export\s*PATH=\"%s\/bin\":\$PATH$/d" - % Genregstring(g_opts.install_path_l)) + path_cmd = r"/^\s*export\s*PATH=\"%s\/bin\":\$PATH$/d" % Genregstring( + g_opts.install_path_l + ) # Clear environment variable LD_LIBRARY_PATH about database - lib_cmd = (r"/^\s*export\s*LD_LIBRARY_PATH=\"%s\/lib\":\"%s\/add-ons\":" - r"\$LD_LIBRARY_PATH$/d" - % (Genregstring(g_opts.install_path_l), - Genregstring(g_opts.install_path_l))) + lib_cmd = ( + r"/^\s*export\s*LD_LIBRARY_PATH=\"%s\/lib\":\"%s\/add-ons\":" + r"\$LD_LIBRARY_PATH$/d" + % (Genregstring(g_opts.install_path_l), Genregstring(g_opts.install_path_l)) + ) # Clear environment variable CTDB_HOME home_cmd = r"/^\s*export\s*CTDB_HOME=\".*\"$/d" # Clear environment variable CMS_HOME @@ -629,8 +684,17 @@ def clean_environment(): mode_cmd = r"/^\s*export\s*CTSQL_SSL_MODE=.*$/d" cipher_cmd = r"/^\s*export\s*CTSQL_SSL_KEY_PASSWD=.*$/d" - cmds = [path_cmd, lib_cmd, home_cmd, cms_cmd, - ca_cmd, cert_cmd, key_cmd, mode_cmd, cipher_cmd] + cmds = [ + path_cmd, + lib_cmd, + home_cmd, + cms_cmd, + ca_cmd, + cert_cmd, + key_cmd, + mode_cmd, + cipher_cmd, + ] if g_opts.clean_data_dir_on == 0: cmds.insert(0, data_cmd) @@ -691,12 +755,12 @@ def get_instance_id(): input: NA output: NA """ - cmd = ("ps ux | grep -v grep | grep cantiand " - "| grep -w '\-D %s' |awk '{print $2}'") % g_opts.gs_data_path + cmd = ( + "ps ux | grep -v grep | grep cantiand " "| grep -w '\-D %s' |awk '{print $2}'" + ) % g_opts.gs_data_path status, output, _ = _exec_popen(cmd) if status: - logExit("Failed to execute cmd: %s. Error:%s." % (str(cmd), - str(output))) + logExit("Failed to execute cmd: %s. Error:%s." % (str(cmd), str(output))) # process exists return output @@ -714,12 +778,17 @@ def kill_instance(instance_pid): logExit("kill process %s failed" % instance_pid) log("Kill cantiand instance succeed") + def kill_process(process_name): # kill process - kill_cmd = (r"proc_pid_list=`ps ux | grep %s | grep -v grep" - r"|awk '{print $2}'` && " % process_name) - kill_cmd += (r"(if [ X\"$proc_pid_list\" != X\"\" ];then echo " - r"$proc_pid_list | xargs kill -9; exit 0; fi)") + kill_cmd = ( + r"proc_pid_list=`ps ux | grep %s | grep -v grep" + r"|awk '{print $2}'` && " % process_name + ) + kill_cmd += ( + r"(if [ X\"$proc_pid_list\" != X\"\" ];then echo " + r"$proc_pid_list | xargs kill -9; exit 0; fi)" + ) log("kill process cmd: %s" % kill_cmd) ret_code, _, _ = _exec_popen(kill_cmd) if ret_code: @@ -743,7 +812,7 @@ def stop_instance(): lsnr_addr = read_cantiand_cfg("LSNR_ADDR") if not lsnr_addr: logExit("Failed to get the listen address of database.") - host_ip = lsnr_addr.split(',')[0] + host_ip = lsnr_addr.split(",")[0] # if the cantian process not exists, and disable sysdba user # tell user the user name and password input interactive are @@ -751,11 +820,15 @@ def stop_instance(): instance_pid = get_instance_id() # specify -P parameter, db password is supported if not instance_pid and g_opts.db_passwd: - log("Notice: Instance '%s' has been stopped." % - g_opts.gs_data_path, True) - log(("Notice: The Database username and password" - " that are interactive entered " - "will not be verified correct and used.", True)) + log("Notice: Instance '%s' has been stopped." % g_opts.gs_data_path, True) + log( + ( + "Notice: The Database username and password" + " that are interactive entered " + "will not be verified correct and used.", + True, + ) + ) kill_process("mysqld") kill_process("cms") if g_opts.use_gss: @@ -775,24 +848,32 @@ def stop_instance(): # not specify -P, db password is empty, login database by sysdba if not g_opts.db_passwd: cmd = "%s/bin/shutdowndb.sh -h %s -p %s -w -m immediate -D %s" % ( - g_opts.install_path_l, host_ip, lsnr_port, g_opts.gs_data_path) + g_opts.install_path_l, + host_ip, + lsnr_port, + g_opts.gs_data_path, + ) else: - cmd = ("echo '%s' | %s/bin/shutdowndb.sh" - " -h %s -p %s -U %s -m immediate -W -D %s") % ( + cmd = ( + "echo '%s' | %s/bin/shutdowndb.sh" + " -h %s -p %s -U %s -m immediate -W -D %s" + ) % ( g_opts.db_passwd, g_opts.install_path_l, host_ip, lsnr_port, g_opts.db_user, - g_opts.gs_data_path) + g_opts.gs_data_path, + ) return_code, stdout, stderr = _exec_popen(cmd) if return_code: g_opts.db_passwd = "" stdout = get_error_msg(stdout, stderr) - if (not g_opts.db_passwd) and stdout.find( - "login as sysdba is prohibited") >= 0: - stdout += ("\nsysdba login is disabled, please specify -P " - "parameter to input password, refer to --help.") + if (not g_opts.db_passwd) and stdout.find("login as sysdba is prohibited") >= 0: + stdout += ( + "\nsysdba login is disabled, please specify -P " + "parameter to input password, refer to --help." + ) logExit("stop cantian instance failed. Error: %s" % stdout) @@ -822,12 +903,31 @@ def check_invalid_symbol(para): :return: NA """ symbols = ( - "|", ";", "&", "$", "<", ">", "`", "\\", "'", "\"", "{", "}", - "(", ")", "[", "]", "~", "*", "?", "!", "\n", + "|", + ";", + "&", + "$", + "<", + ">", + "`", + "\\", + "'", + '"', + "{", + "}", + "(", + ")", + "[", + "]", + "~", + "*", + "?", + "!", + "\n", ) for symbol in symbols: if para.find(symbol) > -1: - logExit("There is invalid symbol \"%s\" in %s" % (symbol, para)) + logExit('There is invalid symbol "%s" in %s' % (symbol, para)) def main(): @@ -862,8 +962,11 @@ def main(): clean_environment() clean_install_path() - log("Cantiand was successfully removed from your computer, " - "for more message please see %s." % g_opts.log_file, True) + log( + "Cantiand was successfully removed from your computer, " + "for more message please see %s." % g_opts.log_file, + True, + ) log("Cantiand was successfully removed from your computer") os.chmod(g_opts.log_file, stat.S_IRUSR) diff --git a/pkg/test/om_test/python_LLT.py b/pkg/test/om_test/python_LLT.py index 1a1e447e..c1838b53 100644 --- a/pkg/test/om_test/python_LLT.py +++ b/pkg/test/om_test/python_LLT.py @@ -6,11 +6,13 @@ import warnings cur_path = os.path.abspath(os.path.dirname(__file__)) sys.path.append(os.path.join(cur_path)) -src_path = os.path.join(cur_path, '..', '..', 'deploy', 'action') +src_path = os.path.join(cur_path, "..", "..", "deploy", "action") sys.path.append(src_path) -cov = coverage.coverage(branch=True, - source=[src_path], - omit=["__init__.py", "test_*", "*config.py", "*log.py", "*constant.py"]) +cov = coverage.coverage( + branch=True, + source=[src_path], + omit=["__init__.py", "test_*", "*config.py", "*log.py", "*constant.py"], +) warnings.filterwarnings("ignore", category=SyntaxWarning) @@ -26,10 +28,10 @@ def main(): # 保存覆盖率数据 cov.save() cov.report() - cov.xml_report(outfile=os.path.join(cur_path, 'coverage.xml')) + cov.xml_report(outfile=os.path.join(cur_path, "coverage.xml")) return result.wasSuccessful() -if __name__ == '__main__': +if __name__ == "__main__": ret = main() sys.exit(0 if ret else 1) diff --git a/pkg/test/om_test/test_dr_deploy_operate/test_dr_deploy_pre_check.py b/pkg/test/om_test/test_dr_deploy_operate/test_dr_deploy_pre_check.py index 0c96a131..cbc6f450 100644 --- a/pkg/test/om_test/test_dr_deploy_operate/test_dr_deploy_pre_check.py +++ b/pkg/test/om_test/test_dr_deploy_operate/test_dr_deploy_pre_check.py @@ -2,6 +2,7 @@ import sys import mock import unittest import collections + sys.modules["requests"] = mock.MagicMock() sys.modules["termios"] = mock.MagicMock() sys.modules["pty"] = mock.MagicMock() @@ -10,17 +11,26 @@ import storage_operate.dr_deploy_operate.dr_deploy_pre_check as pre_check class getConfigTestCase(unittest.TestCase): - @mock.patch("storage_operate.dr_deploy_operate.dr_deploy_pre_check.read_json_config") + @mock.patch( + "storage_operate.dr_deploy_operate.dr_deploy_pre_check.read_json_config" + ) def test_get_config_values_normal(self, mock_json_config): - mock_json_config.return_value = {"deploy_policy": "ModeA", "ModeA": {"config": {"test": "test keys"}}} - result = pre_check.get_config_values("test") + mock_json_config.return_value = { + "deploy_policy": "ModeA", + "ModeA": {"config": {"test": "test keys"}}, + } + result = pre_check.get_config_values("test") self.assertEqual("test keys", result) def test_get_config_values_abnormal(self): - pre_check.read_json_config = mock.Mock(return_value={"deploy_policy": "default", - "ModeA": {"config": {"test": "test keys"}}}) - result = pre_check.get_config_values("test") - self.assertEqual('', result) + pre_check.read_json_config = mock.Mock( + return_value={ + "deploy_policy": "default", + "ModeA": {"config": {"test": "test keys"}}, + } + ) + result = pre_check.get_config_values("test") + self.assertEqual("", result) class FakeDRDeployPreCheck(pre_check.DRDeployPreCheck): @@ -61,8 +71,10 @@ class DRDeployPreCheckTestCase(unittest.TestCase): @mock.patch("storage_operate.dr_deploy_operate.dr_deploy_pre_check.exec_popen") def test_check_dr_process_all_error(self, mock_exec_popen): mock_exec_popen.return_value = (2, 1, 2) - target_error = ("Dr deploy is executing, please check, details:\n1Dr undeploy is executing, please " +\ - "check, details:\n1Dr full sync is executing, please check, details:\n1") + target_error = ( + "Dr deploy is executing, please check, details:\n1Dr undeploy is executing, please " + + "check, details:\n1Dr full sync is executing, please check, details:\n1" + ) with self.assertRaisesRegex(Exception, target_error): pre_check.DRDeployPreCheck.check_dr_process() @@ -76,7 +88,9 @@ class FakeParamCheck(pre_check.ParamCheck): self.dr_deploy_params = {"dm_ip": "127.0.0.1", "dm_user": "admin"} -args = collections.namedtuple("args", ["action", "site", "mysql_cmd", "mysql_user", "display"]) +args = collections.namedtuple( + "args", ["action", "site", "mysql_cmd", "mysql_user", "display"] +) class ParamCheckTestCase(unittest.TestCase): @@ -89,7 +103,14 @@ class ParamCheckTestCase(unittest.TestCase): @mock.patch("cantian_common.mysql_shell.MysqlShell.start_session", return_value="") @mock.patch("logic.storage_operate.StorageInf.login", return_value="") @mock.patch("argparse.ArgumentParser") - def test_execute_normal(self, mock_parser, mock_login, mock_start_session, mocke_close_session, mock_input): + def test_execute_normal( + self, + mock_parser, + mock_login, + mock_start_session, + mocke_close_session, + mock_input, + ): args.action = "deploy" args.site = "active" mock_parser.parse_args.return_value = args @@ -100,10 +121,16 @@ class ParamCheckTestCase(unittest.TestCase): @mock.patch("cantian_common.mysql_shell.MysqlShell.start_session", return_value="") @mock.patch("logic.storage_operate.StorageInf.login", side_effect=Exception("test")) @mock.patch("argparse.ArgumentParser") - def test_execute_abnormal(self, mock_parser, mock_login, mock_start_session, mocke_close_session, mock_input): + def test_execute_abnormal( + self, + mock_parser, + mock_login, + mock_start_session, + mocke_close_session, + mock_input, + ): args.action = "deploy" args.site = "active" mock_parser.parse_args.return_value = args with self.assertRaisesRegex(Exception, "test"): self.param_check.execute() - -- Gitee