From 3f5e738cada145ef7086f3ecd9043f1bb823e6ac Mon Sep 17 00:00:00 2001 From: SunnyQjm Date: Thu, 20 Oct 2022 11:32:19 +0800 Subject: [PATCH 1/7] feat(cec): CecClient support custom callback --- sysom_api/sdk/cec_base/cec_client.py | 42 +++++++++++++++++++++++----- 1 file changed, 35 insertions(+), 7 deletions(-) diff --git a/sysom_api/sdk/cec_base/cec_client.py b/sysom_api/sdk/cec_base/cec_client.py index 905cc730..a04f9df9 100644 --- a/sysom_api/sdk/cec_base/cec_client.py +++ b/sysom_api/sdk/cec_base/cec_client.py @@ -7,7 +7,8 @@ File cec_client.py Description: """ from threading import Thread -from typing import List +from typing import Any, List +from queue import Queue from sdk.cec_base.consumer import Consumer, dispatch_consumer from sdk.cec_base.admin import Admin, dispatch_admin from sdk.cec_base.event import Event @@ -19,13 +20,27 @@ class CecClient: Args: url(str): CEC url + sync_mode(bool): This parameter specifies whether CecClient uses + synchronous mode + 1. sync mode: In synchronous mode, all received events will + be placed inside a FIFO queue, which is fetched via the + get_next_event interface; + 2. callback mode: In callback mode, CecClient will call the + on_receive_event callback every time it receives an event. + """ - def __init__(self, url: str) -> None: + def __init__(self, url: str, sync_mode: bool = False, + custom_callback: Any = None) -> None: self._url = url self._tasks: List[dict] = [] self._consumers: List[Consumer] = [] self._inner_threads: List[Thread] + self._sync_mode = sync_mode + self._inner_queue = None + self._custom_callback = custom_callback + if self._sync_mode: + self._inner_queue = Queue(maxsize=10) self.admin: Admin = dispatch_admin(url) self.producer: Producer = dispatch_producer(url) @@ -45,18 +60,31 @@ class CecClient: Args: consumer(Consumer): CEC Consumer instance - topic(str): Topic name - group_id(str): Group ID - consumer_id(str): Consumer ID - + task: Consume task """ for event in consumer: - self.on_receive_event(consumer, event, task) + if self._sync_mode: + self._inner_queue.put({ + "event": event, + "task": task, + "consumer": consumer + }) + elif self._custom_callback is not None: + self._custom_callback(consumer, event, task) + else: + self.on_receive_event(consumer, event, task) + + def get_next_event(self) -> dict: + """Get next event""" + if not self._sync_mode: + raise Exception("Not in sync mode") + return self._inner_queue.get() def on_receive_event(self, consumer: Consumer, event: Event, task: dict): """Invoke while receive event from CEC Args: + consumer(Consumer): Cec Consumer instance event(Event): CEC Event task: Consume task { -- Gitee From a9264a5d093a0ceb372146218f085a9cd1f10f85 Mon Sep 17 00:00:00 2001 From: SunnyQjm Date: Thu, 20 Oct 2022 11:34:57 +0800 Subject: [PATCH 2/7] refactor: remove not used code --- sysom_api/lib/channels/ssh.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/sysom_api/lib/channels/ssh.py b/sysom_api/lib/channels/ssh.py index 826f4dbf..42b2ce82 100644 --- a/sysom_api/lib/channels/ssh.py +++ b/sysom_api/lib/channels/ssh.py @@ -117,10 +117,6 @@ class Channel(BaseChannel): self.validate_kwargs() - # def check_host_is_exist(self, instance: str) -> bool: - # status, _ = HTTP.request('get', f'{HOST_LIST_API}ip/{instance}/', '', {}) - # return True if status == 200 else False - def validate_kwargs(self): for item in filter( lambda x: not x[1], [(field, self.kwargs.get(field, None)) -- Gitee From cf7c4fe248d627ee0cd5cf3e95118a8501dd5545 Mon Sep 17 00:00:00 2001 From: SunnyQjm Date: Thu, 20 Oct 2022 11:36:45 +0800 Subject: [PATCH 3/7] refactor: Move JWT decode code out of TaskAuth.. --- sysom_api/lib/authentications.py | 63 ++++++++++++++++---------------- 1 file changed, 32 insertions(+), 31 deletions(-) diff --git a/sysom_api/lib/authentications.py b/sysom_api/lib/authentications.py index d924d286..582821a2 100644 --- a/sysom_api/lib/authentications.py +++ b/sysom_api/lib/authentications.py @@ -1,5 +1,6 @@ import logging import os +from typing import List from django.conf import settings from django.utils.translation import ugettext as _ from rest_framework.exceptions import AuthenticationFailed @@ -11,39 +12,39 @@ from .utils import import_module logger = logging.getLogger(__name__) +def get_jwt_decode_classes() -> List[BaseAuthentication]: + jwt_decode_classes = [] + import_strings = [ + f'lib.decode.{f.replace(".py", "")}' for f in os.listdir(settings.JWT_TOKEN_DECODE_DIR) + ] + for string in import_strings: + module = import_module(string) + try: + m = getattr(module, 'JWTTokenDecode') + jwt_decode_classes.append(m) + except Exception as exc: + logger.warn(exc) + return jwt_decode_classes + + +def decode_token(token: str) -> dict: + error_message, success, result = "", False, {} + for auth_class in get_jwt_decode_classes(): + result, success = auth_class().decode(token) + if not success: + error_message += result + else: + break + if not success: + raise AuthenticationFailed(error_message) + return result + + class TaskAuthentication(BaseAuthentication): def authenticate(self, request: Request): token = request.META.get('HTTP_AUTHORIZATION') - payload = self._decode_token(token) + payload = decode_token(token) payload['token'] = token - if 'sub' in payload: payload['id'] = int(payload['sub']) + if 'sub' in payload: + payload['id'] = int(payload['sub']) return payload, _ - - @staticmethod - def get_jwt_decode_classes() -> list: - jwt_decode_classes = [] - import_strings = [ - f'lib.decode.{f.replace(".py", "")}' for f in os.listdir(settings.JWT_TOKEN_DECODE_DIR) - ] - for string in import_strings: - module = import_module(string) - try: - m = getattr(module, 'JWTTokenDecode') - jwt_decode_classes.append(m()) - except Exception as exc: - logger.warn(exc) - return jwt_decode_classes - - def _decode_token(self, token): - error_message, state = "", False - for t in TaskAuthentication.get_jwt_decode_classes(): - r, s = t.decode(token) - if not s: - error_message += r - continue - else: - state = s - break - if not state: - raise AuthenticationFailed(error_message) - return r \ No newline at end of file -- Gitee From 9502d615ddfb31e69fef26fe77e7497277c9c688 Mon Sep 17 00:00:00 2001 From: SunnyQjm Date: Thu, 20 Oct 2022 11:44:13 +0800 Subject: [PATCH 4/7] feat: Add the ability to send events to the mission center to CommonView --- sysom_api/apps/common/common_model_viewset.py | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/sysom_api/apps/common/common_model_viewset.py b/sysom_api/apps/common/common_model_viewset.py index 9dff75cd..00a16e87 100644 --- a/sysom_api/apps/common/common_model_viewset.py +++ b/sysom_api/apps/common/common_model_viewset.py @@ -1,6 +1,8 @@ from typing import Any from rest_framework.viewsets import GenericViewSet from rest_framework.exceptions import ValidationError +from sdk.cec_base.cec_client import CecClient +from django.conf import settings class CommonModelViewSet(GenericViewSet): @@ -8,6 +10,16 @@ class CommonModelViewSet(GenericViewSet): 通用 ModelViewSet 实现,提供一些通用工具方法 """ + def __init__(self, **kwargs: Any) -> None: + super().__init__(**kwargs) + self._inner_cec_client: CecClient = None + + def produce_event_to_cec(self, topic: str, value: dict): + """Produce one event to specific topic""" + if self._inner_cec_client is None: + self._inner_cec_client = CecClient(settings.SYSOM_CEC_URL) + self._inner_cec_client.delivery(topic, value) + def require_param_validate(self, request, require_params): """ 检查请求参数中是否包含所需的所有参数,任何参数缺失将返回错误,错误信息中包含缺失的参数列表 @@ -49,7 +61,7 @@ class CommonModelViewSet(GenericViewSet): not_allow_params.append(key) for param_name in not_allow_params: context.pop(param_name) - + return { "success": True, "message": "", -- Gitee From 4884ad331ba506e738bc1cc6f6c459a7bb947235 Mon Sep 17 00:00:00 2001 From: SunnyQjm Date: Thu, 20 Oct 2022 11:46:03 +0800 Subject: [PATCH 5/7] refactor: Fully migrate diagnostic functions to v2 --- sysom_api/apps/task/apps.py | 11 +- sysom_api/apps/task/channel.py | 38 ---- sysom_api/apps/task/executors.py | 306 +++++++++---------------------- sysom_api/apps/task/seriaizer.py | 9 +- sysom_api/apps/task/views.py | 114 +----------- sysom_api/conf/common.py | 18 +- 6 files changed, 106 insertions(+), 390 deletions(-) delete mode 100644 sysom_api/apps/task/channel.py diff --git a/sysom_api/apps/task/apps.py b/sysom_api/apps/task/apps.py index 859559c1..be96bea0 100644 --- a/sysom_api/apps/task/apps.py +++ b/sysom_api/apps/task/apps.py @@ -11,18 +11,17 @@ class JobConfig(AppConfig): name = 'apps.task' def ready(self): - if 'runserver' in sys.argv or 'manage.py' not in sys.argv: - from django.conf import settings - from sdk.cec_base.consumer import Consumer + from django.conf import settings + if ('runserver' in sys.argv or 'manage.py' not in sys.argv) and settings.IS_MICRO_SERVICES: + from sdk.cec_base.log import LoggerHelper, LoggerLevel from apps.task.executors import TaskDispatcher + LoggerHelper.update_sys_stdout_sink(LoggerLevel.LOGGER_LEVEL_INFO) # 这边微服务正式启动的时候执行一些处理代码 # 启动任务结果处理线程 try: TaskDispatcher( settings.SYSOM_CEC_URL, - ).start_dispatcher(settings.SYSOM_CEC_TASK_RESULT_PROCESS_TOPIC, - Consumer.generate_consumer_id(), - settings.SYSOM_CEC_TASK_RESULT_PROCESS_GROUP) + ).start_dispatcher() except Exception as e: logger.exception(e) else: diff --git a/sysom_api/apps/task/channel.py b/sysom_api/apps/task/channel.py deleted file mode 100644 index c3f2bfe2..00000000 --- a/sysom_api/apps/task/channel.py +++ /dev/null @@ -1,38 +0,0 @@ -from django.conf import settings -from lib.utils import HTTP - - -class Channel: - API_CHANNEL_VALIDATE = settings.CHANNEL_VALIDATE_API - API_CHANNEL_COMMAND = settings.CHANNEL_COMMAND_API - API_CHANNEL_RESULT = settings.CHANNEL_RESULT_API - - @classmethod - def http(cls, *args, **kwargs): - return HTTP.request(*args, **kwargs) - - @classmethod - def get_channel_result(cls, data, token, **kwargs): - """ - response channel result - """ - return cls.http('post', url=Channel.API_CHANNEL_RESULT, data=data, token=token, **kwargs) - - @classmethod - def post_channel(cls, data, token, **kwargs): - """ - channel execute cmd - """ - return cls.http('post', url=Channel.API_CHANNEL_COMMAND, data=data, token=token, **kwargs) - - @classmethod - def channel_validate(cls, data, **kwargs): - """ - channel validate - data parameter - { - "ip": "xxx.xxx.xxx.xxx", - "password": "123456" - } - """ - return cls.http('post', url=Channel.API_CHANNEL_VALIDATE, data=data, **kwargs) diff --git a/sysom_api/apps/task/executors.py b/sysom_api/apps/task/executors.py index ee77b316..79761729 100644 --- a/sysom_api/apps/task/executors.py +++ b/sysom_api/apps/task/executors.py @@ -8,197 +8,15 @@ import ast from apps.task.models import JobModel from django.conf import settings from django.db import connection -from lib.response import ErrorResponse -from lib.utils import HTTP, uuid_8 +from lib.utils import uuid_8 from sdk.cec_base.cec_client import CecClient from sdk.cec_base.event import Event from sdk.cec_base.consumer import Consumer -from .channel import Channel +from .seriaizer import JobDetailSerializer logger = logging.getLogger(__name__) -class SshJob: - def __init__(self, resp_scripts, task_id, **kwargs): - self.resp_scripts = resp_scripts - self.task_id = task_id - self.kwargs = kwargs - self.instance = self.get_object() - - def run(self): - self.update_job(status="Running") - if settings.IS_MICRO_SERVICES: - setattr(self, 'user', self.kwargs.get('user')) - self._api_service() - else: - self._import_service() - - def update_job(self, **kwargs): - try: - self.instance.__dict__.update(**kwargs) - self.instance.save() - except Exception as e: - raise e - finally: - connection.close() - - def get_object(self): - try: - return JobModel.objects.get(task_id=self.task_id) - except JobModel.DoesNotExist: - raise ErrorResponse(msg=f'task id: {self.task_id} not exist!') - - def _api_service(self): - """ - API调用, 执行Task任务 - """ - host_ips = [] - count = 0 - for script in self.resp_scripts: - count = count + 1 - ip = script.get("instance", None) - cmd = script.get("cmd", None) - if not ip or not cmd: - self.update_job(status="Fail", result="script result find not instance or cmd") - break - host_ips.append(ip) - - status_code, res = Channel.post_channel(data=script, token=self.user['token']) - if status_code != 200: - self.update_job(status="Fail") - break - elif res['state'] != 0: - self.update_job(status="Fail", result=res['result']) - break - else: - resp_status = res['state'] - execute_result = res['result'] - - if self.kwargs.get('update_host_status', None): - patch_host_url = f'{settings.HOST_LIST_API}update/{ip}/' - HTTP.request('patch', url=patch_host_url, data={'status': resp_status}, token=self.user['token']) - - # if self.kwargs.get('service_name', None) == "node_delete": - # del_host_url = f'{settings.HOST_LIST_API}del/{ip}/' - # HTTP.request('delete', url=del_host_url, token=self.user['token'], data={}) - - if resp_status != 0: - self.update_job(status="Fail", result=execute_result, host_by=host_ips) - break - - if count == len(self.resp_scripts): - res = execute_result - params = self.instance.params - if params: - service_name = params.get("service_name", None) - if service_name: - SCRIPTS_DIR = settings.SCRIPTS_DIR - service_post_name = service_name + '_post' - service_post_path = os.path.join( - SCRIPTS_DIR, service_post_name) - if os.path.exists(service_post_path): - # 创建一个临时文件,用于暂存中间结果 - fd, path = tempfile.mkstemp() - command_list = [service_post_path, path, self.instance.task_id] - try: - # 将要传递的中间结果写入到临时文件当中 - with os.fdopen(fd, 'w') as tmp: - tmp.write(res['result']) - resp = subprocess.run(command_list, stdout=subprocess.PIPE, - stderr=subprocess.PIPE) - if resp.returncode != 0: - logger.error( - f'执行失败: {resp.stderr.decode("utf-8")}') - self.update_job(status="Fail", result=resp.stderr.decode('utf-8')) - break - stdout = resp.stdout - try: - result = stdout.decode('utf-8') - except UnicodeDecodeError as e: - result = stdout.decode('gbk') - res = json.loads(result) - except Exception as e: - logger.error(f'ERROR: {e}') - self.update_job(status="Fail", result=str(e)) - break - finally: - os.remove(path) - - self.update_job(status="Success", result=res, host_by=host_ips) - - - def _import_service(self): - from apps.host.models import HostModel - from lib.channels.ssh import SSH - - try: - self.update_job(status="Running") - host_ips = [] - count = 0 - for script in self.resp_scripts: - count = count + 1 - ip = script.get("instance", None) - cmd = script.get("cmd", None) - if not ip or not cmd: - self.update_job(status="Fail", - result="script result find not instance or cmd") - break - host_ips.append(ip) - - try: - host = HostModel.objects.get(ip=ip) - except HostModel.DoesNotExist: - self.update_job(status="Fail", - result="host not found by script return IP:%s" % ip) - break - ssh_cli = SSH( - hostname=host.ip, port=host.port, username=host.username) - - status, result = ssh_cli.run_command(cmd) - - if self.kwargs.get('service_name', None) == "node_delete": - logger.info(f'HOST: {host}') - host.delete() - if status != 0: - self.update_job(status="Fail", - result=result, host_by=host_ips) - break - if count == len(self.resp_scripts): - params = self.instance.params - if params: - params = json.loads(params) - service_name = params.get("service_name", None) - if service_name: - SCRIPTS_DIR = settings.SCRIPTS_DIR - service_post_name = service_name + '_post' - service_post_path = os.path.join( - SCRIPTS_DIR, service_post_name) - if os.path.exists(service_post_path): - try: - resp = subprocess.run([service_post_path, result], stdout=subprocess.PIPE, - stderr=subprocess.PIPE) - if resp.returncode != 0: - logger.error( - f'执行失败: {resp.stderr.decode("utf-8")}') - self.update_job(status="Fail", - result=resp.stderr.decode('utf-8')) - break - stdout = resp.stdout - result = stdout.decode('utf-8') - except Exception as e: - logger.error(f'ERROR: {e}') - self.update_job(status="Fail", result=str(e)) - break - self.update_job(status="Success", - result=result, host_by=host_ips) - except socket.timeout: - self.update_job(status="Fail", result="socket time out") - except Exception as e: - logger.error(f'ERROR: {e}') - self.update_job(status="Fail", result=str(e)) - finally: - connection.close() - class TaskDispatcher(CecClient): """A asynchronous task dispatcher implement @@ -212,14 +30,50 @@ class TaskDispatcher(CecClient): def __init__(self, url: str) -> None: CecClient.__init__(self, url) - def start_dispatcher(self, topic: str, consumer_id: str, group_id: str): + def start_dispatcher(self): self.append_group_consume_task( - topic, group_id, consumer_id, ensure_topic_exist=True) + settings.SYSOM_CEC_TASK_RESULT_PROCESS_TOPIC, + settings.SYSOM_CEC_TASK_RESULT_PROCESS_GROUP, + Consumer.generate_consumer_id(), + ensure_topic_exist=True + ) + self.append_group_consume_task( + settings.SYSOM_CEC_TASK_GENERATE_TOPIC, + settings.SYSOM_CEC_TASK_GENERATE_TASK_LISTENER, + Consumer.generate_consumer_id(), + ensure_topic_exist=True + ) self.start() + def on_receive_event(self, consumer: Consumer, event: Event, task: dict): + try: + topic_name = task.get("topic_name", "unknown_topic") + if topic_name == settings.SYSOM_CEC_TASK_RESULT_PROCESS_TOPIC: + # Deal task process result + self._process_task_result(event) + elif topic_name == settings.SYSOM_CEC_TASK_GENERATE_TOPIC: + # Deal generate task + self._generate_task(event) + except Exception as exc: + logger.exception(exc) + finally: + consumer.ack(event) + ################################################################################################ # 诊断任务下发 ################################################################################################ + + def _generate_task(self, event: Event): + from lib.authentications import decode_token + try: + token = event.value.get("token", "") + data = event.value.get("data", {}) + data["user"] = decode_token(token) + self.delivery_task(data) + except Exception as exc: + # TODO: 任务下发过程中发生任何错误,都应该反馈错误到事件中心 + logger.exception(exc) + def delivery_task(self, data: dict): """ 将一个任务投递到事件中心,供任务执行者消费 @@ -319,10 +173,6 @@ class TaskDispatcher(CecClient): # 诊断结果处理 ################################################################################################ - def on_receive_event(self, consumer: Consumer, event: Event, task: dict): - self._process_task_result(event) - consumer.ack(event) - def _process_task_result(self, event: Event): """ 处理任务执行结果回调 @@ -344,48 +194,60 @@ class TaskDispatcher(CecClient): finally: connection.close() task_result = event.value - task_id, status, errMsg, results = task_result.get("task_id", ""), task_result.get( + task_id, status, err_msg, results = task_result.get("task_id", ""), task_result.get( "status", 1), task_result.get("errMsg", ""), task_result.get("results") result = "" if len(results) == 0 else results[-1] + service_name = "unknown" + task = {} try: # 获取到 Task 实例 instance = JobModel.objects.get(task_id=task_id) # 如果任务执行失败,更新状态 if status != 0: - update_job(instance, status="Fail", result=errMsg) + update_job(instance, status="Fail", result=err_msg) return # 如果任务执行成功,则执行后处理脚本 params = instance.params - service_name = params.get("service_name", None) + service_name = params.get("service_name", "unknown") # 执行后处理脚本,将结果整理成前端可识别的规范结构 - if service_name: - SCRIPTS_DIR = settings.SCRIPTS_DIR - service_post_name = service_name + '_post' - service_post_path = os.path.join( - SCRIPTS_DIR, service_post_name) - if os.path.exists(service_post_path): - # 创建一个临时文件,用于暂存中间结果 - fd, path = tempfile.mkstemp() - command_list = [service_post_path, - path, instance.task_id] - try: - # 将要传递的中间结果写入到临时文件当中 - with os.fdopen(fd, 'w') as tmp: - tmp.write(result) - resp = subprocess.run(command_list, stdout=subprocess.PIPE, - stderr=subprocess.PIPE) - if resp.returncode != 0: - logger.error( - f'执行失败: {resp.stderr.decode("utf-8")}') - update_job(instance, status="Fail", - result=resp.stderr.decode('utf-8')) - stdout = resp.stdout - result = json.loads(stdout.decode('utf-8')) - except Exception as e: - logger.error(f'ERROR: {e}') - update_job(instance, status="Fail", result=str(e)) - return - # 后处理脚本执行结束,更新任务状态 - update_job(instance, status="Success", result=result) + SCRIPTS_DIR = settings.SCRIPTS_DIR + service_post_name = service_name + '_post' + service_post_path = os.path.join( + SCRIPTS_DIR, service_post_name) + if os.path.exists(service_post_path): + # 创建一个临时文件,用于暂存中间结果 + fd, path = tempfile.mkstemp() + command_list = [service_post_path, + path, instance.task_id] + try: + # 将要传递的中间结果写入到临时文件当中 + with os.fdopen(fd, 'w') as tmp: + tmp.write(result) + resp = subprocess.run(command_list, stdout=subprocess.PIPE, + stderr=subprocess.PIPE) + if resp.returncode != 0: + logger.error( + f'执行失败: {resp.stderr.decode("utf-8")}') + update_job(instance, status="Fail", + result=resp.stderr.decode('utf-8')) + stdout = resp.stdout + result = json.loads(stdout.decode('utf-8')) + except Exception as e: + logger.error(f'ERROR: {e}') + update_job(instance, status="Fail", result=str(e)) + return + # 后处理脚本执行结束,更新任务状态 + update_job(instance, status="Success", result=result) except Exception as e: + status = 1 + err_msg = str(e) logger.error(e) + finally: + if instance is not None: + task = JobDetailSerializer(instance).data + # 将诊断任务最后是否执行成功写到事件中心当中 + self.delivery(settings.SYSOM_CEC_TASK_EXECUTE_RESULT_TOPIC, { + "status": status, + "err_msg": err_msg, + "task": task + }) diff --git a/sysom_api/apps/task/seriaizer.py b/sysom_api/apps/task/seriaizer.py index c4b6af92..40af11e6 100644 --- a/sysom_api/apps/task/seriaizer.py +++ b/sysom_api/apps/task/seriaizer.py @@ -10,15 +10,18 @@ from apps.task.models import JobModel class JobListSerializer(serializers.ModelSerializer): - class Meta: model = JobModel exclude = ('deleted_at', 'host_by', 'command', 'result') - class JobRetrieveSerializer(serializers.ModelSerializer): + class Meta: + model = JobModel + exclude = ('deleted_at', 'host_by', 'command') + +class JobDetailSerializer(serializers.ModelSerializer): class Meta: model = JobModel - exclude = ('deleted_at', 'host_by', 'command') \ No newline at end of file + exclude = ['deleted_at'] diff --git a/sysom_api/apps/task/views.py b/sysom_api/apps/task/views.py index 26a5dcb7..9f66872a 100644 --- a/sysom_api/apps/task/views.py +++ b/sysom_api/apps/task/views.py @@ -1,15 +1,9 @@ -import json -import os -import ast -import subprocess import logging from django.http.response import FileResponse from django_filters.rest_framework import DjangoFilterBackend from django.shortcuts import get_object_or_404 from rest_framework.filters import SearchFilter, OrderingFilter -from rest_framework.viewsets import GenericViewSet from rest_framework import mixins -from django.db.models import Q from django.conf import settings from apps.task import seriaizer @@ -17,8 +11,6 @@ from apps.task.models import JobModel from apps.task.filter import TaskFilter from apps.common.common_model_viewset import CommonModelViewSet from lib.response import success, other_response, not_found -from lib.utils import uuid_8, scheduler -from lib.exception import APIException from lib.authentications import TaskAuthentication from .executors import TaskDispatcher @@ -54,16 +46,7 @@ class TaskAPIView(CommonModelViewSet, return [auth() for auth in self.authentication_classes] def create(self, request, *args, **kwargs): - try: - data = request.data - for item in filter(lambda x: not x[1], [(field, data.get(field)) for field in self.create_requird_fields]): - return other_response(message=f'Field: {item[0]} is required!', code=400) - if IS_MICRO_SERVICES: - data['user'] = getattr(request, 'user') - return script_task(data) - except Exception as e: - logger.error(e, exc_info=True) - return other_response(message=str(e), code=400, success=False) + return self.create_task_v2(request, *args, **kwargs) def retrieve(self, request, *args, **kwargs): kwargs["task_id"] = kwargs.pop("pk") @@ -125,97 +108,4 @@ class TaskAPIView(CommonModelViewSet, """ res = self._task_executor.delivery_task(data) if res['success']: - return success(result=res["result"]) - - -def script_task(data): - try: - params = data.copy() - service_name = data.pop("service_name", None) - update_host_status = data.pop("update_host_status", False) - task_id = uuid_8() - username = data['username'] if data.get('username') else "admin" - - if IS_MICRO_SERVICES: - data.pop('user') - user = params.pop('user') - user_id = user['id'] - else: - from apps.accounts.models import User - user = User.objects.filter(username=username).first() - user_id = user.pk - - task = JobModel.objects.filter(status__in=["Running"], params=params) - if task: - return other_response( - message=f"node:{params['instance']}, There are tasks in progress, {params['service_name']}", - code=400, - success=False - ) - - SCRIPTS_DIR = settings.SCRIPTS_DIR - service_path = os.path.join(SCRIPTS_DIR, service_name) - if not os.path.exists(service_path): - logger.error("can not find script file, please check service name") - return other_response(message="can not find script file, please check service name", code=400, - success=False) - try: - command_list = [service_path, json.dumps(data)] - resp = subprocess.run(command_list, stdout=subprocess.PIPE, - stderr=subprocess.PIPE) - except Exception as e: - JobModel.objects.create(command='', task_id=task_id, - created_by=user_id, result=str(e), status="Fail") - logger.error(e, exc_info=True) - return other_response(message=str(e), code=400, success=False) - if resp.returncode != 0: - logger.error(str(resp.stderr.decode('utf-8'))) - JobModel.objects.create(command='', task_id=task_id, - created_by=user_id, result=resp.stderr.decode('utf-8'), status="Fail") - return other_response(message=str(resp.stderr.decode('utf-8')), code=400, success=False) - stdout = resp.stdout - stdout = stdout.decode('utf-8') - resp = ast.literal_eval(stdout) - resp_scripts = resp.get("commands") - if not resp_scripts: - logger.error("not find commands, Please check the script return") - JobModel.objects.create(command='', task_id=task_id, - created_by=user_id, result="not find commands, Please check the script return", - status="Fail") - return other_response(message="not find commands, Please check the script return", code=400, - success=False) - [script.update(data) for script in resp_scripts] - ssh_job( - resp_scripts, - task_id, - user_id, - params, - update_host_status=update_host_status, - service_name=service_name, - user=user - ) - return success(result={"task_id": task_id}) - except Exception as e: - logger.error(e, exc_info=True) - return other_response(message=str(e), code=400, success=False) - - -def ssh_job(resp_scripts, task_id: str, user_id: int, data=None, **kwargs): - """ - 创建任务, 并下发到调度器开始执行 - """ - create_args = dict() - create_args['command'] = resp_scripts - create_args['task_id'] = task_id - create_args['created_by'] = user_id - - if data: - create_args['params'] = data - try: - JobModel.objects.create(**create_args) - except: - raise APIException(message='任务创建失败!') - - from .executors import SshJob - sch_job = SshJob(resp_scripts, task_id, **kwargs) - scheduler.add_job(sch_job.run) + return success(result=res["result"]) \ No newline at end of file diff --git a/sysom_api/conf/common.py b/sysom_api/conf/common.py index 3d716939..edb29c2e 100644 --- a/sysom_api/conf/common.py +++ b/sysom_api/conf/common.py @@ -133,19 +133,19 @@ KEY_PATH = os.path.join(BASE_DIR, 'conf', 'ssh-key') ################################################################## SYSOM_CEC_URL = "redis://localhost:6379?cec_default_max_len=1000&cec_auto_mk_topic=true" SYSOM_CEC_ALARM_TOPIC = "CEC-SYSOM-ALARM" -# 任务模块用于下发任务的主题 +# 任务模块用于生成诊断任务的主题(可以通过将诊断参数发送到这个主题来触发诊断任务的执行) +SYSOM_CEC_TASK_GENERATE_TOPIC = "CEC-SYSOM-TASK-GENERATE-TOPCI" +SYSOM_CEC_TASK_GENERATE_TASK_LISTENER = "CEC-SYSOM-TASK-GENERATE-TASK-LISTENER" +# 任务模块用于下发任务的主题 => 执行完预处理脚本,待下发到node执行 SYSOM_CEC_TASK_DELIVERY_TOPIC = "CEC-SYSOM-TASK-DELIVERY" SYSOM_CEC_TASK_DELIVERY_GROUP = "CEC-SYSOM-TASK-DELIVERY-GROUP" # 读取任务的消费组 -# 任务模块用于下发任务处理结果的主题 +# 任务模块用于下发任务处理结果的主题 => node执行结束,待后处理脚本执行 SYSOM_CEC_TASK_RESULT_PROCESS_TOPIC = "CEC-SYSOM-TASK-RESULT-PROCESS" SYSOM_CEC_TASK_RESULT_PROCESS_GROUP = "CEC-SYSOM-TASK-RESULT-PROCESS-GROUP" - -# API -TASK_API = f'{TASK_SERVICE}/api/v1/tasks/' -HOST_LIST_API = f'{SYSOM_SERVICE}/api/v1/host/' -CHANNEL_VALIDATE_API = f'{CHANNEL_SERVICE}/api/v1/channel/validate/' -CHANNEL_COMMAND_API = f'{CHANNEL_SERVICE}/api/v1/channel/' -CHANNEL_RESULT_API = f'{CHANNEL_SERVICE}/api/v1/channel/exec_result/' +# 诊断任务最终执行结果报告 => 后处理脚本执行结束,报告最终任务的执行结果 +SYSOM_CEC_TASK_EXECUTE_RESULT_TOPIC = "CEC-SYSOM-TASK-EXECUTE-RESULT" +# 主机模块监听诊断任务执行结果消费组 +SYSOM_CEC_TASK_RESULT_LISTENER_HOST_GROUP = "CEC-SYSOM-TASK-RESULT-LISTENER-HOST-GROUP" SERVER_LOGS_FILE = os.path.join(BASE_DIR, 'logs', 'sys_om_info.log') ERROR_LOGS_FILE = os.path.join(BASE_DIR, 'logs', 'sys_om_error.log') -- Gitee From 857e2d9e9064ae8518115c95a0550f3dea3f5048 Mon Sep 17 00:00:00 2001 From: SunnyQjm Date: Thu, 20 Oct 2022 11:48:02 +0800 Subject: [PATCH 6/7] refactor: Modify the Host module to issue diagnostic tasks through the Event Center. --- sysom_api/apps/host/apps.py | 51 ++++++++++++++++++++++++++++++++++-- sysom_api/apps/host/views.py | 44 ++++++++++++++----------------- tools/deploy/deploy.sh | 6 ++--- 3 files changed, 72 insertions(+), 29 deletions(-) diff --git a/sysom_api/apps/host/apps.py b/sysom_api/apps/host/apps.py index 9c2f3652..558d25cd 100644 --- a/sysom_api/apps/host/apps.py +++ b/sysom_api/apps/host/apps.py @@ -1,13 +1,60 @@ import logging +import sys from django.apps import AppConfig - +from sdk.cec_base.cec_client import CecClient +from sdk.cec_base.consumer import Consumer +from sdk.cec_base.event import Event logger = logging.getLogger(__name__) +def _on_receive_task_result(consumer: Consumer, event: Event, task: dict): + """Host module deal task result here + + { + "status":0, + "err_msg":"", + "task": {} + } + """ + from .models import HostModel + try: + result = event.value + status = result.get("status", 1) + task = result.get("task", {}) + params = task.get("params", {}) + service_name = params.get("service_name", "Unknown") + if service_name != "node_init": + return + instance_ip = params.get("instance", "Unknown") + instance = HostModel.objects.get(ip=instance_ip) + instance.status = status + instance.save() + except Exception as exc: + logger.exception(exc) + finally: + consumer.ack(event) + + class HostConfig(AppConfig): default_auto_field = 'django.db.models.BigAutoField' name = 'apps.host' def ready(self): - logger.info(">>> Host module loading success") \ No newline at end of file + if 'runserver' in sys.argv or 'manage.py' not in sys.argv: + from django.conf import settings + # 这边微服务正式启动的时候执行一些处理代码 + # 启动任务结果处理线程 + try: + cec_client = CecClient(settings.SYSOM_CEC_URL, custom_callback=_on_receive_task_result) + cec_client.append_group_consume_task( + settings.SYSOM_CEC_TASK_EXECUTE_RESULT_TOPIC, + settings.SYSOM_CEC_TASK_RESULT_LISTENER_HOST_GROUP + ) + cec_client.start() + except Exception as e: + logger.exception(e) + else: + # 这边执行数据库迁移等操作的时候执行一些处理代码 + pass + logger.info(">>> Host module loading success") diff --git a/sysom_api/apps/host/views.py b/sysom_api/apps/host/views.py index 048f5732..16a9aa62 100644 --- a/sysom_api/apps/host/views.py +++ b/sysom_api/apps/host/views.py @@ -2,6 +2,7 @@ import re import logging import os import threading +from typing import Any from drf_yasg import openapi from drf_yasg.utils import swagger_auto_schema from rest_framework.request import Request @@ -43,6 +44,9 @@ class HostModelViewSet(CommonModelViewSet, filterset_fields = ['ip', 'hostname', 'cluster', 'status'] http_method_names = ['get', 'post', 'patch', 'delete'] + def __init__(self, **kwargs: Any) -> None: + super().__init__(**kwargs) + def get_authenticators(self): if self.request.method == "GET": return [] @@ -135,10 +139,7 @@ class HostModelViewSet(CommonModelViewSet, def _destroy_host_tasks(self, instance): ser = serializer.HostSerializer(instance=instance) self.perform_destroy(instance) - status, content = self.client_deploy_cmd_delete(instance) - if status != 200: - raise APIException( - message=f'主机已删除,清除脚本执行失败,错误如下:{content["message"]}') + self.client_deploy_cmd_delete(instance) return ser def check_instance_exist(self, request, *args, **kwargs): @@ -154,28 +155,23 @@ class HostModelViewSet(CommonModelViewSet, kwargs = {} kwargs['data'] = data - kwargs['url'] = settings.TASK_API kwargs['token'] = request.META.get('HTTP_AUTHORIZATION') - status, res = HTTP.request('post', **kwargs) - logger.info(f'node init task create success, res {res}') + self.produce_event_to_cec( + settings.SYSOM_CEC_TASK_GENERATE_TOPIC, kwargs) + logger.info(f'node init task create success') def client_deploy_cmd_delete(self, instance): request: Request = getattr(self, 'request') - try: - data = {} - data['service_name'] = "node_delete" - data['instance'] = instance.ip - - kwargs = {} - kwargs['data'] = data - kwargs['url'] = settings.TASK_API - kwargs['token'] = request.META.get('HTTP_AUTHORIZATION') - - status, res = HTTP.request('post', **kwargs) - logger.info(f'node delete task create success, res {res}') - return status, res - except Exception as e: - return False, str(e) + data = {} + data['service_name'] = "node_delete" + data['instance'] = instance.ip + + kwargs = {} + kwargs['data'] = data + kwargs['token'] = request.META.get('HTTP_AUTHORIZATION') + self.produce_event_to_cec( + settings.SYSOM_CEC_TASK_GENERATE_TOPIC, kwargs) + logger.info(f'node delete task create success') def _get_cluster_instance(self, cluster_name): try: @@ -324,7 +320,7 @@ class HostModelViewSet(CommonModelViewSet, HostModel.objects.get(ip=host_ip) return success(result={}) except HostModel.DoesNotExist: - raise APIException(message=f'Error: ip {host_ip} not exist!') + raise APIException(message=f'Error: ip {host_ip} not exist!') def _validate_ip_format(self, ip) -> bool: p = '((\d{1,2})|([01]\d{2})|(2[0-4]\d)|(25[0-5]))' @@ -447,7 +443,7 @@ class ClusterViewSet(CommonModelViewSet, else: # 包含主机不允许删除 kwargs.update( - {'message': f"Cluster({instance.cluster_name}) contains hosts, delete failed!"}) + {'message': f"Cluster({instance.cluster_name}) contains hosts, delete failed!"}) kwargs.update({'collected_time': human_datetime()}) _create_alarm_message(kwargs) fail_list.append(instance.cluster_name) diff --git a/tools/deploy/deploy.sh b/tools/deploy/deploy.sh index 1da8dd30..bb1db53e 100755 --- a/tools/deploy/deploy.sh +++ b/tools/deploy/deploy.sh @@ -83,18 +83,18 @@ init_conf() { cp tools/deploy/sysom.conf /etc/nginx/conf.d/ cp tools/deploy/sysom.ini /etc/supervisord.d/ cp tools/deploy/task-service.ini /etc/supervisord.d/ - cp tools/deploy/channel-service.ini /etc/supervisord.d/ + # cp tools/deploy/channel-service.ini /etc/supervisord.d/ cp tools/deploy/task-executor-service.ini /etc/supervisord.d/ ###change the install dir base on param $1### sed -i "s;/usr/local/sysom;${APP_HOME};g" /etc/nginx/conf.d/sysom.conf sed -i "s;/usr/local/sysom;${APP_HOME};g" /etc/supervisord.d/sysom.ini sed -i "s;/usr/local/sysom;${APP_HOME};g" /etc/supervisord.d/task-service.ini - sed -i "s;/usr/local/sysom;${APP_HOME};g" /etc/supervisord.d/channel-service.ini + # sed -i "s;/usr/local/sysom;${APP_HOME};g" /etc/supervisord.d/channel-service.ini sed -i "s;/usr/local/sysom;${APP_HOME};g" /etc/supervisord.d/task-executor-service.ini cp tools/deploy/sysom-server.service /usr/lib/systemd/system/ cpu_num=`cat /proc/cpuinfo | grep processor | wc -l` sed -i "s/threads = 3/threads = $cpu_num/g" ${TARGET_PATH}/${API_DIR}/conf/task_gunicorn.py - sed -i "s/threads = 3/threads = $cpu_num/g" ${TARGET_PATH}/${API_DIR}/conf/channel_gunicorn.py + # sed -i "s/threads = 3/threads = $cpu_num/g" ${TARGET_PATH}/${API_DIR}/conf/channel_gunicorn.py } start_script_server() { -- Gitee From 13e5f1a9e94ffb883b2e7c9534f1ed91cf395513 Mon Sep 17 00:00:00 2001 From: SunnyQjm Date: Sun, 23 Oct 2022 22:17:25 +0800 Subject: [PATCH 7/7] reconstruct: Extract diagnostic and channel module --- package.sh | 4 +- script/server/0_sysom_api/init.sh | 28 ++- script/server/1_monitor/grafana_api_set.sh | 2 +- .../server/1_monitor/monitor_server_deploy.sh | 4 +- sysom_api/apps/channel/apps.py | 46 ----- sysom_api/apps/channel/models.py | 26 --- sysom_api/apps/channel/urls.py | 8 - sysom_api/apps/channel/views.py | 139 ------------- sysom_api/microservices/README.md | 0 sysom_api/microservices/task_executor/app.py | 27 --- sysom_api/microservices/task_executor/conf.py | 1 - .../sysom/services_conf/channel_settings.py | 28 --- .../sysom/services_conf/task_settings.py | 28 --- sysom_server/.gitignore | 3 + sysom_server/sdk/.gitignore | 3 + .../sdk/cec_base/__init__.py | 0 .../sdk/cec_base/admin.py | 2 +- .../sdk/cec_base/base.py | 0 .../sdk/cec_base/cec_client.py | 15 +- .../sdk/cec_base/cli.py | 0 .../sdk/cec_base/consumer.py | 4 +- .../sdk/cec_base/event.py | 0 .../sdk/cec_base/exceptions.py | 0 .../sdk/cec_base/log.py | 0 .../sdk/cec_base/meta.py | 0 .../sdk/cec_base/producer.py | 2 +- .../sdk/cec_base/url.py | 0 .../sdk/cec_redis/__init__.py | 0 .../sdk/cec_redis/admin_static.py | 8 +- .../sdk/cec_redis/common.py | 2 +- .../sdk/cec_redis/consume_status_storage.py | 2 +- .../sdk/cec_redis/heartbeat.py | 4 +- .../sdk/cec_redis/redis_admin.py | 12 +- .../sdk/cec_redis/redis_consumer.py | 10 +- .../sdk/cec_redis/redis_producer.py | 10 +- .../sdk/cec_redis/utils.py | 6 +- sysom_server/sdk/setup_cec_base.py | 22 ++ sysom_server/sdk/setup_cec_redis.py | 17 ++ .../sysom_api}/.gitignore | 0 .../sysom_api}/apps/__init__.py | 0 .../sysom_api}/apps/accounts/__init__.py | 0 .../sysom_api}/apps/accounts/admin.py | 0 .../sysom_api}/apps/accounts/apps.py | 0 .../apps/accounts/authentication.py | 0 .../sysom_api}/apps/accounts/models.py | 0 .../sysom_api}/apps/accounts/permissions.py | 0 .../sysom_api}/apps/accounts/role.json | 0 .../sysom_api}/apps/accounts/serializer.py | 0 .../sysom_api}/apps/accounts/tests.py | 0 .../sysom_api}/apps/accounts/urls.py | 0 .../sysom_api}/apps/accounts/user.json | 0 .../sysom_api}/apps/accounts/views.py | 0 .../sysom_api}/apps/alarm/__init__.py | 0 .../sysom_api}/apps/alarm/admin.py | 0 .../sysom_api}/apps/alarm/apps.py | 0 .../sysom_api}/apps/alarm/models.py | 0 .../sysom_api}/apps/alarm/serializer.py | 0 .../sysom_api}/apps/alarm/subscribe.json | 0 .../sysom_api}/apps/alarm/urls.py | 0 .../sysom_api}/apps/alarm/views.py | 4 +- .../sysom_api/apps/common}/__init__.py | 0 .../apps/common/common_model_viewset.py | 2 +- .../sysom_api/apps/host}/__init__.py | 0 .../sysom_api/apps/host}/admin.py | 0 .../sysom_api}/apps/host/apps.py | 81 ++++++-- .../sysom_api}/apps/host/models.py | 0 .../sysom_api}/apps/host/serializer.py | 0 .../sysom_api/apps/host}/tests.py | 0 .../sysom_api}/apps/host/urls.py | 0 .../sysom_api}/apps/host/views.py | 91 +++++---- .../sysom_api/apps/monitor}/__init__.py | 0 .../sysom_api/apps/monitor}/admin.py | 0 .../sysom_api}/apps/monitor/apps.py | 0 .../sysom_api}/apps/monitor/models.py | 0 .../sysom_api/apps/monitor}/tests.py | 0 .../sysom_api}/apps/monitor/urls.py | 0 .../sysom_api}/apps/monitor/views.py | 0 .../sysom_api/apps/vmcore}/__init__.py | 0 .../sysom_api/apps/vmcore}/admin.py | 0 .../sysom_api}/apps/vmcore/apps.py | 0 .../sysom_api}/apps/vmcore/models.py | 0 .../sysom_api}/apps/vmcore/serializer.py | 0 .../sysom_api/apps/vmcore}/tests.py | 0 .../sysom_api}/apps/vmcore/urls.py | 0 .../sysom_api}/apps/vmcore/views.py | 0 .../sysom_api}/apps/vmcore/vmcore.json | 0 .../sysom_api/apps/vul}/__init__.py | 0 .../sysom_api/apps/vul}/admin.py | 0 .../sysom_api}/apps/vul/apps.py | 31 ++- .../sysom_api}/apps/vul/models.py | 0 .../sysom_api}/apps/vul/serializer.py | 0 .../sysom_api}/apps/vul/ssh_pool.py | 2 +- .../sysom_api/apps/vul}/tests.py | 0 .../sysom_api}/apps/vul/urls.py | 0 .../sysom_api}/apps/vul/views.py | 0 .../sysom_api}/apps/vul/vul.py | 0 .../sysom_api/conf}/__init__.py | 0 .../sysom_api}/conf/common.py | 37 ++-- .../sysom_api}/conf/develop.py | 0 .../sysom_api}/conf/product.py | 0 .../sysom_api}/conf/testing.py | 0 .../sysom_api}/consumer/__init__.py | 0 .../sysom_api}/consumer/consumers.py | 4 +- .../sysom_api}/consumer/middleware.py | 0 .../sysom_api}/consumer/routing.py | 0 .../sysom_api}/lib/__init__.py | 0 .../sysom_api}/lib/authentications.py | 0 .../sysom_api}/lib/base_model.py | 0 .../sysom_api}/lib/decode/sysom_decode.py | 0 .../sysom_api}/lib/excel.py | 0 .../sysom_api}/lib/exception.py | 0 .../sysom_api}/lib/paginations.py | 0 .../sysom_api}/lib/renderers.py | 0 .../sysom_api}/lib/response.py | 0 sysom_server/sysom_api/lib/ssh.py | 100 ++++++++++ .../sysom_api}/lib/utils.py | 8 + .../sysom_api}/manage.py | 0 .../sysom_api}/sysom/__init__.py | 0 .../sysom_api}/sysom/asgi.py | 0 .../sysom_api}/sysom/settings.py | 0 .../sysom_api}/sysom/urls.py | 0 .../sysom_api}/sysom/wsgi.py | 0 .../sysom_channel/apps/channel}/__init__.py | 0 .../sysom_channel/apps/channel}/admin.py | 0 .../sysom_channel/apps/channel/apps.py | 102 ++++++++++ .../sysom_channel/apps/channel}/executor.py | 156 ++++++++------- .../sysom_channel/apps/channel/models.py | 14 ++ .../sysom_channel/apps/channel}/seriaizer.py | 6 +- .../sysom_channel/apps/channel}/tests.py | 0 .../sysom_channel/apps/channel/urls.py | 13 ++ .../sysom_channel/apps/channel/views.py | 12 ++ .../sysom_channel}/conf/channel_gunicorn.py | 3 +- sysom_server/sysom_channel/conf/common.py | 188 ++++++++++++++++++ sysom_server/sysom_channel/conf/develop.py | 7 + sysom_server/sysom_channel/conf/product.py | 7 + sysom_server/sysom_channel/conf/ssh/sysom_id | 27 +++ .../sysom_channel/conf/ssh/sysom_id.pub | 1 + sysom_server/sysom_channel/conf/testing.py | 6 + sysom_server/sysom_channel/lib/base_model.py | 20 ++ sysom_server/sysom_channel/lib/base_view.py | 90 +++++++++ .../sysom_channel/lib/channels}/__init__.py | 0 .../sysom_channel/lib/channels/base.py | 31 +++ .../sysom_channel/lib/channels/ssh.py | 62 ++++++ sysom_server/sysom_channel/lib/exception.py | 63 ++++++ sysom_server/sysom_channel/lib/paginations.py | 26 +++ sysom_server/sysom_channel/lib/renderers.py | 42 ++++ sysom_server/sysom_channel/lib/response.py | 68 +++++++ sysom_server/sysom_channel/lib/ssh.py | 100 ++++++++++ sysom_server/sysom_channel/lib/utils.py | 78 ++++++++ sysom_server/sysom_channel/manage.py | 22 ++ .../sysom_channel/sysom_channel/__init__.py | 3 + .../sysom_channel/sysom_channel/asgi.py | 16 ++ .../sysom_channel/sysom_channel/settings.py | 15 ++ .../sysom_channel/sysom_channel/urls.py | 26 +++ .../sysom_channel/sysom_channel/wsgi.py | 16 ++ .../sysom_diagnosis/apps/task}/__init__.py | 0 .../sysom_diagnosis/apps/task}/admin.py | 0 .../sysom_diagnosis}/apps/task/apps.py | 9 +- .../sysom_diagnosis}/apps/task/executors.py | 124 +++++++++--- .../sysom_diagnosis}/apps/task/filter.py | 0 .../sysom_diagnosis}/apps/task/models.py | 0 .../sysom_diagnosis/apps/task/seriaizer.py | 27 +++ .../sysom_diagnosis/apps/task}/tests.py | 0 .../sysom_diagnosis}/apps/task/urls.py | 0 .../sysom_diagnosis}/apps/task/views.py | 14 +- sysom_server/sysom_diagnosis/conf/common.py | 183 +++++++++++++++++ sysom_server/sysom_diagnosis/conf/develop.py | 7 + .../conf/diagnosis_gunicorn.py | 10 +- sysom_server/sysom_diagnosis/conf/product.py | 7 + sysom_server/sysom_diagnosis/conf/testing.py | 6 + .../sysom_diagnosis/lib/authentications.py | 50 +++++ .../sysom_diagnosis/lib/base_model.py | 20 ++ sysom_server/sysom_diagnosis/lib/base_view.py | 90 +++++++++ .../sysom_diagnosis/lib/channels}/__init__.py | 0 .../sysom_diagnosis}/lib/channels/base.py | 0 .../sysom_diagnosis}/lib/channels/ssh.py | 12 +- .../lib/decode/sysom_decode.py | 17 ++ sysom_server/sysom_diagnosis/lib/exception.py | 63 ++++++ .../sysom_diagnosis/lib/paginations.py | 26 +++ sysom_server/sysom_diagnosis/lib/renderers.py | 42 ++++ sysom_server/sysom_diagnosis/lib/response.py | 68 +++++++ sysom_server/sysom_diagnosis/lib/utils.py | 69 +++++++ sysom_server/sysom_diagnosis/manage.py | 22 ++ .../service_scripts/filecache | 0 .../service_scripts/filecache_post | 0 .../service_scripts/iosdiag_latency | 0 .../service_scripts/iosdiag_latency_post | 0 .../sysom_diagnosis}/service_scripts/loadtask | 0 .../service_scripts/loadtask_post | 0 .../sysom_diagnosis}/service_scripts/memgraph | 0 .../service_scripts/memgraph_post | 0 .../service_scripts/node_delete | 0 .../service_scripts/node_init | 0 .../sysom_diagnosis}/service_scripts/oomcheck | 0 .../service_scripts/oomcheck_post | 0 .../sysom_diagnosis}/service_scripts/ossre | 0 .../service_scripts/ossre_post | 0 .../service_scripts/pingtrace | 0 .../service_scripts/pingtrace_post | 0 .../service_scripts/schedmoni | 0 .../service_scripts/schedmoni_post | 0 .../sysom_diagnosis}/service_scripts/test | 0 .../sysom_diagnosis}/service_scripts/test.sh | 0 .../sysom_diagnosis}/service_scripts/tpython | 0 .../service_scripts/vmcore_analyse | 0 .../sysom_diagnosis/__init__.py | 3 + .../sysom_diagnosis/sysom_diagnosis/asgi.py | 16 ++ .../sysom_diagnosis/settings.py | 15 ++ .../sysom_diagnosis/sysom_diagnosis/urls.py | 26 +++ .../sysom_diagnosis/sysom_diagnosis/wsgi.py | 16 ++ tools/deploy/channel-service.ini | 4 +- tools/deploy/deploy.sh | 22 +- ...task-service.ini => diagnosis-service.ini} | 10 +- tools/deploy/sysom.conf | 9 +- tools/deploy/sysom.ini | 2 +- tools/deploy/task-executor-service.ini | 9 - 216 files changed, 2441 insertions(+), 600 deletions(-) delete mode 100644 sysom_api/apps/channel/apps.py delete mode 100644 sysom_api/apps/channel/models.py delete mode 100644 sysom_api/apps/channel/urls.py delete mode 100644 sysom_api/apps/channel/views.py delete mode 100644 sysom_api/microservices/README.md delete mode 100644 sysom_api/microservices/task_executor/app.py delete mode 100644 sysom_api/microservices/task_executor/conf.py delete mode 100644 sysom_api/sysom/services_conf/channel_settings.py delete mode 100644 sysom_api/sysom/services_conf/task_settings.py create mode 100644 sysom_server/.gitignore create mode 100644 sysom_server/sdk/.gitignore rename {sysom_api => sysom_server}/sdk/cec_base/__init__.py (100%) rename {sysom_api => sysom_server}/sdk/cec_base/admin.py (99%) rename {sysom_api => sysom_server}/sdk/cec_base/base.py (100%) rename {sysom_api => sysom_server}/sdk/cec_base/cec_client.py (91%) rename {sysom_api => sysom_server}/sdk/cec_base/cli.py (100%) rename {sysom_api => sysom_server}/sdk/cec_base/consumer.py (98%) rename {sysom_api => sysom_server}/sdk/cec_base/event.py (100%) rename {sysom_api => sysom_server}/sdk/cec_base/exceptions.py (100%) rename {sysom_api => sysom_server}/sdk/cec_base/log.py (100%) rename {sysom_api => sysom_server}/sdk/cec_base/meta.py (100%) rename {sysom_api => sysom_server}/sdk/cec_base/producer.py (98%) rename {sysom_api => sysom_server}/sdk/cec_base/url.py (100%) rename {sysom_api => sysom_server}/sdk/cec_redis/__init__.py (100%) rename {sysom_api => sysom_server}/sdk/cec_redis/admin_static.py (99%) rename {sysom_api => sysom_server}/sdk/cec_redis/common.py (99%) rename {sysom_api => sysom_server}/sdk/cec_redis/consume_status_storage.py (99%) rename {sysom_api => sysom_server}/sdk/cec_redis/heartbeat.py (99%) rename {sysom_api => sysom_server}/sdk/cec_redis/redis_admin.py (98%) rename {sysom_api => sysom_server}/sdk/cec_redis/redis_consumer.py (98%) rename {sysom_api => sysom_server}/sdk/cec_redis/redis_producer.py (97%) rename {sysom_api => sysom_server}/sdk/cec_redis/utils.py (96%) create mode 100644 sysom_server/sdk/setup_cec_base.py create mode 100644 sysom_server/sdk/setup_cec_redis.py rename {sysom_api => sysom_server/sysom_api}/.gitignore (100%) rename {sysom_api => sysom_server/sysom_api}/apps/__init__.py (100%) rename {sysom_api => sysom_server/sysom_api}/apps/accounts/__init__.py (100%) rename {sysom_api => sysom_server/sysom_api}/apps/accounts/admin.py (100%) rename {sysom_api => sysom_server/sysom_api}/apps/accounts/apps.py (100%) rename {sysom_api => sysom_server/sysom_api}/apps/accounts/authentication.py (100%) rename {sysom_api => sysom_server/sysom_api}/apps/accounts/models.py (100%) rename {sysom_api => sysom_server/sysom_api}/apps/accounts/permissions.py (100%) rename {sysom_api => sysom_server/sysom_api}/apps/accounts/role.json (100%) rename {sysom_api => sysom_server/sysom_api}/apps/accounts/serializer.py (100%) rename {sysom_api => sysom_server/sysom_api}/apps/accounts/tests.py (100%) rename {sysom_api => sysom_server/sysom_api}/apps/accounts/urls.py (100%) rename {sysom_api => sysom_server/sysom_api}/apps/accounts/user.json (100%) rename {sysom_api => sysom_server/sysom_api}/apps/accounts/views.py (100%) rename {sysom_api => sysom_server/sysom_api}/apps/alarm/__init__.py (100%) rename {sysom_api => sysom_server/sysom_api}/apps/alarm/admin.py (100%) rename {sysom_api => sysom_server/sysom_api}/apps/alarm/apps.py (100%) rename {sysom_api => sysom_server/sysom_api}/apps/alarm/models.py (100%) rename {sysom_api => sysom_server/sysom_api}/apps/alarm/serializer.py (100%) rename {sysom_api => sysom_server/sysom_api}/apps/alarm/subscribe.json (100%) rename {sysom_api => sysom_server/sysom_api}/apps/alarm/urls.py (100%) rename {sysom_api => sysom_server/sysom_api}/apps/alarm/views.py (97%) rename {sysom_api/apps/channel => sysom_server/sysom_api/apps/common}/__init__.py (100%) rename {sysom_api => sysom_server/sysom_api}/apps/common/common_model_viewset.py (98%) rename {sysom_api/apps/common => sysom_server/sysom_api/apps/host}/__init__.py (100%) rename {sysom_api/apps/channel => sysom_server/sysom_api/apps/host}/admin.py (100%) rename {sysom_api => sysom_server/sysom_api}/apps/host/apps.py (31%) rename {sysom_api => sysom_server/sysom_api}/apps/host/models.py (100%) rename {sysom_api => sysom_server/sysom_api}/apps/host/serializer.py (100%) rename {sysom_api/apps/channel => sysom_server/sysom_api/apps/host}/tests.py (100%) rename {sysom_api => sysom_server/sysom_api}/apps/host/urls.py (100%) rename {sysom_api => sysom_server/sysom_api}/apps/host/views.py (90%) rename {sysom_api/apps/host => sysom_server/sysom_api/apps/monitor}/__init__.py (100%) rename {sysom_api/apps/host => sysom_server/sysom_api/apps/monitor}/admin.py (100%) rename {sysom_api => sysom_server/sysom_api}/apps/monitor/apps.py (100%) rename {sysom_api => sysom_server/sysom_api}/apps/monitor/models.py (100%) rename {sysom_api/apps/host => sysom_server/sysom_api/apps/monitor}/tests.py (100%) rename {sysom_api => sysom_server/sysom_api}/apps/monitor/urls.py (100%) rename {sysom_api => sysom_server/sysom_api}/apps/monitor/views.py (100%) rename {sysom_api/apps/monitor => sysom_server/sysom_api/apps/vmcore}/__init__.py (100%) rename {sysom_api/apps/monitor => sysom_server/sysom_api/apps/vmcore}/admin.py (100%) rename {sysom_api => sysom_server/sysom_api}/apps/vmcore/apps.py (100%) rename {sysom_api => sysom_server/sysom_api}/apps/vmcore/models.py (100%) rename {sysom_api => sysom_server/sysom_api}/apps/vmcore/serializer.py (100%) rename {sysom_api/apps/monitor => sysom_server/sysom_api/apps/vmcore}/tests.py (100%) rename {sysom_api => sysom_server/sysom_api}/apps/vmcore/urls.py (100%) rename {sysom_api => sysom_server/sysom_api}/apps/vmcore/views.py (100%) rename {sysom_api => sysom_server/sysom_api}/apps/vmcore/vmcore.json (100%) rename {sysom_api/apps/task => sysom_server/sysom_api/apps/vul}/__init__.py (100%) rename {sysom_api/apps/task => sysom_server/sysom_api/apps/vul}/admin.py (100%) rename {sysom_api => sysom_server/sysom_api}/apps/vul/apps.py (42%) rename {sysom_api => sysom_server/sysom_api}/apps/vul/models.py (100%) rename {sysom_api => sysom_server/sysom_api}/apps/vul/serializer.py (100%) rename {sysom_api => sysom_server/sysom_api}/apps/vul/ssh_pool.py (98%) rename {sysom_api/apps/task => sysom_server/sysom_api/apps/vul}/tests.py (100%) rename {sysom_api => sysom_server/sysom_api}/apps/vul/urls.py (100%) rename {sysom_api => sysom_server/sysom_api}/apps/vul/views.py (100%) rename {sysom_api => sysom_server/sysom_api}/apps/vul/vul.py (100%) rename {sysom_api/apps/vmcore => sysom_server/sysom_api/conf}/__init__.py (100%) rename {sysom_api => sysom_server/sysom_api}/conf/common.py (81%) rename {sysom_api => sysom_server/sysom_api}/conf/develop.py (100%) rename {sysom_api => sysom_server/sysom_api}/conf/product.py (100%) rename {sysom_api => sysom_server/sysom_api}/conf/testing.py (100%) rename {sysom_api => sysom_server/sysom_api}/consumer/__init__.py (100%) rename {sysom_api => sysom_server/sysom_api}/consumer/consumers.py (97%) rename {sysom_api => sysom_server/sysom_api}/consumer/middleware.py (100%) rename {sysom_api => sysom_server/sysom_api}/consumer/routing.py (100%) rename {sysom_api => sysom_server/sysom_api}/lib/__init__.py (100%) rename {sysom_api => sysom_server/sysom_api}/lib/authentications.py (100%) rename {sysom_api => sysom_server/sysom_api}/lib/base_model.py (100%) rename {sysom_api => sysom_server/sysom_api}/lib/decode/sysom_decode.py (100%) rename {sysom_api => sysom_server/sysom_api}/lib/excel.py (100%) rename {sysom_api => sysom_server/sysom_api}/lib/exception.py (100%) rename {sysom_api => sysom_server/sysom_api}/lib/paginations.py (100%) rename {sysom_api => sysom_server/sysom_api}/lib/renderers.py (100%) rename {sysom_api => sysom_server/sysom_api}/lib/response.py (100%) create mode 100644 sysom_server/sysom_api/lib/ssh.py rename {sysom_api => sysom_server/sysom_api}/lib/utils.py (95%) rename {sysom_api => sysom_server/sysom_api}/manage.py (100%) rename {sysom_api => sysom_server/sysom_api}/sysom/__init__.py (100%) rename {sysom_api => sysom_server/sysom_api}/sysom/asgi.py (100%) rename {sysom_api => sysom_server/sysom_api}/sysom/settings.py (100%) rename {sysom_api => sysom_server/sysom_api}/sysom/urls.py (100%) rename {sysom_api => sysom_server/sysom_api}/sysom/wsgi.py (100%) rename {sysom_api/apps/vul => sysom_server/sysom_channel/apps/channel}/__init__.py (100%) rename {sysom_api/apps/vmcore => sysom_server/sysom_channel/apps/channel}/admin.py (100%) create mode 100644 sysom_server/sysom_channel/apps/channel/apps.py rename {sysom_api/microservices/task_executor => sysom_server/sysom_channel/apps/channel}/executor.py (32%) create mode 100644 sysom_server/sysom_channel/apps/channel/models.py rename {sysom_api/apps/task => sysom_server/sysom_channel/apps/channel}/seriaizer.py (76%) rename {sysom_api/apps/vmcore => sysom_server/sysom_channel/apps/channel}/tests.py (100%) create mode 100644 sysom_server/sysom_channel/apps/channel/urls.py create mode 100644 sysom_server/sysom_channel/apps/channel/views.py rename {sysom_api => sysom_server/sysom_channel}/conf/channel_gunicorn.py (86%) create mode 100644 sysom_server/sysom_channel/conf/common.py create mode 100644 sysom_server/sysom_channel/conf/develop.py create mode 100644 sysom_server/sysom_channel/conf/product.py create mode 100644 sysom_server/sysom_channel/conf/ssh/sysom_id create mode 100644 sysom_server/sysom_channel/conf/ssh/sysom_id.pub create mode 100644 sysom_server/sysom_channel/conf/testing.py create mode 100644 sysom_server/sysom_channel/lib/base_model.py create mode 100644 sysom_server/sysom_channel/lib/base_view.py rename {sysom_api/conf => sysom_server/sysom_channel/lib/channels}/__init__.py (100%) create mode 100644 sysom_server/sysom_channel/lib/channels/base.py create mode 100644 sysom_server/sysom_channel/lib/channels/ssh.py create mode 100644 sysom_server/sysom_channel/lib/exception.py create mode 100644 sysom_server/sysom_channel/lib/paginations.py create mode 100644 sysom_server/sysom_channel/lib/renderers.py create mode 100644 sysom_server/sysom_channel/lib/response.py create mode 100644 sysom_server/sysom_channel/lib/ssh.py create mode 100644 sysom_server/sysom_channel/lib/utils.py create mode 100755 sysom_server/sysom_channel/manage.py create mode 100644 sysom_server/sysom_channel/sysom_channel/__init__.py create mode 100644 sysom_server/sysom_channel/sysom_channel/asgi.py create mode 100644 sysom_server/sysom_channel/sysom_channel/settings.py create mode 100644 sysom_server/sysom_channel/sysom_channel/urls.py create mode 100644 sysom_server/sysom_channel/sysom_channel/wsgi.py rename {sysom_api/lib/channels => sysom_server/sysom_diagnosis/apps/task}/__init__.py (100%) rename {sysom_api/apps/vul => sysom_server/sysom_diagnosis/apps/task}/admin.py (100%) rename {sysom_api => sysom_server/sysom_diagnosis}/apps/task/apps.py (82%) rename {sysom_api => sysom_server/sysom_diagnosis}/apps/task/executors.py (72%) rename {sysom_api => sysom_server/sysom_diagnosis}/apps/task/filter.py (100%) rename {sysom_api => sysom_server/sysom_diagnosis}/apps/task/models.py (100%) create mode 100644 sysom_server/sysom_diagnosis/apps/task/seriaizer.py rename {sysom_api/apps/vul => sysom_server/sysom_diagnosis/apps/task}/tests.py (100%) rename {sysom_api => sysom_server/sysom_diagnosis}/apps/task/urls.py (100%) rename {sysom_api => sysom_server/sysom_diagnosis}/apps/task/views.py (92%) create mode 100644 sysom_server/sysom_diagnosis/conf/common.py create mode 100644 sysom_server/sysom_diagnosis/conf/develop.py rename sysom_api/conf/task_gunicorn.py => sysom_server/sysom_diagnosis/conf/diagnosis_gunicorn.py (51%) create mode 100644 sysom_server/sysom_diagnosis/conf/product.py create mode 100644 sysom_server/sysom_diagnosis/conf/testing.py create mode 100644 sysom_server/sysom_diagnosis/lib/authentications.py create mode 100644 sysom_server/sysom_diagnosis/lib/base_model.py create mode 100644 sysom_server/sysom_diagnosis/lib/base_view.py rename {sysom_api/sysom/services_conf => sysom_server/sysom_diagnosis/lib/channels}/__init__.py (100%) rename {sysom_api => sysom_server/sysom_diagnosis}/lib/channels/base.py (100%) rename {sysom_api => sysom_server/sysom_diagnosis}/lib/channels/ssh.py (93%) create mode 100644 sysom_server/sysom_diagnosis/lib/decode/sysom_decode.py create mode 100644 sysom_server/sysom_diagnosis/lib/exception.py create mode 100644 sysom_server/sysom_diagnosis/lib/paginations.py create mode 100644 sysom_server/sysom_diagnosis/lib/renderers.py create mode 100644 sysom_server/sysom_diagnosis/lib/response.py create mode 100644 sysom_server/sysom_diagnosis/lib/utils.py create mode 100755 sysom_server/sysom_diagnosis/manage.py rename {sysom_api => sysom_server/sysom_diagnosis}/service_scripts/filecache (100%) rename {sysom_api => sysom_server/sysom_diagnosis}/service_scripts/filecache_post (100%) rename {sysom_api => sysom_server/sysom_diagnosis}/service_scripts/iosdiag_latency (100%) rename {sysom_api => sysom_server/sysom_diagnosis}/service_scripts/iosdiag_latency_post (100%) rename {sysom_api => sysom_server/sysom_diagnosis}/service_scripts/loadtask (100%) rename {sysom_api => sysom_server/sysom_diagnosis}/service_scripts/loadtask_post (100%) rename {sysom_api => sysom_server/sysom_diagnosis}/service_scripts/memgraph (100%) rename {sysom_api => sysom_server/sysom_diagnosis}/service_scripts/memgraph_post (100%) rename {sysom_api => sysom_server/sysom_diagnosis}/service_scripts/node_delete (100%) rename {sysom_api => sysom_server/sysom_diagnosis}/service_scripts/node_init (100%) rename {sysom_api => sysom_server/sysom_diagnosis}/service_scripts/oomcheck (100%) rename {sysom_api => sysom_server/sysom_diagnosis}/service_scripts/oomcheck_post (100%) rename {sysom_api => sysom_server/sysom_diagnosis}/service_scripts/ossre (100%) rename {sysom_api => sysom_server/sysom_diagnosis}/service_scripts/ossre_post (100%) rename {sysom_api => sysom_server/sysom_diagnosis}/service_scripts/pingtrace (100%) rename {sysom_api => sysom_server/sysom_diagnosis}/service_scripts/pingtrace_post (100%) rename {sysom_api => sysom_server/sysom_diagnosis}/service_scripts/schedmoni (100%) rename {sysom_api => sysom_server/sysom_diagnosis}/service_scripts/schedmoni_post (100%) rename {sysom_api => sysom_server/sysom_diagnosis}/service_scripts/test (100%) rename {sysom_api => sysom_server/sysom_diagnosis}/service_scripts/test.sh (100%) rename {sysom_api => sysom_server/sysom_diagnosis}/service_scripts/tpython (100%) rename {sysom_api => sysom_server/sysom_diagnosis}/service_scripts/vmcore_analyse (100%) create mode 100644 sysom_server/sysom_diagnosis/sysom_diagnosis/__init__.py create mode 100644 sysom_server/sysom_diagnosis/sysom_diagnosis/asgi.py create mode 100644 sysom_server/sysom_diagnosis/sysom_diagnosis/settings.py create mode 100644 sysom_server/sysom_diagnosis/sysom_diagnosis/urls.py create mode 100644 sysom_server/sysom_diagnosis/sysom_diagnosis/wsgi.py rename tools/deploy/{task-service.ini => diagnosis-service.ini} (35%) delete mode 100644 tools/deploy/task-executor-service.ini diff --git a/package.sh b/package.sh index 23462147..3995ff2e 100644 --- a/package.sh +++ b/package.sh @@ -16,7 +16,7 @@ check_cmd yarn check_cmd tar RELEASE=sysomRelease-$(date +"%Y%m%d%H%M%S") -APIDIR=sysom_api +SERVERDIR=sysom_server WEBDIR=sysom_web SCRIPTDIR=script TOOLSDIR=tools @@ -27,7 +27,7 @@ yarn build popd || exit mkdir -p "${RELEASE}" -cp -r ${APIDIR}/ ${TOOLSDIR}/ "${RELEASE}"/ +cp -r ${SERVERDIR}/ ${TOOLSDIR}/ "${RELEASE}"/ cp -r ${WEBDIR}/dist/ "${RELEASE}"/${WEBDIR}/ mkdir -p "${RELEASE}"/${WEBDIR}/download/ cp ${TOOLSDIR}/deploy/deploy.sh "${RELEASE}"/ diff --git a/script/server/0_sysom_api/init.sh b/script/server/0_sysom_api/init.sh index 733b1967..3758274c 100755 --- a/script/server/0_sysom_api/init.sh +++ b/script/server/0_sysom_api/init.sh @@ -9,7 +9,11 @@ ALIYUN_MIRROR="https://mirrors.aliyun.com/pypi/simple/" APP_NAME="sysom" -API_DIR="sysom_api" +SERVER_DIR="sysom_server" +API_DIR=$SERVER_DIR/sysom_api +DIAGNOSIS_DIR=$SERVER_DIR/sysom_diagnosis +CHANNEL_DIR=$SERVER_DIR/sysom_channel +SDK_DIR=$SERVER_DIR/sdk WEB_DIR="sysom_web" VIRTUALENV_HOME="${SERVER_HOME}/virtualenv" @@ -107,6 +111,27 @@ init_conf() { python manage.py makemigrations channel python manage.py migrate popd + + pushd ${TARGET_PATH}/${DIAGNOSIS_DIR} + rm -f apps/*/migrations/00*.py + python manage.py makemigrations task + python manage.py migrate + popd + + + pushd ${TARGET_PATH}/${CHANNEL_DIR} + rm -f apps/*/migrations/00*.py + python manage.py makemigrations channel + python manage.py migrate + popd +} + +install_sdk() { + pushd ${TARGET_PATH}/${SDK_DIR} + python setup_cec_base.py install + python setup_cec_redis.py install + sudo rm -r *.egg-info build dist + popd } start_app() { @@ -122,6 +147,7 @@ deploy() { check_selinux_status touch_virtualenv check_requirements + install_sdk setup_database | tee -a ${SERVER_HOME}/logs/${APP_NAME}_setup_database.log 2>&1 init_conf start_app diff --git a/script/server/1_monitor/grafana_api_set.sh b/script/server/1_monitor/grafana_api_set.sh index 174b7cb8..52c1ede4 100755 --- a/script/server/1_monitor/grafana_api_set.sh +++ b/script/server/1_monitor/grafana_api_set.sh @@ -1,6 +1,6 @@ #!/bin/bash -x GRAFANA_CONFIG=/etc/grafana/grafana.ini -SYSOM_CONF=${SERVER_HOME}/target/sysom_api/conf/common.py +SYSOM_CONF=${SERVER_HOME}/target/sysom_server/sysom_api/conf/common.py ###grafana configure mysql### SYSOM_DATABASE_HOST=`cat $SYSOM_CONF | grep "'HOST'" | awk -F"'" '{print $4}'` diff --git a/script/server/1_monitor/monitor_server_deploy.sh b/script/server/1_monitor/monitor_server_deploy.sh index 089cbbfe..3ebe7983 100755 --- a/script/server/1_monitor/monitor_server_deploy.sh +++ b/script/server/1_monitor/monitor_server_deploy.sh @@ -16,8 +16,8 @@ PROMETHEUS_DL_URL=https://github.com/prometheus/prometheus/releases/download/v${ NODE_DL_URL=https://github.com/prometheus/node_exporter/releases/download/v${NODE_EXPORTER_VER} NODE_INIT_DIR=sysom_node_init NODE_INIT_PKG=sysom_node_init.tar.gz -NODE_INIT_SCRIPT=${SERVER_HOME}/target/sysom_api/service_scripts/node_init -NODE_DELETE_SCRIPT=${SERVER_HOME}/target/sysom_api/service_scripts/node_delete +NODE_INIT_SCRIPT=${SERVER_HOME}/target/sysom_server/sysom_api/service_scripts/node_init +NODE_DELETE_SCRIPT=${SERVER_HOME}/target/sysom_server/sysom_api/service_scripts/node_delete BASE_DIR=`dirname $0` diff --git a/sysom_api/apps/channel/apps.py b/sysom_api/apps/channel/apps.py deleted file mode 100644 index f343c59a..00000000 --- a/sysom_api/apps/channel/apps.py +++ /dev/null @@ -1,46 +0,0 @@ -import json -import logging -from django.apps import AppConfig -from django.db.models.signals import post_migrate - - -logger = logging.getLogger(__file__) - - -class ChannelConfig(AppConfig): - default_auto_field = 'django.db.models.BigAutoField' - name = 'apps.channel' - - def ready(self) -> None: - post_migrate.connect(initialization_settings, sender=self) - # write key to file - write_key_to_file() - logger.info(">>> Channel module loading success") - -def write_key_to_file(): - try: - from .models import SettingsModel - from django.conf import settings - - instance = SettingsModel.objects.get(key='ssh_key') - if instance: - with open(settings.KEY_PATH, 'w') as f: - f.write(instance.value) - except Exception as e: - logger.error(e) - pass - -def initialization_settings(sender, **kwargs): - init_model() - - -def init_model(): - try: - from .models import SettingsModel - from lib.utils import generate_key - - k, v = generate_key() - ssh_key = json.dumps({"private_key": k, "public_key": v}) - SettingsModel.objects.create(key='ssh_key', value=ssh_key, description='系统自动生成公私匙') - except Exception as e: - pass diff --git a/sysom_api/apps/channel/models.py b/sysom_api/apps/channel/models.py deleted file mode 100644 index bf91c659..00000000 --- a/sysom_api/apps/channel/models.py +++ /dev/null @@ -1,26 +0,0 @@ -from django.db import models -from lib.base_model import BaseModel - - -class SettingsModel(BaseModel): - key = models.CharField(max_length=128, unique=True) - value = models.TextField() - description = models.TextField(default='', verbose_name='描述') - - def __str__(self) -> str: - return self.key - - class Meta: - db_table = 'sys_settings' - - -class ExecuteResult(BaseModel): - invoke_id = models.CharField(max_length=64, verbose_name='任务ID') - result = models.JSONField(verbose_name='执行结果') - channel_name = models.CharField(max_length=32, verbose_name='执行通道', default='') - - def __str__(self) -> str: - return self.invoke_id - - class Meta: - db_table = 'sys_execute_result' diff --git a/sysom_api/apps/channel/urls.py b/sysom_api/apps/channel/urls.py deleted file mode 100644 index 76527da0..00000000 --- a/sysom_api/apps/channel/urls.py +++ /dev/null @@ -1,8 +0,0 @@ -from django.urls import path -from .views import ChannelAPIView - -urlpatterns = [ - path('api/v1/channel/', ChannelAPIView.as_view({'post': 'channel_post'})), - path('api/v1/channel/validate/', ChannelAPIView.as_view({'post': 'validate_host_channel'})), - path('api/v1/channel/exec_result/', ChannelAPIView.as_view({'post': 'get_exec_result'})), -] diff --git a/sysom_api/apps/channel/views.py b/sysom_api/apps/channel/views.py deleted file mode 100644 index 6ad55064..00000000 --- a/sysom_api/apps/channel/views.py +++ /dev/null @@ -1,139 +0,0 @@ -import logging -import re -import os -from django.conf import settings -from django.db import connection -from rest_framework.viewsets import GenericViewSet -from .models import ExecuteResult -from lib.response import other_response -from lib.channels.ssh import SSH -from lib.exception import APIException -from lib.utils import import_string - - -logger = logging.getLogger(__name__) - - -class ChannelAPIView(GenericViewSet): - authentication_classes = [] - result = dict() - - def get_exec_result(self, request): - try: - instance = ExecuteResult.objects.get(invoke_id=request.data.get('invoke_id')) - except ExecuteResult.DoesNotExist as e: - return other_response(code=400, message='invoke_id 不存在!') - return other_response(message='操作成功', result=instance.result) - - def valid_channel(self, request): - data = getattr(request, 'data') - channel_type = data.pop('channel', 'ssh') - package = import_string(f'lib.channels.{channel_type}') - try: - package.Channel(**data) - return import_string(f'lib.channels.{channel_type}') - except Exception as e: - logger.error(e) - - channels_path = os.path.join(settings.BASE_DIR, 'lib', 'channel', 'channels') - packages = [dir.replace('.py', '') for dir in os.listdir(channels_path) if not dir.startswith('__')] - packages.remove('base') - packages.remove('ssh') - - for i, pkg in enumerate(packages): - try: - package = import_string(f'lib.channels.{pkg}') - package.Channel(**request.data) - channel_type = pkg - break - except Exception as e: - logger.error(e) - if i+1 == len(packages): - raise APIException(message='No channels available!') - continue - return import_string(f'lib.channels.{channel_type}') - - def channel_post(self, request, *args, **kwargs): - channel_type = request.data.get('channel', 'ssh') - package = self.valid_channel(request) - - data = request.data - data['channel_name'] = channel_type - - channel = package.Channel(**data) - result = channel.run_command() - - kwargs.update(result) - kwargs.pop('state') - kwargs['channel_name'] = channel_type - self._save_execute_result(kwargs) - - return other_response(result=result, message='操作成功') - - def _save_execute_result(self, kwargs): - """ - 执行Command信息添加到数据库 - kwargs: - channel_name str 执行脚本通道文件名 默认 ssh - invoke_id str 执行脚本ID - result dict 执行脚本后的结果,例如:{'state': 0, 'result': 'xxxxxxxx'} 0为执行成功, 1位=为执行失败 - """ - if not kwargs.get('channel_name', None): - kwargs['channel_name'] = 'ssh' - try: - ExecuteResult.objects.create(**kwargs) - except Exception as e: - raise APIException(message=str(e)) - finally: - connection.close() - - def validate_ssh_channel_parame(self, data): - res, message = False, '' - if not data.get('instance') or not self._validate_ip_format(data['instance']): - message = 'instance: 检查instance字段' - else: - res = True - return res, message - - def permission_validate_host_ip(self): - """ - 检验当前请求是否有权限操作该host - """ - pass - - def validate_host_channel(self, request, *args, **kwarg): - """ - 检验host主机password, 校验成功后自动添加public-key到该机器 - 校验失败返回400, 及错误原因 - - ip (主机IP 必填) - - password (主机密码 必填) - - username (主机用户名 默认root) - - port (主机开放端口 默认22) - """ - requied_fields = ['ip', 'password'] - data = request.data - - for item in filter(lambda x: not x[1], [(field, data.get(field)) for field in requied_fields]): - return other_response(code=400, message=f'{item[0]}: required field', success=False) - - if not self._validate_ip_format(data['ip']): - return other_response(code=400, message='IP: 检查ip格式', success=False) - - kwargs = {} - kwargs['ip'] = data.get('ip') - kwargs['password'] = data.get('password') - if 'username' in data: - kwargs['username'] = data.get('username') - if 'port' in data: - kwargs['port'] = data.get('port') - - s, m = SSH.validate_ssh_host(**kwargs) - if s: - return other_response(message=m) - else: - return other_response(message=m, code=400, success=False) - - def _validate_ip_format(self, ip) -> bool: - p = '((\d{1,2})|([01]\d{2})|(2[0-4]\d)|(25[0-5]))' - pattern = '^' + '\.'.join([p]*4) + '$' - return bool(re.match(pattern, ip)) diff --git a/sysom_api/microservices/README.md b/sysom_api/microservices/README.md deleted file mode 100644 index e69de29b..00000000 diff --git a/sysom_api/microservices/task_executor/app.py b/sysom_api/microservices/task_executor/app.py deleted file mode 100644 index 6135bc1c..00000000 --- a/sysom_api/microservices/task_executor/app.py +++ /dev/null @@ -1,27 +0,0 @@ -# -*- coding: utf-8 -*- # -""" -Time 2022/10/11 16:13 -Author: mingfeng (SunnyQjm) -Email mfeng@linux.alibaba.com -File apps.py -Description: -""" -import signal -from sdk.cec_base.consumer import Consumer -from sdk.cec_base.log import LoggerHelper, LoggerLevel -from .executor import TaskExecutor -from .conf import * - -if __name__ == '__main__': - LoggerHelper.update_sys_stdout_sink(LoggerLevel.LOGGER_LEVEL_INFO) - task_executor = TaskExecutor( - SYSOM_CEC_URL, - SYSOM_CEC_TASK_DELIVERY_TOPIC, - Consumer.generate_consumer_id(), - SYSOM_CEC_TASK_DELIVERY_GROUP, - SYSOM_CEC_TASK_RESULT_PROCESS_TOPIC - ) - signal.signal(signal.SIGINT, task_executor.stop) - signal.signal(signal.SIGTERM, task_executor.stop) - task_executor.start() - task_executor.join() \ No newline at end of file diff --git a/sysom_api/microservices/task_executor/conf.py b/sysom_api/microservices/task_executor/conf.py deleted file mode 100644 index 8d8a03c6..00000000 --- a/sysom_api/microservices/task_executor/conf.py +++ /dev/null @@ -1 +0,0 @@ -from sysom.settings import * \ No newline at end of file diff --git a/sysom_api/sysom/services_conf/channel_settings.py b/sysom_api/sysom/services_conf/channel_settings.py deleted file mode 100644 index 359e6d7f..00000000 --- a/sysom_api/sysom/services_conf/channel_settings.py +++ /dev/null @@ -1,28 +0,0 @@ -from conf.common import * - -""" -Channels Model Micro Service Settings -""" - -INSTALLED_APPS = [ - 'apps.channel', - - # 第三方模块 - 'rest_framework' -] - -IS_MICRO_SERVICES = True - -REST_FRAMEWORK = { - 'DEFAULT_PERMISSION_CLASSES': (), - 'DEFAULT_AUTHENTICATION_CLASSES': [], - 'UNAUTHENTICATED_USER': None, - 'DEFAULT_VERSIONING_CLASS': "rest_framework.versioning.URLPathVersioning", - 'DEFAULT_VERSION': 'v1', # 默认版本 - 'ALLOWED_VERSIONS': ['v1', 'v2'], # 允许的版本 - 'VERSION_PARAM': 'version', - - 'DEFAULT_PAGINATION_CLASS': 'lib.paginations.Pagination', - 'UNICODE_JSON': False, - 'EXCEPTION_HANDLER': 'lib.exception.exception_handler' -} diff --git a/sysom_api/sysom/services_conf/task_settings.py b/sysom_api/sysom/services_conf/task_settings.py deleted file mode 100644 index f47c1883..00000000 --- a/sysom_api/sysom/services_conf/task_settings.py +++ /dev/null @@ -1,28 +0,0 @@ -from conf.common import * - -""" -Task Model Micro Service Settings -""" - -INSTALLED_APPS = [ - 'apps.task', - - # 第三方模块 - 'rest_framework' -] - -IS_MICRO_SERVICES = True - -REST_FRAMEWORK = { - 'DEFAULT_PERMISSION_CLASSES': (), - 'DEFAULT_AUTHENTICATION_CLASSES': [], - 'UNAUTHENTICATED_USER': None, - 'DEFAULT_VERSIONING_CLASS': "rest_framework.versioning.URLPathVersioning", - 'DEFAULT_VERSION': 'v1', # 默认版本 - 'ALLOWED_VERSIONS': ['v1', 'v2'], # 允许的版本 - 'VERSION_PARAM': 'version', - - 'DEFAULT_PAGINATION_CLASS': 'lib.paginations.Pagination', - 'UNICODE_JSON': False, - 'EXCEPTION_HANDLER': 'lib.exception.exception_handler' -} diff --git a/sysom_server/.gitignore b/sysom_server/.gitignore new file mode 100644 index 00000000..98c4b8d7 --- /dev/null +++ b/sysom_server/.gitignore @@ -0,0 +1,3 @@ +/conf/ +__pycache__ +logs \ No newline at end of file diff --git a/sysom_server/sdk/.gitignore b/sysom_server/sdk/.gitignore new file mode 100644 index 00000000..d6b60f16 --- /dev/null +++ b/sysom_server/sdk/.gitignore @@ -0,0 +1,3 @@ +/*.egg-info +/dist +/build \ No newline at end of file diff --git a/sysom_api/sdk/cec_base/__init__.py b/sysom_server/sdk/cec_base/__init__.py similarity index 100% rename from sysom_api/sdk/cec_base/__init__.py rename to sysom_server/sdk/cec_base/__init__.py diff --git a/sysom_api/sdk/cec_base/admin.py b/sysom_server/sdk/cec_base/admin.py similarity index 99% rename from sysom_api/sdk/cec_base/admin.py rename to sysom_server/sdk/cec_base/admin.py index 6a479908..ce5449bb 100644 --- a/sysom_api/sdk/cec_base/admin.py +++ b/sysom_server/sdk/cec_base/admin.py @@ -451,7 +451,7 @@ def dispatch_admin(url: str, **kwargs) -> Admin: cec_url = CecUrl.parse(url) if cec_url.proto not in Admin.proto_dict: # Check if dynamic import is possible - target_module = f"sdk.cec_{cec_url.proto}.{cec_url.proto}_admin" + target_module = f"cec_{cec_url.proto}.{cec_url.proto}_admin" try: module = importlib.import_module(target_module) Admin.register( diff --git a/sysom_api/sdk/cec_base/base.py b/sysom_server/sdk/cec_base/base.py similarity index 100% rename from sysom_api/sdk/cec_base/base.py rename to sysom_server/sdk/cec_base/base.py diff --git a/sysom_api/sdk/cec_base/cec_client.py b/sysom_server/sdk/cec_base/cec_client.py similarity index 91% rename from sysom_api/sdk/cec_base/cec_client.py rename to sysom_server/sdk/cec_base/cec_client.py index a04f9df9..cb4416bc 100644 --- a/sysom_api/sdk/cec_base/cec_client.py +++ b/sysom_server/sdk/cec_base/cec_client.py @@ -9,10 +9,10 @@ Description: from threading import Thread from typing import Any, List from queue import Queue -from sdk.cec_base.consumer import Consumer, dispatch_consumer -from sdk.cec_base.admin import Admin, dispatch_admin -from sdk.cec_base.event import Event -from sdk.cec_base.producer import Producer, dispatch_producer +from cec_base.consumer import Consumer, dispatch_consumer +from cec_base.admin import Admin, dispatch_admin +from cec_base.event import Event +from cec_base.producer import Producer, dispatch_producer class CecClient: @@ -70,9 +70,9 @@ class CecClient: "consumer": consumer }) elif self._custom_callback is not None: - self._custom_callback(consumer, event, task) + self._custom_callback(consumer, self.producer, event, task) else: - self.on_receive_event(consumer, event, task) + self.on_receive_event(consumer, self.producer, event, task) def get_next_event(self) -> dict: """Get next event""" @@ -80,7 +80,8 @@ class CecClient: raise Exception("Not in sync mode") return self._inner_queue.get() - def on_receive_event(self, consumer: Consumer, event: Event, task: dict): + def on_receive_event(self, consumer: Consumer, producer: Producer, event: Event, + task: dict): """Invoke while receive event from CEC Args: diff --git a/sysom_api/sdk/cec_base/cli.py b/sysom_server/sdk/cec_base/cli.py similarity index 100% rename from sysom_api/sdk/cec_base/cli.py rename to sysom_server/sdk/cec_base/cli.py diff --git a/sysom_api/sdk/cec_base/consumer.py b/sysom_server/sdk/cec_base/consumer.py similarity index 98% rename from sysom_api/sdk/cec_base/consumer.py rename to sysom_server/sdk/cec_base/consumer.py index f7013e87..4dfcaca3 100644 --- a/sysom_api/sdk/cec_base/consumer.py +++ b/sysom_server/sdk/cec_base/consumer.py @@ -130,7 +130,6 @@ class Consumer(Connectable, metaclass=ABCMeta): ... , start_from_now=False) >>> consumer.consume(200, auto_ack=False, batch_consume_limit=20) """ - raise NotImplemented @abstractmethod def ack(self, event: Event, **kwargs) -> int: @@ -160,7 +159,6 @@ class Consumer(Connectable, metaclass=ABCMeta): >>> msg = msgs[0] >>> consumer.ack(msg) """ - raise NotImplemented def __iter__(self): return self @@ -298,7 +296,7 @@ def dispatch_consumer(url: str, topic_name: str, consumer_id: str = "", cec_url = CecUrl.parse(url) if cec_url.proto not in Consumer.proto_dict: # Check if dynamic import is possible - target_module = f"sdk.cec_{cec_url.proto}.{cec_url.proto}_consumer" + target_module = f"cec_{cec_url.proto}.{cec_url.proto}_consumer" try: module = importlib.import_module(target_module) Consumer.register( diff --git a/sysom_api/sdk/cec_base/event.py b/sysom_server/sdk/cec_base/event.py similarity index 100% rename from sysom_api/sdk/cec_base/event.py rename to sysom_server/sdk/cec_base/event.py diff --git a/sysom_api/sdk/cec_base/exceptions.py b/sysom_server/sdk/cec_base/exceptions.py similarity index 100% rename from sysom_api/sdk/cec_base/exceptions.py rename to sysom_server/sdk/cec_base/exceptions.py diff --git a/sysom_api/sdk/cec_base/log.py b/sysom_server/sdk/cec_base/log.py similarity index 100% rename from sysom_api/sdk/cec_base/log.py rename to sysom_server/sdk/cec_base/log.py diff --git a/sysom_api/sdk/cec_base/meta.py b/sysom_server/sdk/cec_base/meta.py similarity index 100% rename from sysom_api/sdk/cec_base/meta.py rename to sysom_server/sdk/cec_base/meta.py diff --git a/sysom_api/sdk/cec_base/producer.py b/sysom_server/sdk/cec_base/producer.py similarity index 98% rename from sysom_api/sdk/cec_base/producer.py rename to sysom_server/sdk/cec_base/producer.py index 0a956faf..93965374 100644 --- a/sysom_api/sdk/cec_base/producer.py +++ b/sysom_server/sdk/cec_base/producer.py @@ -126,7 +126,7 @@ def dispatch_producer(url: str, **kwargs) -> Producer: cec_url = CecUrl.parse(url) if cec_url.proto not in Producer.proto_dict: # Check if dynamic import is possible - target_module = f"sdk.cec_{cec_url.proto}.{cec_url.proto}_producer" + target_module = f"cec_{cec_url.proto}.{cec_url.proto}_producer" try: module = importlib.import_module(target_module) Producer.register( diff --git a/sysom_api/sdk/cec_base/url.py b/sysom_server/sdk/cec_base/url.py similarity index 100% rename from sysom_api/sdk/cec_base/url.py rename to sysom_server/sdk/cec_base/url.py diff --git a/sysom_api/sdk/cec_redis/__init__.py b/sysom_server/sdk/cec_redis/__init__.py similarity index 100% rename from sysom_api/sdk/cec_redis/__init__.py rename to sysom_server/sdk/cec_redis/__init__.py diff --git a/sysom_api/sdk/cec_redis/admin_static.py b/sysom_server/sdk/cec_redis/admin_static.py similarity index 99% rename from sysom_api/sdk/cec_redis/admin_static.py rename to sysom_server/sdk/cec_redis/admin_static.py index eb2a9ef2..de5386d3 100644 --- a/sysom_api/sdk/cec_redis/admin_static.py +++ b/sysom_server/sdk/cec_redis/admin_static.py @@ -12,13 +12,13 @@ from itertools import chain from redis import Redis from loguru import logger import redis.exceptions -from ..cec_base.exceptions import TopicNotExistsException, \ +from cec_base.exceptions import TopicNotExistsException, \ TopicAlreadyExistsException, ConsumerGroupNotExistsException, \ ConsumerGroupAlreadyExistsException -from ..cec_base.meta import TopicMeta, PartitionMeta, ConsumerGroupMeta, \ +from cec_base.meta import TopicMeta, PartitionMeta, ConsumerGroupMeta, \ ConsumerGroupMemberMeta -from ..cec_base.exceptions import CecException -from ..cec_base.log import LoggerHelper +from cec_base.exceptions import CecException +from cec_base.log import LoggerHelper from .utils import raise_if_not_ignore from .consume_status_storage import ConsumeStatusStorage from .common import StaticConst diff --git a/sysom_api/sdk/cec_redis/common.py b/sysom_server/sdk/cec_redis/common.py similarity index 99% rename from sysom_api/sdk/cec_redis/common.py rename to sysom_server/sdk/cec_redis/common.py index 02ea037c..333fb7a3 100644 --- a/sysom_api/sdk/cec_redis/common.py +++ b/sysom_server/sdk/cec_redis/common.py @@ -8,7 +8,7 @@ Description: """ from redis import Redis -from ..cec_base.url import CecUrl +from cec_base.url import CecUrl class StaticConst: diff --git a/sysom_api/sdk/cec_redis/consume_status_storage.py b/sysom_server/sdk/cec_redis/consume_status_storage.py similarity index 99% rename from sysom_api/sdk/cec_redis/consume_status_storage.py rename to sysom_server/sdk/cec_redis/consume_status_storage.py index aeaf0f01..b60ef5ed 100644 --- a/sysom_api/sdk/cec_redis/consume_status_storage.py +++ b/sysom_server/sdk/cec_redis/consume_status_storage.py @@ -10,7 +10,7 @@ Description: from typing import Union from redis import Redis from redis.client import Pipeline -from ..cec_base.event import Event +from cec_base.event import Event from .common import StaticConst diff --git a/sysom_api/sdk/cec_redis/heartbeat.py b/sysom_server/sdk/cec_redis/heartbeat.py similarity index 99% rename from sysom_api/sdk/cec_redis/heartbeat.py rename to sysom_server/sdk/cec_redis/heartbeat.py index eb85f957..ec0213f8 100644 --- a/sysom_api/sdk/cec_redis/heartbeat.py +++ b/sysom_server/sdk/cec_redis/heartbeat.py @@ -16,8 +16,8 @@ from redis import Redis from redis.client import PubSub from atomic import AtomicLong from schedule import Scheduler -from ..cec_base.exceptions import CecException -from ..cec_base.log import LoggerHelper +from cec_base.exceptions import CecException +from cec_base.log import LoggerHelper from .common import StaticConst from .admin_static import static_del_consumer diff --git a/sysom_api/sdk/cec_redis/redis_admin.py b/sysom_server/sdk/cec_redis/redis_admin.py similarity index 98% rename from sysom_api/sdk/cec_redis/redis_admin.py rename to sysom_server/sdk/cec_redis/redis_admin.py index 4abb817f..3ea15c4f 100644 --- a/sysom_api/sdk/cec_redis/redis_admin.py +++ b/sysom_server/sdk/cec_redis/redis_admin.py @@ -11,12 +11,12 @@ import json from typing import Optional, List import redis.exceptions -from ..cec_base.admin import Admin, ConsumeStatusItem -from ..cec_base.exceptions import CecException -from ..cec_base.event import Event -from ..cec_base.meta import ConsumerGroupMeta, TopicMeta -from ..cec_base.log import LoggerHelper -from ..cec_base.url import CecUrl +from cec_base.admin import Admin, ConsumeStatusItem +from cec_base.exceptions import CecException +from cec_base.event import Event +from cec_base.meta import ConsumerGroupMeta, TopicMeta +from cec_base.log import LoggerHelper +from cec_base.url import CecUrl from redis import Redis from redis.exceptions import ResponseError from loguru import logger diff --git a/sysom_api/sdk/cec_redis/redis_consumer.py b/sysom_server/sdk/cec_redis/redis_consumer.py similarity index 98% rename from sysom_api/sdk/cec_redis/redis_consumer.py rename to sysom_server/sdk/cec_redis/redis_consumer.py index 34a03532..78b976f0 100644 --- a/sysom_api/sdk/cec_redis/redis_consumer.py +++ b/sysom_server/sdk/cec_redis/redis_consumer.py @@ -13,10 +13,10 @@ from typing import List, Optional import redis.exceptions from redis import Redis from loguru import logger -from ..cec_base.consumer import Consumer, ConsumeMode -from ..cec_base.event import Event -from ..cec_base.url import CecUrl -from ..cec_base.log import LoggerHelper +from cec_base.consumer import Consumer, ConsumeMode +from cec_base.event import Event +from cec_base.url import CecUrl +from cec_base.log import LoggerHelper from .utils import do_connect_by_cec_url from .redis_admin import RedisAdmin from .consume_status_storage import ConsumeStatusStorage @@ -268,7 +268,7 @@ class RedisConsumer(Consumer, ClientBase): return self._redis_client.xread({ self.inner_topic_name: '$' if self.consume_mode == - ConsumeMode.CONSUME_FROM_NOW else '0-0' + ConsumeMode.CONSUME_FROM_NOW else '0-0' }, count=batch_consume_limit, block=timeout) # Take out the messages in sequential order return self._redis_client.xread({ diff --git a/sysom_api/sdk/cec_redis/redis_producer.py b/sysom_server/sdk/cec_redis/redis_producer.py similarity index 97% rename from sysom_api/sdk/cec_redis/redis_producer.py rename to sysom_server/sdk/cec_redis/redis_producer.py index df261708..f1b3a643 100644 --- a/sysom_api/sdk/cec_redis/redis_producer.py +++ b/sysom_server/sdk/cec_redis/redis_producer.py @@ -11,11 +11,11 @@ from typing import Callable, Union, Optional from redis import Redis from loguru import logger -from ..cec_base.producer import Producer -from ..cec_base.event import Event -from ..cec_base.url import CecUrl -from ..cec_base.exceptions import TopicNotExistsException -from ..cec_base.log import LoggerHelper +from cec_base.producer import Producer +from cec_base.event import Event +from cec_base.url import CecUrl +from cec_base.exceptions import TopicNotExistsException +from cec_base.log import LoggerHelper from .utils import do_connect_by_cec_url from .redis_admin import RedisAdmin from .common import StaticConst, ClientBase diff --git a/sysom_api/sdk/cec_redis/utils.py b/sysom_server/sdk/cec_redis/utils.py similarity index 96% rename from sysom_api/sdk/cec_redis/utils.py rename to sysom_server/sdk/cec_redis/utils.py index b9c70bc4..c16b0e9a 100644 --- a/sysom_api/sdk/cec_redis/utils.py +++ b/sysom_server/sdk/cec_redis/utils.py @@ -8,9 +8,9 @@ Description: """ from redis import Redis import redis -from ..cec_base.url import CecUrl -from ..cec_base.exceptions import CecConnectionException -from ..cec_base.log import LoggerHelper +from cec_base.url import CecUrl +from cec_base.exceptions import CecConnectionException +from cec_base.log import LoggerHelper def do_connect(url: str) -> Redis: diff --git a/sysom_server/sdk/setup_cec_base.py b/sysom_server/sdk/setup_cec_base.py new file mode 100644 index 00000000..432386fb --- /dev/null +++ b/sysom_server/sdk/setup_cec_base.py @@ -0,0 +1,22 @@ +# -*- coding: utf-8 -*- # +""" +Author: mingfeng (SunnyQjm) +Created: 2022/07/24 +Description: +""" +import setuptools + +setuptools.setup( + name="cec_base", + version="0.0.1", + author="mingfeng(SunnyQjm)", + author_email="mfeng@linux.alibaba.com", + description="A common event center interface definition package", + url="", + packages=setuptools.find_packages(), + classifiers=[ + "Programming Language :: Python :: 3", + "License :: OSI Approved :: GPL License", + "Operating System :: OS Independent", + ] +) diff --git a/sysom_server/sdk/setup_cec_redis.py b/sysom_server/sdk/setup_cec_redis.py new file mode 100644 index 00000000..0f83f1b6 --- /dev/null +++ b/sysom_server/sdk/setup_cec_redis.py @@ -0,0 +1,17 @@ +import setuptools + +setuptools.setup( + name="cec_redis", + version="0.0.1", + author="mingfeng(SunnyQjm)", + author_email="mfeng@linux.alibaba.com", + description="A redis implement for common event center", + url="", + packages=setuptools.find_packages(), + install_requires=[], + classifiers=[ + "Programming Language :: Python :: 3", + "License :: OSI Approved :: GPL License", + "Operating System :: OS Independent", + ] +) diff --git a/sysom_api/.gitignore b/sysom_server/sysom_api/.gitignore similarity index 100% rename from sysom_api/.gitignore rename to sysom_server/sysom_api/.gitignore diff --git a/sysom_api/apps/__init__.py b/sysom_server/sysom_api/apps/__init__.py similarity index 100% rename from sysom_api/apps/__init__.py rename to sysom_server/sysom_api/apps/__init__.py diff --git a/sysom_api/apps/accounts/__init__.py b/sysom_server/sysom_api/apps/accounts/__init__.py similarity index 100% rename from sysom_api/apps/accounts/__init__.py rename to sysom_server/sysom_api/apps/accounts/__init__.py diff --git a/sysom_api/apps/accounts/admin.py b/sysom_server/sysom_api/apps/accounts/admin.py similarity index 100% rename from sysom_api/apps/accounts/admin.py rename to sysom_server/sysom_api/apps/accounts/admin.py diff --git a/sysom_api/apps/accounts/apps.py b/sysom_server/sysom_api/apps/accounts/apps.py similarity index 100% rename from sysom_api/apps/accounts/apps.py rename to sysom_server/sysom_api/apps/accounts/apps.py diff --git a/sysom_api/apps/accounts/authentication.py b/sysom_server/sysom_api/apps/accounts/authentication.py similarity index 100% rename from sysom_api/apps/accounts/authentication.py rename to sysom_server/sysom_api/apps/accounts/authentication.py diff --git a/sysom_api/apps/accounts/models.py b/sysom_server/sysom_api/apps/accounts/models.py similarity index 100% rename from sysom_api/apps/accounts/models.py rename to sysom_server/sysom_api/apps/accounts/models.py diff --git a/sysom_api/apps/accounts/permissions.py b/sysom_server/sysom_api/apps/accounts/permissions.py similarity index 100% rename from sysom_api/apps/accounts/permissions.py rename to sysom_server/sysom_api/apps/accounts/permissions.py diff --git a/sysom_api/apps/accounts/role.json b/sysom_server/sysom_api/apps/accounts/role.json similarity index 100% rename from sysom_api/apps/accounts/role.json rename to sysom_server/sysom_api/apps/accounts/role.json diff --git a/sysom_api/apps/accounts/serializer.py b/sysom_server/sysom_api/apps/accounts/serializer.py similarity index 100% rename from sysom_api/apps/accounts/serializer.py rename to sysom_server/sysom_api/apps/accounts/serializer.py diff --git a/sysom_api/apps/accounts/tests.py b/sysom_server/sysom_api/apps/accounts/tests.py similarity index 100% rename from sysom_api/apps/accounts/tests.py rename to sysom_server/sysom_api/apps/accounts/tests.py diff --git a/sysom_api/apps/accounts/urls.py b/sysom_server/sysom_api/apps/accounts/urls.py similarity index 100% rename from sysom_api/apps/accounts/urls.py rename to sysom_server/sysom_api/apps/accounts/urls.py diff --git a/sysom_api/apps/accounts/user.json b/sysom_server/sysom_api/apps/accounts/user.json similarity index 100% rename from sysom_api/apps/accounts/user.json rename to sysom_server/sysom_api/apps/accounts/user.json diff --git a/sysom_api/apps/accounts/views.py b/sysom_server/sysom_api/apps/accounts/views.py similarity index 100% rename from sysom_api/apps/accounts/views.py rename to sysom_server/sysom_api/apps/accounts/views.py diff --git a/sysom_api/apps/alarm/__init__.py b/sysom_server/sysom_api/apps/alarm/__init__.py similarity index 100% rename from sysom_api/apps/alarm/__init__.py rename to sysom_server/sysom_api/apps/alarm/__init__.py diff --git a/sysom_api/apps/alarm/admin.py b/sysom_server/sysom_api/apps/alarm/admin.py similarity index 100% rename from sysom_api/apps/alarm/admin.py rename to sysom_server/sysom_api/apps/alarm/admin.py diff --git a/sysom_api/apps/alarm/apps.py b/sysom_server/sysom_api/apps/alarm/apps.py similarity index 100% rename from sysom_api/apps/alarm/apps.py rename to sysom_server/sysom_api/apps/alarm/apps.py diff --git a/sysom_api/apps/alarm/models.py b/sysom_server/sysom_api/apps/alarm/models.py similarity index 100% rename from sysom_api/apps/alarm/models.py rename to sysom_server/sysom_api/apps/alarm/models.py diff --git a/sysom_api/apps/alarm/serializer.py b/sysom_server/sysom_api/apps/alarm/serializer.py similarity index 100% rename from sysom_api/apps/alarm/serializer.py rename to sysom_server/sysom_api/apps/alarm/serializer.py diff --git a/sysom_api/apps/alarm/subscribe.json b/sysom_server/sysom_api/apps/alarm/subscribe.json similarity index 100% rename from sysom_api/apps/alarm/subscribe.json rename to sysom_server/sysom_api/apps/alarm/subscribe.json diff --git a/sysom_api/apps/alarm/urls.py b/sysom_server/sysom_api/apps/alarm/urls.py similarity index 100% rename from sysom_api/apps/alarm/urls.py rename to sysom_server/sysom_api/apps/alarm/urls.py diff --git a/sysom_api/apps/alarm/views.py b/sysom_server/sysom_api/apps/alarm/views.py similarity index 97% rename from sysom_api/apps/alarm/views.py rename to sysom_server/sysom_api/apps/alarm/views.py index 120a4872..a0612a63 100644 --- a/sysom_api/apps/alarm/views.py +++ b/sysom_server/sysom_api/apps/alarm/views.py @@ -9,8 +9,8 @@ from rest_framework.filters import SearchFilter, OrderingFilter from apps.alarm.models import AlarmModel, SubscribeModel from lib.response import success, other_response, not_found from django.conf import settings -from sdk.cec_base.producer import Producer, dispatch_producer -from sdk.cec_base.admin import Admin, dispatch_admin +from cec_base.producer import Producer, dispatch_producer +from cec_base.admin import Admin, dispatch_admin logger = logging.getLogger(__name__) diff --git a/sysom_api/apps/channel/__init__.py b/sysom_server/sysom_api/apps/common/__init__.py similarity index 100% rename from sysom_api/apps/channel/__init__.py rename to sysom_server/sysom_api/apps/common/__init__.py diff --git a/sysom_api/apps/common/common_model_viewset.py b/sysom_server/sysom_api/apps/common/common_model_viewset.py similarity index 98% rename from sysom_api/apps/common/common_model_viewset.py rename to sysom_server/sysom_api/apps/common/common_model_viewset.py index 00a16e87..d2303192 100644 --- a/sysom_api/apps/common/common_model_viewset.py +++ b/sysom_server/sysom_api/apps/common/common_model_viewset.py @@ -1,7 +1,7 @@ from typing import Any from rest_framework.viewsets import GenericViewSet from rest_framework.exceptions import ValidationError -from sdk.cec_base.cec_client import CecClient +from cec_base.cec_client import CecClient from django.conf import settings diff --git a/sysom_api/apps/common/__init__.py b/sysom_server/sysom_api/apps/host/__init__.py similarity index 100% rename from sysom_api/apps/common/__init__.py rename to sysom_server/sysom_api/apps/host/__init__.py diff --git a/sysom_api/apps/channel/admin.py b/sysom_server/sysom_api/apps/host/admin.py similarity index 100% rename from sysom_api/apps/channel/admin.py rename to sysom_server/sysom_api/apps/host/admin.py diff --git a/sysom_api/apps/host/apps.py b/sysom_server/sysom_api/apps/host/apps.py similarity index 31% rename from sysom_api/apps/host/apps.py rename to sysom_server/sysom_api/apps/host/apps.py index 558d25cd..320937c2 100644 --- a/sysom_api/apps/host/apps.py +++ b/sysom_server/sysom_api/apps/host/apps.py @@ -1,39 +1,80 @@ import logging import sys from django.apps import AppConfig -from sdk.cec_base.cec_client import CecClient -from sdk.cec_base.consumer import Consumer -from sdk.cec_base.event import Event +from cec_base.cec_client import CecClient +from cec_base.consumer import Consumer +from cec_base.producer import Producer +from cec_base.event import Event logger = logging.getLogger(__name__) -def _on_receive_task_result(consumer: Consumer, event: Event, task: dict): +def _on_receive_task_result(consumer: Consumer, producer: Producer, event: Event, task: dict): """Host module deal task result here { - "status":0, - "err_msg":"", - "task": {} + "code": 0, + "err_msg": "", + "result": "xxx", + "echo": { + "instance": "127.0.0.1" + "label": "host_init" + } } """ from .models import HostModel + from apps.alarm.views import _create_alarm_message + from datetime import datetime + from django.conf import settings + try: result = event.value - status = result.get("status", 1) - task = result.get("task", {}) - params = task.get("params", {}) - service_name = params.get("service_name", "Unknown") - if service_name != "node_init": - return - instance_ip = params.get("instance", "Unknown") - instance = HostModel.objects.get(ip=instance_ip) - instance.status = status - instance.save() + code = result.get("code", 1) + err_msg = result.get("err_msg", "") + echo = result.get("echo", {}) + label = echo.pop("label", "") + if label == "host_init": + instance_ip = echo.pop("instance", "") + instance = HostModel.objects.get(ip=instance_ip) + instance.status = 0 if code == 0 else 1 + instance.save() + alarm_msg = { + "sub": 1, + "item": "host" + } + if code == 0: + alarm_msg['message'] = f"Host: {instance_ip} init success!" + alarm_msg['collected_time'] = datetime.now().strftime( + '%Y-%m-%d %H:%M:%S') + alarm_msg['level'] = 3 + _create_alarm_message(alarm_msg) + + # 主机添加成功通知插件初始化 + producer.produce( + settings.SYSOM_CEC_PLUGIN_TOPIC, + { + "type": "init", + "params": { + "channel": "ssh", + "instance": instance.ip, + "username": instance.username, + "port": instance.port, + "token": echo.pop("token", "") + } + } + ) + else: + alarm_msg['message'] = f"Host: {instance_ip} init failed: {err_msg}" + alarm_msg['collected_time'] = datetime.now().strftime( + '%Y-%m-%d %H:%M:%S') + alarm_msg['level'] = 2 + _create_alarm_message(alarm_msg) except Exception as exc: logger.exception(exc) + err_msg += str(exc) finally: consumer.ack(event) + class HostConfig(AppConfig): @@ -43,13 +84,15 @@ class HostConfig(AppConfig): def ready(self): if 'runserver' in sys.argv or 'manage.py' not in sys.argv: from django.conf import settings + from cec_base.log import LoggerHelper, LoggerLevel # 这边微服务正式启动的时候执行一些处理代码 # 启动任务结果处理线程 + LoggerHelper.update_sys_stdout_sink(LoggerLevel.LOGGER_LEVEL_INFO) try: cec_client = CecClient(settings.SYSOM_CEC_URL, custom_callback=_on_receive_task_result) cec_client.append_group_consume_task( - settings.SYSOM_CEC_TASK_EXECUTE_RESULT_TOPIC, - settings.SYSOM_CEC_TASK_RESULT_LISTENER_HOST_GROUP + settings.SYSOM_CEC_CHANNEL_RESULT_TOPIC, + settings.SYSOM_CEC_API_HOST_CONSUMER_GROUP ) cec_client.start() except Exception as e: diff --git a/sysom_api/apps/host/models.py b/sysom_server/sysom_api/apps/host/models.py similarity index 100% rename from sysom_api/apps/host/models.py rename to sysom_server/sysom_api/apps/host/models.py diff --git a/sysom_api/apps/host/serializer.py b/sysom_server/sysom_api/apps/host/serializer.py similarity index 100% rename from sysom_api/apps/host/serializer.py rename to sysom_server/sysom_api/apps/host/serializer.py diff --git a/sysom_api/apps/channel/tests.py b/sysom_server/sysom_api/apps/host/tests.py similarity index 100% rename from sysom_api/apps/channel/tests.py rename to sysom_server/sysom_api/apps/host/tests.py diff --git a/sysom_api/apps/host/urls.py b/sysom_server/sysom_api/apps/host/urls.py similarity index 100% rename from sysom_api/apps/host/urls.py rename to sysom_server/sysom_api/apps/host/urls.py diff --git a/sysom_api/apps/host/views.py b/sysom_server/sysom_api/apps/host/views.py similarity index 90% rename from sysom_api/apps/host/views.py rename to sysom_server/sysom_api/apps/host/views.py index 16a9aa62..43b24e13 100644 --- a/sysom_api/apps/host/views.py +++ b/sysom_server/sysom_api/apps/host/views.py @@ -21,10 +21,8 @@ from lib.response import * from lib.utils import human_datetime, datetime from lib.exception import APIException from lib.excel import Excel -from lib.utils import HTTP from concurrent.futures import ThreadPoolExecutor, as_completed from apps.alarm.views import _create_alarm_message -from lib.channels.ssh import SSH logger = logging.getLogger(__name__) @@ -112,27 +110,21 @@ class HostModelViewSet(CommonModelViewSet, return ErrorResponse(msg=res["message"]) def _validate_and_initialize_host(self, context): - s, e = SSH.validate_ssh_host( - ip=context['ip'], - password=context['host_password'], - username=context['username'], - port=int(context['port']) - ) - if not s: - raise APIException(message=e) create_serializer = self.get_serializer(data=context) - try: create_serializer.is_valid(raise_exception=True) except ValidationError as e: raise APIException( message=f"{self.get_format_err_msg_for_validation_error(context, e)}。主机添加失败") self.perform_create(create_serializer) + instance = create_serializer.instance - # 检查输入client部署命令 更新host状态 - thread = threading.Thread( - target=self.client_deploy_cmd_init, args=(instance,)) - thread.start() + self.client_deploy_cmd_init(instance, { + "instance": context['ip'], + "password": context['host_password'], + "username": context['username'], + "port": int(context['port']) + }) ser = serializer.HostSerializer(instance=instance) return ser @@ -146,32 +138,61 @@ class HostModelViewSet(CommonModelViewSet, instance = self.get_queryset().filter(**kwargs).first() return instance if instance else None - def client_deploy_cmd_init(self, instance): + def client_deploy_cmd_init(self, instance, params: dict): request: Request = getattr(self, 'request') - data = {} - data['service_name'] = 'node_init' - data['instance'] = instance.ip - data['update_host_status'] = True - - kwargs = {} - kwargs['data'] = data - kwargs['token'] = request.META.get('HTTP_AUTHORIZATION') self.produce_event_to_cec( - settings.SYSOM_CEC_TASK_GENERATE_TOPIC, kwargs) + settings.SYSOM_CEC_CHANNEL_TOPIC, + { + "channel": "ssh", + "type": "init", + "params": { + "instance": instance.ip, + "username": instance.username, + "port": instance.port, + **params + }, + "echo": { + "instance": instance.ip, + "token": request.META.get('HTTP_AUTHORIZATION'), + "label": "host_init" + } + } + ) logger.info(f'node init task create success') - def client_deploy_cmd_delete(self, instance): + def client_deploy_cmd_delete(self, instance: HostModel): request: Request = getattr(self, 'request') - data = {} - data['service_name'] = "node_delete" - data['instance'] = instance.ip - - kwargs = {} - kwargs['data'] = data - kwargs['token'] = request.META.get('HTTP_AUTHORIZATION') + print("begin client_deploy_cmd_delete") + # 通知所有插件执行清理操作 self.produce_event_to_cec( - settings.SYSOM_CEC_TASK_GENERATE_TOPIC, kwargs) - logger.info(f'node delete task create success') + settings.SYSOM_CEC_PLUGIN_TOPIC, + { + "type": "clean", + "params": { + "channel": "ssh", + "instance": instance.ip, + "username": instance.username, + "port": instance.port, + "token": request.META.get('HTTP_AUTHORIZATION') + }, + "echo": { + "instance": instance.ip, + "label": "host_init" + } + } + ) + print("end client_deploy_cmd_delete") + # request: Request = getattr(self, 'request') + # data = {} + # data['service_name'] = "node_delete" + # data['instance'] = instance.ip + + # kwargs = {} + # kwargs['data'] = data + # kwargs['token'] = request.META.get('HTTP_AUTHORIZATION') + # self.produce_event_to_cec( + # settings.SYSOM_CEC_TASK_GENERATE_TOPIC, kwargs) + # logger.info(f'node delete task create success') def _get_cluster_instance(self, cluster_name): try: diff --git a/sysom_api/apps/host/__init__.py b/sysom_server/sysom_api/apps/monitor/__init__.py similarity index 100% rename from sysom_api/apps/host/__init__.py rename to sysom_server/sysom_api/apps/monitor/__init__.py diff --git a/sysom_api/apps/host/admin.py b/sysom_server/sysom_api/apps/monitor/admin.py similarity index 100% rename from sysom_api/apps/host/admin.py rename to sysom_server/sysom_api/apps/monitor/admin.py diff --git a/sysom_api/apps/monitor/apps.py b/sysom_server/sysom_api/apps/monitor/apps.py similarity index 100% rename from sysom_api/apps/monitor/apps.py rename to sysom_server/sysom_api/apps/monitor/apps.py diff --git a/sysom_api/apps/monitor/models.py b/sysom_server/sysom_api/apps/monitor/models.py similarity index 100% rename from sysom_api/apps/monitor/models.py rename to sysom_server/sysom_api/apps/monitor/models.py diff --git a/sysom_api/apps/host/tests.py b/sysom_server/sysom_api/apps/monitor/tests.py similarity index 100% rename from sysom_api/apps/host/tests.py rename to sysom_server/sysom_api/apps/monitor/tests.py diff --git a/sysom_api/apps/monitor/urls.py b/sysom_server/sysom_api/apps/monitor/urls.py similarity index 100% rename from sysom_api/apps/monitor/urls.py rename to sysom_server/sysom_api/apps/monitor/urls.py diff --git a/sysom_api/apps/monitor/views.py b/sysom_server/sysom_api/apps/monitor/views.py similarity index 100% rename from sysom_api/apps/monitor/views.py rename to sysom_server/sysom_api/apps/monitor/views.py diff --git a/sysom_api/apps/monitor/__init__.py b/sysom_server/sysom_api/apps/vmcore/__init__.py similarity index 100% rename from sysom_api/apps/monitor/__init__.py rename to sysom_server/sysom_api/apps/vmcore/__init__.py diff --git a/sysom_api/apps/monitor/admin.py b/sysom_server/sysom_api/apps/vmcore/admin.py similarity index 100% rename from sysom_api/apps/monitor/admin.py rename to sysom_server/sysom_api/apps/vmcore/admin.py diff --git a/sysom_api/apps/vmcore/apps.py b/sysom_server/sysom_api/apps/vmcore/apps.py similarity index 100% rename from sysom_api/apps/vmcore/apps.py rename to sysom_server/sysom_api/apps/vmcore/apps.py diff --git a/sysom_api/apps/vmcore/models.py b/sysom_server/sysom_api/apps/vmcore/models.py similarity index 100% rename from sysom_api/apps/vmcore/models.py rename to sysom_server/sysom_api/apps/vmcore/models.py diff --git a/sysom_api/apps/vmcore/serializer.py b/sysom_server/sysom_api/apps/vmcore/serializer.py similarity index 100% rename from sysom_api/apps/vmcore/serializer.py rename to sysom_server/sysom_api/apps/vmcore/serializer.py diff --git a/sysom_api/apps/monitor/tests.py b/sysom_server/sysom_api/apps/vmcore/tests.py similarity index 100% rename from sysom_api/apps/monitor/tests.py rename to sysom_server/sysom_api/apps/vmcore/tests.py diff --git a/sysom_api/apps/vmcore/urls.py b/sysom_server/sysom_api/apps/vmcore/urls.py similarity index 100% rename from sysom_api/apps/vmcore/urls.py rename to sysom_server/sysom_api/apps/vmcore/urls.py diff --git a/sysom_api/apps/vmcore/views.py b/sysom_server/sysom_api/apps/vmcore/views.py similarity index 100% rename from sysom_api/apps/vmcore/views.py rename to sysom_server/sysom_api/apps/vmcore/views.py diff --git a/sysom_api/apps/vmcore/vmcore.json b/sysom_server/sysom_api/apps/vmcore/vmcore.json similarity index 100% rename from sysom_api/apps/vmcore/vmcore.json rename to sysom_server/sysom_api/apps/vmcore/vmcore.json diff --git a/sysom_api/apps/task/__init__.py b/sysom_server/sysom_api/apps/vul/__init__.py similarity index 100% rename from sysom_api/apps/task/__init__.py rename to sysom_server/sysom_api/apps/vul/__init__.py diff --git a/sysom_api/apps/task/admin.py b/sysom_server/sysom_api/apps/vul/admin.py similarity index 100% rename from sysom_api/apps/task/admin.py rename to sysom_server/sysom_api/apps/vul/admin.py diff --git a/sysom_api/apps/vul/apps.py b/sysom_server/sysom_api/apps/vul/apps.py similarity index 42% rename from sysom_api/apps/vul/apps.py rename to sysom_server/sysom_api/apps/vul/apps.py index 327791d4..8f439c76 100644 --- a/sysom_api/apps/vul/apps.py +++ b/sysom_server/sysom_api/apps/vul/apps.py @@ -3,23 +3,46 @@ from django.apps import AppConfig from django.db.models.signals import post_migrate logger = logging.getLogger(__name__) + class VulConfig(AppConfig): default_auto_field = 'django.db.models.BigAutoField' name = 'apps.vul' def ready(self) -> None: post_migrate.connect(initialization_vul_config, sender=self) + bind_ssh_key() logger.info(">>> Vul module loading success") +def bind_ssh_key(): + from django.conf import settings + from lib.ssh import SSH + + def private_key_getter() -> str: + result = "" + with open(settings.SSH_CHANNEL_KEY_PRIVATE) as f: + result = f.read() + return result + + def public_key_getter() -> str: + result = "" + with open(settings.SSH_CHANNEL_KEY_PUB) as f: + result = f.read() + return result + + SSH.set_private_key_getter(private_key_getter) + SSH.set_public_key_getter(public_key_getter) + + def initialization_vul_config(sender, **kwargs): try: from .models import VulAddrModel VulAddrModel.objects.create( name="Anolis 漏洞数据", - description = "Anolis 漏洞数据", - url = "https://anas.openanolis.cn/api/portal/v1/cves/?page_num=1&page_size=50", - parser = {"cve_item_path":"data/data","cve_id_flag":"cve_id","score_flag":"score","pub_time_flag":"publish_date"} + description="Anolis 漏洞数据", + url="https://anas.openanolis.cn/api/portal/v1/cves/?page_num=1&page_size=50", + parser={"cve_item_path": "data/data", "cve_id_flag": "cve_id", + "score_flag": "score", "pub_time_flag": "publish_date"} ) except Exception as e: - pass \ No newline at end of file + pass diff --git a/sysom_api/apps/vul/models.py b/sysom_server/sysom_api/apps/vul/models.py similarity index 100% rename from sysom_api/apps/vul/models.py rename to sysom_server/sysom_api/apps/vul/models.py diff --git a/sysom_api/apps/vul/serializer.py b/sysom_server/sysom_api/apps/vul/serializer.py similarity index 100% rename from sysom_api/apps/vul/serializer.py rename to sysom_server/sysom_api/apps/vul/serializer.py diff --git a/sysom_api/apps/vul/ssh_pool.py b/sysom_server/sysom_api/apps/vul/ssh_pool.py similarity index 98% rename from sysom_api/apps/vul/ssh_pool.py rename to sysom_server/sysom_api/apps/vul/ssh_pool.py index a1dab728..92cf3745 100644 --- a/sysom_api/apps/vul/ssh_pool.py +++ b/sysom_server/sysom_api/apps/vul/ssh_pool.py @@ -11,7 +11,7 @@ import logging import multiprocessing import time -from lib.channels.ssh import SSH +from lib.ssh import SSH class SshProcessQueueManager: diff --git a/sysom_api/apps/task/tests.py b/sysom_server/sysom_api/apps/vul/tests.py similarity index 100% rename from sysom_api/apps/task/tests.py rename to sysom_server/sysom_api/apps/vul/tests.py diff --git a/sysom_api/apps/vul/urls.py b/sysom_server/sysom_api/apps/vul/urls.py similarity index 100% rename from sysom_api/apps/vul/urls.py rename to sysom_server/sysom_api/apps/vul/urls.py diff --git a/sysom_api/apps/vul/views.py b/sysom_server/sysom_api/apps/vul/views.py similarity index 100% rename from sysom_api/apps/vul/views.py rename to sysom_server/sysom_api/apps/vul/views.py diff --git a/sysom_api/apps/vul/vul.py b/sysom_server/sysom_api/apps/vul/vul.py similarity index 100% rename from sysom_api/apps/vul/vul.py rename to sysom_server/sysom_api/apps/vul/vul.py diff --git a/sysom_api/apps/vmcore/__init__.py b/sysom_server/sysom_api/conf/__init__.py similarity index 100% rename from sysom_api/apps/vmcore/__init__.py rename to sysom_server/sysom_api/conf/__init__.py diff --git a/sysom_api/conf/common.py b/sysom_server/sysom_api/conf/common.py similarity index 81% rename from sysom_api/conf/common.py rename to sysom_server/sysom_api/conf/common.py index edb29c2e..2678a525 100644 --- a/sysom_api/conf/common.py +++ b/sysom_server/sysom_api/conf/common.py @@ -20,10 +20,8 @@ INSTALLED_APPS = [ 'apps.accounts', 'apps.host', 'apps.monitor', - 'apps.task', 'apps.vmcore', 'apps.alarm', - 'apps.channel', 'rest_framework', 'corsheaders', @@ -120,32 +118,29 @@ IS_MICRO_SERVICES = False # 是否微服务 WEB_DIR = os.path.join(BASE_DIR.parent, 'sysom_web') DOWNLOAD_DIR = os.path.join(WEB_DIR, 'download') -# sysom service -SYSOM_SERVICE = 'http://127.0.0.1:7001' -TASK_SERVICE = 'http://127.0.0.1:7002' -CHANNEL_SERVICE = 'http://127.0.0.1:7003' -# key path -KEY_PATH = os.path.join(BASE_DIR, 'conf', 'ssh-key') +################################################################## +# SSH channel settings +################################################################## +SSH_CHANNEL_KEY_DIR = os.path.join(BASE_DIR.parent, 'conf', 'ssh') +if not os.path.exists(SSH_CHANNEL_KEY_DIR): + os.makedirs(SSH_CHANNEL_KEY_DIR) +SSH_CHANNEL_KEY_PRIVATE = os.path.join(SSH_CHANNEL_KEY_DIR, "sysom_id") +SSH_CHANNEL_KEY_PUB = os.path.join(SSH_CHANNEL_KEY_DIR, "sysom_id.pub") ################################################################## # Cec settings ################################################################## SYSOM_CEC_URL = "redis://localhost:6379?cec_default_max_len=1000&cec_auto_mk_topic=true" SYSOM_CEC_ALARM_TOPIC = "CEC-SYSOM-ALARM" -# 任务模块用于生成诊断任务的主题(可以通过将诊断参数发送到这个主题来触发诊断任务的执行) -SYSOM_CEC_TASK_GENERATE_TOPIC = "CEC-SYSOM-TASK-GENERATE-TOPCI" -SYSOM_CEC_TASK_GENERATE_TASK_LISTENER = "CEC-SYSOM-TASK-GENERATE-TASK-LISTENER" -# 任务模块用于下发任务的主题 => 执行完预处理脚本,待下发到node执行 -SYSOM_CEC_TASK_DELIVERY_TOPIC = "CEC-SYSOM-TASK-DELIVERY" -SYSOM_CEC_TASK_DELIVERY_GROUP = "CEC-SYSOM-TASK-DELIVERY-GROUP" # 读取任务的消费组 -# 任务模块用于下发任务处理结果的主题 => node执行结束,待后处理脚本执行 -SYSOM_CEC_TASK_RESULT_PROCESS_TOPIC = "CEC-SYSOM-TASK-RESULT-PROCESS" -SYSOM_CEC_TASK_RESULT_PROCESS_GROUP = "CEC-SYSOM-TASK-RESULT-PROCESS-GROUP" -# 诊断任务最终执行结果报告 => 后处理脚本执行结束,报告最终任务的执行结果 -SYSOM_CEC_TASK_EXECUTE_RESULT_TOPIC = "CEC-SYSOM-TASK-EXECUTE-RESULT" -# 主机模块监听诊断任务执行结果消费组 -SYSOM_CEC_TASK_RESULT_LISTENER_HOST_GROUP = "CEC-SYSOM-TASK-RESULT-LISTENER-HOST-GROUP" +# 通道模块用于对外开放,投递操作的主题 +SYSOM_CEC_CHANNEL_TOPIC = "SYSOM_CEC_CHANNEL_TOPIC" +# 通道模块用于投递执行结果的主题 +SYSOM_CEC_CHANNEL_RESULT_TOPIC = "SYSOM_CEC_CHANNEL_RESULT_TOPIC" +# 用于分发插件系统相关事件的主题 +SYSOM_CEC_PLUGIN_TOPIC = "SYSOM_CEC_PLUGIN_TOPIC" +# API主机模块消费组 +SYSOM_CEC_API_HOST_CONSUMER_GROUP = "SYSOM_CEC_API_HOST_CONSUMER_GROUP" SERVER_LOGS_FILE = os.path.join(BASE_DIR, 'logs', 'sys_om_info.log') ERROR_LOGS_FILE = os.path.join(BASE_DIR, 'logs', 'sys_om_error.log') diff --git a/sysom_api/conf/develop.py b/sysom_server/sysom_api/conf/develop.py similarity index 100% rename from sysom_api/conf/develop.py rename to sysom_server/sysom_api/conf/develop.py diff --git a/sysom_api/conf/product.py b/sysom_server/sysom_api/conf/product.py similarity index 100% rename from sysom_api/conf/product.py rename to sysom_server/sysom_api/conf/product.py diff --git a/sysom_api/conf/testing.py b/sysom_server/sysom_api/conf/testing.py similarity index 100% rename from sysom_api/conf/testing.py rename to sysom_server/sysom_api/conf/testing.py diff --git a/sysom_api/consumer/__init__.py b/sysom_server/sysom_api/consumer/__init__.py similarity index 100% rename from sysom_api/consumer/__init__.py rename to sysom_server/sysom_api/consumer/__init__.py diff --git a/sysom_api/consumer/consumers.py b/sysom_server/sysom_api/consumer/consumers.py similarity index 97% rename from sysom_api/consumer/consumers.py rename to sysom_server/sysom_api/consumer/consumers.py index 3c4e22c7..7705d519 100644 --- a/sysom_api/consumer/consumers.py +++ b/sysom_server/sysom_api/consumer/consumers.py @@ -7,7 +7,7 @@ from channels.exceptions import StopConsumer from django.conf import settings import os from django.conf import settings -from sdk.cec_base.consumer import Consumer, dispatch_consumer +from cec_base.consumer import Consumer, dispatch_consumer logger = logging.getLogger(__name__) @@ -43,7 +43,7 @@ class SshConsumer(WebsocketConsumer): def _connect_host_init(self): """初始化host连接""" from apps.host.models import HostModel - from lib.channels.ssh import SSH + from lib.ssh import SSH instance = get_host_instance( model=HostModel, ip=self.host_ip, created_by=self.user.id) if not instance: diff --git a/sysom_api/consumer/middleware.py b/sysom_server/sysom_api/consumer/middleware.py similarity index 100% rename from sysom_api/consumer/middleware.py rename to sysom_server/sysom_api/consumer/middleware.py diff --git a/sysom_api/consumer/routing.py b/sysom_server/sysom_api/consumer/routing.py similarity index 100% rename from sysom_api/consumer/routing.py rename to sysom_server/sysom_api/consumer/routing.py diff --git a/sysom_api/lib/__init__.py b/sysom_server/sysom_api/lib/__init__.py similarity index 100% rename from sysom_api/lib/__init__.py rename to sysom_server/sysom_api/lib/__init__.py diff --git a/sysom_api/lib/authentications.py b/sysom_server/sysom_api/lib/authentications.py similarity index 100% rename from sysom_api/lib/authentications.py rename to sysom_server/sysom_api/lib/authentications.py diff --git a/sysom_api/lib/base_model.py b/sysom_server/sysom_api/lib/base_model.py similarity index 100% rename from sysom_api/lib/base_model.py rename to sysom_server/sysom_api/lib/base_model.py diff --git a/sysom_api/lib/decode/sysom_decode.py b/sysom_server/sysom_api/lib/decode/sysom_decode.py similarity index 100% rename from sysom_api/lib/decode/sysom_decode.py rename to sysom_server/sysom_api/lib/decode/sysom_decode.py diff --git a/sysom_api/lib/excel.py b/sysom_server/sysom_api/lib/excel.py similarity index 100% rename from sysom_api/lib/excel.py rename to sysom_server/sysom_api/lib/excel.py diff --git a/sysom_api/lib/exception.py b/sysom_server/sysom_api/lib/exception.py similarity index 100% rename from sysom_api/lib/exception.py rename to sysom_server/sysom_api/lib/exception.py diff --git a/sysom_api/lib/paginations.py b/sysom_server/sysom_api/lib/paginations.py similarity index 100% rename from sysom_api/lib/paginations.py rename to sysom_server/sysom_api/lib/paginations.py diff --git a/sysom_api/lib/renderers.py b/sysom_server/sysom_api/lib/renderers.py similarity index 100% rename from sysom_api/lib/renderers.py rename to sysom_server/sysom_api/lib/renderers.py diff --git a/sysom_api/lib/response.py b/sysom_server/sysom_api/lib/response.py similarity index 100% rename from sysom_api/lib/response.py rename to sysom_server/sysom_api/lib/response.py diff --git a/sysom_server/sysom_api/lib/ssh.py b/sysom_server/sysom_api/lib/ssh.py new file mode 100644 index 00000000..9de53fc4 --- /dev/null +++ b/sysom_server/sysom_api/lib/ssh.py @@ -0,0 +1,100 @@ +import logging +from typing import Callable +import paramiko +from io import StringIO +from paramiko.client import SSHClient, AutoAddPolicy +from paramiko.rsakey import RSAKey + +DEFAULT_CONNENT_TIMEOUT = 5 # 默认ssh链接超时时间 5s +DEFAULT_NODE_USER = 'root' # 默认节点用户名 root + +logger = logging.getLogger(__name__) + + +class SSH: + """A SSH client used to run command in remote node + + args: + hostname(str): Host name + + Keyword Args: + username(str): User name, default 'root' + port(str): SSH communicate port, default 22 + connect_timeout(int): Connection timeout duration, default 5s + password(str) + """ + + # key_pair cached the key pair generated by initialization stage + _key_pair = {} + _private_key_getter: Callable[[], str] = None + _public_key_getter: Callable[[], str] = None + + def __init__(self, hostname: str, **kwargs) -> None: + self.connect_args = { + 'hostname': hostname, + 'username': kwargs.get('username', DEFAULT_NODE_USER), + 'port': kwargs.get('port', 22), + 'timeout': kwargs.get('timeout', DEFAULT_CONNENT_TIMEOUT), + } + if 'password' in kwargs and kwargs['password'] is not None: + self.connect_args['password'] = kwargs.get('password') + else: + if SSH._private_key_getter is None: + raise Exception("_private_key_getter not set") + self.connect_args['pkey'] = RSAKey.from_private_key( + StringIO(SSH._private_key_getter()) + ) + + self._client: SSHClient = self.client() + + def client(self): + try: + client = SSHClient() + client.set_missing_host_key_policy(AutoAddPolicy) + client.connect(**self.connect_args) + return client + except paramiko.AuthenticationException: + raise Exception('authorization fail, password or pkey error!') + except: + raise Exception('authorization fail!') + + @classmethod + def set_private_key_getter(cls, private_key_getter: Callable[[], str]): + cls._private_key_getter = private_key_getter + + @classmethod + def set_public_key_getter(cls, public_key_getter: Callable[[], str]): + cls._public_key_getter = public_key_getter + + def run_command(self, command): + if self._client: + ssh_session = self._client.get_transport().open_session() + ssh_session.set_combine_stderr(True) + ssh_session.exec_command(command) + stdout = ssh_session.makefile("rb", -1) + statue = ssh_session.recv_exit_status() + output = stdout.read().decode() + return statue, output + else: + raise Exception('No client!') + + def add_public_key(self): + if self._public_key_getter is None: + raise Exception("_public_key_getter not set") + public_key = SSH._public_key_getter() + command = f'mkdir -p -m 700 ~/.ssh && \ + echo {public_key!r} >> ~/.ssh/authorized_keys && \ + chmod 600 ~/.ssh/authorized_keys' + statue, _ = self.run_command(command) + if statue != 0: + raise Exception('add public key faild!') + + @staticmethod + def validate_ssh_host(ip: str, password: str, port: int = 22, username: str = 'root'): + try: + ssh = SSH(hostname=ip, password=password, + port=port, username=username, timeout=2) + ssh.add_public_key() + return True, 'authorization success' + except Exception as e: + return False, f'error: {e}' diff --git a/sysom_api/lib/utils.py b/sysom_server/sysom_api/lib/utils.py similarity index 95% rename from sysom_api/lib/utils.py rename to sysom_server/sysom_api/lib/utils.py index 5ee1eb2b..c553c245 100644 --- a/sysom_api/lib/utils.py +++ b/sysom_server/sysom_api/lib/utils.py @@ -8,6 +8,7 @@ """ import time import uuid as UUID +from typing import List import json import logging import jwt @@ -136,6 +137,13 @@ def import_string(dotted_path: str): module_path, class_name) ) from err +def valid_params(require_params: dict, current_params: dict) -> List[str]: + missing_param_list = [] + for param in require_params: + if param not in current_params: + missing_param_list.append(param) + return missing_param_list + def generate_key(): key_obj = StringIO() key = RSAKey.generate(2048) diff --git a/sysom_api/manage.py b/sysom_server/sysom_api/manage.py similarity index 100% rename from sysom_api/manage.py rename to sysom_server/sysom_api/manage.py diff --git a/sysom_api/sysom/__init__.py b/sysom_server/sysom_api/sysom/__init__.py similarity index 100% rename from sysom_api/sysom/__init__.py rename to sysom_server/sysom_api/sysom/__init__.py diff --git a/sysom_api/sysom/asgi.py b/sysom_server/sysom_api/sysom/asgi.py similarity index 100% rename from sysom_api/sysom/asgi.py rename to sysom_server/sysom_api/sysom/asgi.py diff --git a/sysom_api/sysom/settings.py b/sysom_server/sysom_api/sysom/settings.py similarity index 100% rename from sysom_api/sysom/settings.py rename to sysom_server/sysom_api/sysom/settings.py diff --git a/sysom_api/sysom/urls.py b/sysom_server/sysom_api/sysom/urls.py similarity index 100% rename from sysom_api/sysom/urls.py rename to sysom_server/sysom_api/sysom/urls.py diff --git a/sysom_api/sysom/wsgi.py b/sysom_server/sysom_api/sysom/wsgi.py similarity index 100% rename from sysom_api/sysom/wsgi.py rename to sysom_server/sysom_api/sysom/wsgi.py diff --git a/sysom_api/apps/vul/__init__.py b/sysom_server/sysom_channel/apps/channel/__init__.py similarity index 100% rename from sysom_api/apps/vul/__init__.py rename to sysom_server/sysom_channel/apps/channel/__init__.py diff --git a/sysom_api/apps/vmcore/admin.py b/sysom_server/sysom_channel/apps/channel/admin.py similarity index 100% rename from sysom_api/apps/vmcore/admin.py rename to sysom_server/sysom_channel/apps/channel/admin.py diff --git a/sysom_server/sysom_channel/apps/channel/apps.py b/sysom_server/sysom_channel/apps/channel/apps.py new file mode 100644 index 00000000..39103436 --- /dev/null +++ b/sysom_server/sysom_channel/apps/channel/apps.py @@ -0,0 +1,102 @@ +import json +import logging +from multiprocessing.connection import wait +import sys +from unicodedata import name +from django.apps import AppConfig +from django.db.models.signals import post_migrate + + +logger = logging.getLogger(__name__) + + +class ChannelConfig(AppConfig): + default_auto_field = 'django.db.models.BigAutoField' + name = 'apps.channel' + + def ready(self): + # 1. Perform the necessary initialization operations when the + # database is changed + post_migrate.connect(on_post_migrate, sender=self) + + # 2. Read key pair from database then save as file + export_key_pair() + + # 3. Start channel listener + start_channel_listener() + + logger.info(">>> Channel module loading success") + + +def on_post_migrate(sender, **kwargs): + """Invoke after migrate successful""" + + # 1. Try to auto generate a pair of SSH key used to connect remote node + try: + from .models import ChannelSettingModel + from lib.utils import generate_key + + k, v = generate_key() + ssh_key = json.dumps({"private_key": k, "public_key": v}) + ChannelSettingModel.objects.create( + name="ssh_key", value=ssh_key, description="SysOM Channel auto generated key") + except Exception as exc: + pass + + +def export_key_pair(): + """Export key pair as file + """ + from django.conf import settings + from .models import ChannelSettingModel + from lib.ssh import SSH + + def private_key_getter() -> str: + result = "" + with open(settings.SSH_CHANNEL_KEY_PRIVATE) as f: + result = f.read() + return result + + def public_key_getter() -> str: + result = "" + with open(settings.SSH_CHANNEL_KEY_PUB) as f: + result = f.read() + return result + + try: + instance = ChannelSettingModel.objects.get(name="ssh_key") + key_pair = json.loads(instance.value) + + # 1. Export private key + # TODO: The exported secret key needs to be encrypted here + with open(settings.SSH_CHANNEL_KEY_PRIVATE, 'w') as f: + f.write(key_pair["private_key"]) + + # 2. Export public key + with open(settings.SSH_CHANNEL_KEY_PUB, "w") as f: + f.write(key_pair["public_key"]) + + # 3. Set SSH key getter + SSH.set_private_key_getter(private_key_getter) + SSH.set_public_key_getter(public_key_getter) + + except Exception as exc: + logger.exception(exc) + + +def start_channel_listener(): + if ('runserver' in sys.argv or 'manage.py' not in sys.argv): + from cec_base.log import LoggerHelper, LoggerLevel + from apps.channel.executor import ChannelListener + + LoggerHelper.update_sys_stdout_sink(LoggerLevel.LOGGER_LEVEL_INFO) + # 这边微服务正式启动的时候执行一些处理代码 + # 启动任务结果处理线程 + try: + ChannelListener().start() + except Exception as e: + logger.exception(e) + else: + # 这边执行数据库迁移等操作的时候执行一些处理代码 + pass + pass diff --git a/sysom_api/microservices/task_executor/executor.py b/sysom_server/sysom_channel/apps/channel/executor.py similarity index 32% rename from sysom_api/microservices/task_executor/executor.py rename to sysom_server/sysom_channel/apps/channel/executor.py index 27e67d8c..4631c062 100644 --- a/sysom_api/microservices/task_executor/executor.py +++ b/sysom_server/sysom_channel/apps/channel/executor.py @@ -10,72 +10,80 @@ import logging from concurrent.futures import ThreadPoolExecutor from importlib import import_module import os -from sdk.cec_base.event import Event -from sdk.cec_base.consumer import Consumer -from sdk.cec_base.cec_client import CecClient -from sdk.cec_base.log import LoggerHelper -from .conf import * +from typing import Callable +from cec_base.event import Event +from cec_base.consumer import Consumer +from cec_base.producer import Producer +from cec_base.cec_client import CecClient +from cec_base.log import LoggerHelper +from django.conf import settings logger = logging.getLogger(__name__) -class TaskExecutor(CecClient): - """ A cec-based task executor +class ChannelListener(CecClient): + """ A cec-based channel listener - 一个基于事件中心实现的任务执行器 + A cec-based channel lilster, ssed to listen to requests for channels from + other modules and output the results to cec after performing the corresponding + operation on the target node Args: - url(str): 用于连接事件中心的地址 - task_delivery_topic(str): 用于获取任务的主题 - consumer_id(str): 用于消费任务的消费者ID - group_id(str): 用于消费任务的消费者组ID - task_result_topic(str): 用于投递任务处理结果的主题 - task_process_thread_num(str): 用于执行任务的线程池中包含的线程的数量 + task_process_thread_num(str): The number of threads contained in the thread + pool used to execute the task + """ - def __init__(self, url: str, task_delivery_topic: str, consumer_id: str, group_id: str, - task_result_topic: str, - task_process_thread_num: int = 5) -> None: - CecClient.__init__(self, url) + def __init__(self, task_process_thread_num: int = 5) -> None: + CecClient.__init__(self, settings.SYSOM_CEC_URL) self.append_group_consume_task( - task_delivery_topic, group_id, consumer_id, ensure_topic_exist=True + settings.SYSOM_CEC_CHANNEL_TOPIC, + settings.SYSOM_CEC_CHANNEL_CONSUMER_GROUP, + Consumer.generate_consumer_id(), + ensure_topic_exist=True ) - self._target_topic = task_result_topic + self._target_topic = settings.SYSOM_CEC_CHANNEL_RESULT_TOPIC + + # Define opt table + self._opt_table = { + 'init': self._do_init_channel, + "cmd": self._do_run_command + } # 执行任务的线程池数量 self._task_process_thread_poll = ThreadPoolExecutor( max_workers=task_process_thread_num) - def _get_channel(self, data: dict): + def _get_channel(self, channel_type): """ 根据要执行的命令,动态引入一个 Channel 的实现用于执行命令 """ - channel_type = data.pop('channel', 'ssh') try: - package = import_module(f'lib.channels.{channel_type}') - return package.Channel(**data, channel_name=channel_type) + return import_module(f'lib.channels.{channel_type}').Channel except Exception as e: raise Exception(f'No channels available => {str(e)}') - def _do_run_command(self, data: dict): - # 1. 首先在指定的channel尝试执行 + def _perform_opt(self, opt_func: Callable[[str, dict], dict], + default_channel: str, params: dict) -> dict: + """ + Use the specified channel to perform operations on the remote + node and return the results. + """ result, err = {}, None try: - result = self._get_channel(data).run_command() + result = opt_func(default_channel, params) except Exception as exc: logger.error(exc) err = exc - # 2. 默认通道执行失败,在此处遍历尝试其它可能可用的channel - channels_path = os.path.join(BASE_DIR, 'lib', 'channels') + channels_path = os.path.join(settings.BASE_DIR, 'lib', 'channels') packages = [dir.replace('.py', '') for dir in os.listdir( channels_path) if not dir.startswith('__')] packages.remove('base') - packages.remove('ssh') + packages.remove(default_channel) for _, pkg in enumerate(packages): try: - result = self._get_channel( - {**data, "channel": pkg}).run_command() + result = opt_func(pkg, params) err = None break except Exception as exc: @@ -85,57 +93,55 @@ class TaskExecutor(CecClient): raise err return result + def _do_run_command(self, channel_type: str, params: dict) -> dict: + """cmd opt""" + return self._get_channel(channel_type)(**params).run_command() + + def _do_init_channel(self, channel_type: str, params: dict) -> dict: + """init opt""" + return self._get_channel(channel_type).initial(**params) + def _process_each_task(self, consumer: Consumer, event: Event): """ 处理每个单独的任务 """ + task = event.value + result = { + "code": 0, + "err_msg": "", + "echo": task.get("echo", {}), + "result": {} + } try: - task = event.value - command_list = task.get('command', []) - task_id = task.get("task_id", "") - result = { - "status": 0, - "task_id": task_id, - "errMsg": "", - "results": [] - } - for script in command_list: - # 检查参数 - ip = script.get("instance", None) - cmd = script.get("cmd", None) - if not ip or not cmd: - result["status"] = 1 - result["errMsg"] = "script result find not instance or cmd" - break + opt_type = task.get("type", "cmd") + channel_type = task.get("channel", "ssh") + params = task.get("params", {}) - # # 根据参数动态引入 Channel 包,构造一个 Channel 实例用于执行命令 - # channel = self._get_channel(script) - - # 使用对应的 Channel 执行命令,并得到结果 - # 取出任务执行的结果:{'state': 0, 'result': 'xxxxxxxx'} 0为执行成功, 1位=为执行失败 - channel_result = self._do_run_command(script) - status = channel_result.get("state", 1) - res = channel_result.get("result", {}).get("result") - # 如果本轮的命令执行出错,则直接停止执行任务,返回错误信息 - if status != 0: - if not res: - res = f"Sysak doesn't return any error msg, state = {status}" - result["status"] = 1 - result["errMsg"] = res - break - - # 如果命令执行成功,则将执行的结果保存 - result["results"].append(res) + if opt_type not in self._opt_table: + result["code"] = 1 + result["err_msg"] = f"Not support opt: {opt_type}" + else: + code, raw_result = self._perform_opt( + self._opt_table[opt_type], + channel_type, + params + ) + result["code"] = code + if code != 0: + result["err_msg"] = raw_result + else: + result["result"] = raw_result except Exception as e: LoggerHelper.get_lazy_logger().error(e) LoggerHelper.get_lazy_logger().exception(e) - result["status"] = 1 - result["errMsg"] = str(e) - # 执行消息确认 - res = consumer.ack(event) - # 将任务执行的结果写入到事件中心,供 Task 模块获取 - self.delivery(self._target_topic, result) - - def on_receive_event(self, consumer: Consumer, event: Event, task: dict): + result["code"] = 1 + result["err_msg"] = str(e) + finally: + # 执行消息确认 + res = consumer.ack(event) + # 将任务执行的结果写入到事件中心,供 Task 模块获取 + self.delivery(self._target_topic, result) + + def on_receive_event(self, consumer: Consumer, producer: Producer, event: Event, task: dict): self._task_process_thread_poll.submit( self._process_each_task, consumer, event) diff --git a/sysom_server/sysom_channel/apps/channel/models.py b/sysom_server/sysom_channel/apps/channel/models.py new file mode 100644 index 00000000..65c4bcf1 --- /dev/null +++ b/sysom_server/sysom_channel/apps/channel/models.py @@ -0,0 +1,14 @@ +from django.db import models +from lib.base_model import BaseModel + + +class ChannelSettingModel(BaseModel): + name = models.CharField(max_length=254, verbose_name="Channel config name", unique=True) + value = models.TextField(verbose_name="Channel config value") + description = models.TextField(default="", verbose_name="Channel config description") + + class Meta: + db_table = 'sys_channel_setting' + + def __str__(self) -> str: + return f"ChannelSetting: {self.name}: {self.value}" \ No newline at end of file diff --git a/sysom_api/apps/task/seriaizer.py b/sysom_server/sysom_channel/apps/channel/seriaizer.py similarity index 76% rename from sysom_api/apps/task/seriaizer.py rename to sysom_server/sysom_channel/apps/channel/seriaizer.py index 40af11e6..7fd52a5f 100644 --- a/sysom_api/apps/task/seriaizer.py +++ b/sysom_server/sysom_channel/apps/channel/seriaizer.py @@ -12,16 +12,16 @@ from apps.task.models import JobModel class JobListSerializer(serializers.ModelSerializer): class Meta: model = JobModel - exclude = ('deleted_at', 'host_by', 'command', 'result') + exclude = ('host_by', 'command', 'result') class JobRetrieveSerializer(serializers.ModelSerializer): class Meta: model = JobModel - exclude = ('deleted_at', 'host_by', 'command') + exclude = ('host_by', 'command') class JobDetailSerializer(serializers.ModelSerializer): class Meta: model = JobModel - exclude = ['deleted_at'] + fields = '__all__' diff --git a/sysom_api/apps/vmcore/tests.py b/sysom_server/sysom_channel/apps/channel/tests.py similarity index 100% rename from sysom_api/apps/vmcore/tests.py rename to sysom_server/sysom_channel/apps/channel/tests.py diff --git a/sysom_server/sysom_channel/apps/channel/urls.py b/sysom_server/sysom_channel/apps/channel/urls.py new file mode 100644 index 00000000..b434fe61 --- /dev/null +++ b/sysom_server/sysom_channel/apps/channel/urls.py @@ -0,0 +1,13 @@ +# -*- encoding: utf-8 -*- +""" +@File : urls.py +@Time : 2021/11/22 10:38 +@Author : DM +@Software: PyCharm +""" +from rest_framework.routers import DefaultRouter + +router = DefaultRouter() + +urlpatterns = [ +] diff --git a/sysom_server/sysom_channel/apps/channel/views.py b/sysom_server/sysom_channel/apps/channel/views.py new file mode 100644 index 00000000..020b519e --- /dev/null +++ b/sysom_server/sysom_channel/apps/channel/views.py @@ -0,0 +1,12 @@ +import logging +from django.http.response import FileResponse +from django_filters.rest_framework import DjangoFilterBackend +from django.shortcuts import get_object_or_404 +from rest_framework.filters import SearchFilter, OrderingFilter +from lib.base_view import CommonModelViewSet + +logger = logging.getLogger(__name__) + + +class ChannelView(CommonModelViewSet): + pass \ No newline at end of file diff --git a/sysom_api/conf/channel_gunicorn.py b/sysom_server/sysom_channel/conf/channel_gunicorn.py similarity index 86% rename from sysom_api/conf/channel_gunicorn.py rename to sysom_server/sysom_channel/conf/channel_gunicorn.py index db2ba9c6..5f576609 100644 --- a/sysom_api/conf/channel_gunicorn.py +++ b/sysom_server/sysom_channel/conf/channel_gunicorn.py @@ -12,10 +12,11 @@ worker_class = 'gevent' # 工作模式线程, 默认为sync模式 max_requests = 2000 # 设置最大并发数量为2000 (每个worker处理请求的工作线程) accesslog = '/usr/local/sysom/server/logs/channel-service.log' + errorlog = '/usr/local/sysom/server/logs/channel-service-error.log' loglevel = 'info' proc_name = 'channel_service' -raw_env = 'DJANGO_SETTINGS_MODULE=sysom.services_conf.channel_settings' +raw_env = 'DJANGO_SETTINGS_MODULE=sysom_channel.settings' diff --git a/sysom_server/sysom_channel/conf/common.py b/sysom_server/sysom_channel/conf/common.py new file mode 100644 index 00000000..d897fe94 --- /dev/null +++ b/sysom_server/sysom_channel/conf/common.py @@ -0,0 +1,188 @@ +import os +import socket +from pathlib import Path + + +def get_ip_address(): + """ip address""" + hostname = socket.gethostname() + return socket.gethostbyname(hostname) + + +BASE_DIR = Path(__file__).resolve().parent.parent +SECRET_KEY = 'django-insecure-^d8b9di9w&-mmsbpt@)o#e+2^z+^m4nhf+z8304%9@8y#ko46l' + +ALLOWED_HOSTS = ['*'] + +INSTALLED_APPS = [ + 'apps.channel', + + 'rest_framework', + 'corsheaders', + 'drf_yasg', # 在线API文档 + 'channels', + 'django_filters', + 'django_apscheduler', +] + +MIDDLEWARE = [ + 'corsheaders.middleware.CorsMiddleware', + 'django.middleware.security.SecurityMiddleware', + 'django.middleware.common.CommonMiddleware', +] + +DEBUG = True + +SCRIPTS_DIR = os.path.join(BASE_DIR, 'service_scripts') + +# Mysql数据库 +DATABASES = { + 'default': { + 'ENGINE': 'django.db.backends.mysql', + 'NAME': 'sysom', + 'USER': 'sysom', + 'PASSWORD': 'sysom_admin', + 'HOST': '127.0.0.1', + 'PORT': '3306', + } +} + +ROOT_URLCONF = 'sysom_channel.urls' + +WSGI_APPLICATION = 'sysom_channel.wsgi.application' +ASGI_APPLICATION = 'sysom_channel.asgi.application' + +DEFAULT_AUTO_FIELD = 'django.db.models.BigAutoField' + +LANGUAGE_CODE = 'zh-hans' +TIME_ZONE = 'Asia/Shanghai' +USE_I18N = True +USE_L10N = True +USE_TZ = True +STATIC_URL = '/static/' +STATIC_ROOT = os.path.join(BASE_DIR, 'index/status') + +# rest_framework settings +REST_FRAMEWORK = { + 'DEFAULT_PERMISSION_CLASSES': ( + # 'rest_framework.permissions.IsAuthenticated' + ), + # 'DEFAULT_AUTHENTICATION_CLASSES': [ + # 'apps.accounts.authentication.Authentication' + # ], + 'UNAUTHENTICATED_USER': None, + 'DEFAULT_VERSIONING_CLASS': "rest_framework.versioning.URLPathVersioning", + 'DEFAULT_VERSION': 'v1', # 默认版本 + 'ALLOWED_VERSIONS': ['v1', 'v2'], # 允许的版本 + 'VERSION_PARAM': 'version', + + # 'DEFAULT_RENDERER_CLASSES': ( + # 'lib.renderers.SysomJsonRender', + # ), + 'DEFAULT_PAGINATION_CLASS': 'lib.paginations.Pagination', + 'UNICODE_JSON': True, + 'EXCEPTION_HANDLER': 'lib.exception.exception_handler' +} + + +################################################################## +# SSH channel settings +################################################################## +SSH_CHANNEL_KEY_DIR = os.path.join(BASE_DIR.parent, 'conf', 'ssh') +if not os.path.exists(SSH_CHANNEL_KEY_DIR): + os.makedirs(SSH_CHANNEL_KEY_DIR) +SSH_CHANNEL_KEY_PRIVATE = os.path.join(SSH_CHANNEL_KEY_DIR, "sysom_id") +SSH_CHANNEL_KEY_PUB = os.path.join(SSH_CHANNEL_KEY_DIR, "sysom_id.pub") + +################################################################## +# Cec settings +################################################################## +SYSOM_CEC_URL = "redis://localhost:6379?cec_default_max_len=1000&cec_auto_mk_topic=true" +# 通道模块用于对外开放,投递操作的主题 +SYSOM_CEC_CHANNEL_TOPIC = "SYSOM_CEC_CHANNEL_TOPIC" +# 通道模块消费组 +SYSOM_CEC_CHANNEL_CONSUMER_GROUP = "SYSOM_CEC_CHANNEL_CONSUMER_GROUP" +# 通道模块用于投递执行结果的主题 +SYSOM_CEC_CHANNEL_RESULT_TOPIC = "SYSOM_CEC_CHANNEL_RESULT_TOPIC" + +SERVER_LOGS_FILE = os.path.join(BASE_DIR, 'logs', 'sys_om_info.log') +ERROR_LOGS_FILE = os.path.join(BASE_DIR, 'logs', 'sys_om_error.log') +if not os.path.exists(os.path.join(BASE_DIR, 'logs')): + os.makedirs(os.path.join(BASE_DIR, 'logs')) + + +# JWT Token Decode DIR +JWT_TOKEN_DECODE_DIR = os.path.join(BASE_DIR, 'lib', 'decode') +if not os.path.exists(JWT_TOKEN_DECODE_DIR): + os.makedirs(JWT_TOKEN_DECODE_DIR) + +# 格式:[2020-04-22 23:33:01][micoservice.apps.ready():16] [INFO] 这是一条日志: +# 格式:[日期][模块.函数名称():行号] [级别] 信息 +STANDARD_LOG_FORMAT = '[%(levelname).4s] -- %(asctime)s -- P_%(process) -- d_T_%(thread)d ' \ + '- <%(module)s:%(lineno)d>: %(message)s' +CONSOLE_LOG_FORMAT = '[%(levelname).4s] -- %(asctime)s -- P_%(process) -- d_T_%(thread)d ' \ + '- <%(module)s:%(lineno)d>: %(message)s' + +LOGGING = { + 'version': 1, + 'disable_existing_loggers': False, + 'formatters': { + 'standard': { + 'format': STANDARD_LOG_FORMAT + }, + 'console': { + 'format': CONSOLE_LOG_FORMAT, + 'datefmt': '%Y-%m-%d %H:%M:%S', + }, + 'file': { + 'format': CONSOLE_LOG_FORMAT, + 'datefmt': '%Y-%m-%d %H:%M:%S', + }, + }, + 'handlers': { + 'file': { + 'level': 'INFO', + 'class': 'logging.handlers.RotatingFileHandler', + 'filename': SERVER_LOGS_FILE, + 'maxBytes': 1024 * 1024 * 100, # 100 MB + 'backupCount': 5, # 最多备份5个 + 'formatter': 'standard', + 'encoding': 'utf-8', + }, + 'error': { + 'level': 'ERROR', + 'class': 'logging.handlers.RotatingFileHandler', + 'filename': ERROR_LOGS_FILE, + 'maxBytes': 1024 * 1024 * 100, # 100 MB + 'backupCount': 3, # 最多备份3个 + 'formatter': 'standard', + 'encoding': 'utf-8', + }, + 'console': { + 'level': 'INFO', + 'class': 'logging.StreamHandler', + 'formatter': 'console', + } + }, + 'loggers': { + # default日志 + '': { + 'handlers': ['console', 'error', 'file'], + 'level': 'INFO', + }, + 'django': { + 'handlers': ['console', 'error', 'file'], + 'level': 'INFO', + }, + 'scripts': { + 'handlers': ['console', 'error', 'file'], + 'level': 'INFO', + }, + # 数据库相关日志 + 'django.db.backends': { + 'handlers': [], + 'propagate': True, + 'level': 'INFO', + }, + } +} diff --git a/sysom_server/sysom_channel/conf/develop.py b/sysom_server/sysom_channel/conf/develop.py new file mode 100644 index 00000000..55a9b74d --- /dev/null +++ b/sysom_server/sysom_channel/conf/develop.py @@ -0,0 +1,7 @@ +from .common import * + +''' +开发环境配置项 +''' + +DEBUG = True \ No newline at end of file diff --git a/sysom_server/sysom_channel/conf/product.py b/sysom_server/sysom_channel/conf/product.py new file mode 100644 index 00000000..d286e370 --- /dev/null +++ b/sysom_server/sysom_channel/conf/product.py @@ -0,0 +1,7 @@ +from .common import * + +''' +生产环境配置项 +''' + +DEBUG = False diff --git a/sysom_server/sysom_channel/conf/ssh/sysom_id b/sysom_server/sysom_channel/conf/ssh/sysom_id new file mode 100644 index 00000000..d57fb701 --- /dev/null +++ b/sysom_server/sysom_channel/conf/ssh/sysom_id @@ -0,0 +1,27 @@ +-----BEGIN RSA PRIVATE KEY----- +MIIEpAIBAAKCAQEA2M3cr/EcY4I3Li7BMUUkYyC6+4psKnegWnjo4rbU56N6K9VY +njFUT8a3BOJghjJNUbMj6xax6VIEHuRvS2GjuUPdfluAkdNLeYE1H3CAlukj+zHE +6ycvJ6Fk1kVS7+k3CE51QAdniKLAE6jYQn8N+NNcPibvlT1zRuSFQKaJaTjn30yY +itY+XO/Twd3h1bZ/N1GmrkYdZuudU8aqK3tqEApUXnAwXtS2jOzzSHMwN77UOZF8 +PFfbxzKZczFCtObScvHzs/v/mwCydABgfETK/arvkEyNucrpGo3I4eL7xAlU5xna +0n2yoeqyhn6Nw4Ra3Jd+6gH9zdIPmfRpBIvTxwIDAQABAoIBAQCB4HEABVSYV/1u +cRvSxDTUHVtmLLo3hopNKHl6t0Q7r7cUYy4/NhXn5IbnKiXgvmmLM+Gkh2f0+ej2 +CKqTYd6OsZUSiQYwVX+1Pi3ws5iUzv17EYmBxWt+NjUOF2FjzzXkh/u9a7IeLEEc +hd1nmG1qTlPH99M2eCWCposc+6b9nLsmABNZR1fh9xPxsEEHRKov2EgnUBqBHG+U +uH4Ik2XtBlaEZsKGxvxxaeNhqt4DykDd9eMSr8c01Mk5L+d2Mb0YWOXNafiYIhZq +316n1AYu3t0oRXk2Xx36n2w14R+IuyFBgtheRZOhrnJZkRl8LZlEdofxF6HdkIJc +q0Dah/phAoGBAO01KBn5xWLUC/z0DH4Xvc3LWM9B8/VXWSMRuNM75qmk85NcEZ64 +ll1AfVBKkrz5wBTCCT5ZDNUu3I/Zd4DOiS2TNFQ/Asqe++2tX/Bu2Wzu5BjZnlkJ +YM/9df+YDXFQZL6Lg4HGBGojfHVGXXtXBzvDHSBc6Xd3SE8SURuqR1klAoGBAOn6 +5iyooHyInCAx+TnbTaJlGKLX6OAktLgrn2NiIE2MUPke+T7NH2uHFJp94AjwRHZU +gvC0t5hQbTYi9uqt46kjTrtm/qUm1CJajh4olHXaKVN/f7DlJrLVLJuVVZjI97ML +W1ZAKBoZD3SE8EBszsbFSPC4V+l++gMBquHa2VN7AoGAAiHBK4KYKIw5U1uqJRP3 +i7rxIGO6WpEfZvZIz/Uw3IN0APYJ3oBQMaRbu/X/nZ0tAcLArIPGXsAeEVjl/w1c +JEG3ba5nSwTkxL0TKKkGYTlAaYzDJI+2BAiGH/bw3/zV+vRsCSWLcpzBCJjKTEp6 +8SmOwHS1UHKM/tDuU65TF9UCgYAeWapDvrfaj4nY70KJy7WdTAnPDbyqXniGrznk +lVTlWj8ioW7pQ4hA1WvJwmwuHKR0yO+XM4nhGwWfkReAR4ARb4AF9Q0w9QFtV1u5 +87+WLe9QYmh41LCZX+eCYvOQdvkSY3W8kWPiRr/neFsFsFd+FxcoCnF020Co5S16 +UH4CQwKBgQDRI9ObCqejlr3pSm6uAPb1341QgIWz5Ymy5sF71tgiCpn7XQG9GuUs +NUkcVuljJYVcQeJazRwCucAe2IBjwzWC4+2vP9Q9TINU+NSAuSzNqepAKyh1WuUV +Al0CKJeXsBBFfPODxeERc2WH+jONCTXvBl1RL48veh0/IXdotck0iA== +-----END RSA PRIVATE KEY----- diff --git a/sysom_server/sysom_channel/conf/ssh/sysom_id.pub b/sysom_server/sysom_channel/conf/ssh/sysom_id.pub new file mode 100644 index 00000000..adc75ce8 --- /dev/null +++ b/sysom_server/sysom_channel/conf/ssh/sysom_id.pub @@ -0,0 +1 @@ +ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDYzdyv8RxjgjcuLsExRSRjILr7imwqd6BaeOjittTno3or1VieMVRPxrcE4mCGMk1RsyPrFrHpUgQe5G9LYaO5Q91+W4CR00t5gTUfcICW6SP7McTrJy8noWTWRVLv6TcITnVAB2eIosATqNhCfw3401w+Ju+VPXNG5IVApolpOOffTJiK1j5c79PB3eHVtn83UaauRh1m651Txqore2oQClRecDBe1LaM7PNIczA3vtQ5kXw8V9vHMplzMUK05tJy8fOz+/+bALJ0AGB8RMr9qu+QTI25yukajcjh4vvECVTnGdrSfbKh6rKGfo3DhFrcl37qAf3N0g+Z9GkEi9PH \ No newline at end of file diff --git a/sysom_server/sysom_channel/conf/testing.py b/sysom_server/sysom_channel/conf/testing.py new file mode 100644 index 00000000..3039c80c --- /dev/null +++ b/sysom_server/sysom_channel/conf/testing.py @@ -0,0 +1,6 @@ +from .common import * + +''' +测试环境配置项 +''' +DEBUG = True diff --git a/sysom_server/sysom_channel/lib/base_model.py b/sysom_server/sysom_channel/lib/base_model.py new file mode 100644 index 00000000..863123d4 --- /dev/null +++ b/sysom_server/sysom_channel/lib/base_model.py @@ -0,0 +1,20 @@ +# -*- encoding: utf-8 -*- +""" +@File : base_model.py +@Time : 2021/10/28 11:04 +@Author : DM +@Email : smmic@isoftstone.com +@Software: PyCharm +""" + +from django.db import models +from lib.utils import human_datetime + + +class BaseModel(models.Model): + """abstract model""" + created_at = models.CharField(max_length=20, default=human_datetime, verbose_name="创建时间") + updated_at = models.CharField(max_length=20, default=human_datetime, verbose_name="更新时间") + + class Meta: + abstract = True diff --git a/sysom_server/sysom_channel/lib/base_view.py b/sysom_server/sysom_channel/lib/base_view.py new file mode 100644 index 00000000..36172850 --- /dev/null +++ b/sysom_server/sysom_channel/lib/base_view.py @@ -0,0 +1,90 @@ +# -*- coding: utf-8 -*- # +""" +Time 2022/10/21 10:08 +Author: mingfeng (SunnyQjm) +Email mfeng@linux.alibaba.com +File admin_static.py +Description: +""" +from typing import Any +from rest_framework.viewsets import GenericViewSet +from rest_framework.exceptions import ValidationError +from cec_base.cec_client import CecClient +from django.conf import settings + + +class CommonModelViewSet(GenericViewSet): + """ + 通用 ModelViewSet 实现,提供一些通用工具方法 + """ + + def __init__(self, **kwargs: Any) -> None: + super().__init__(**kwargs) + self._inner_cec_client: CecClient = None + + def produce_event_to_cec(self, topic: str, value: dict): + """Produce one event to specific topic""" + if self._inner_cec_client is None: + self._inner_cec_client = CecClient(settings.SYSOM_CEC_URL) + self._inner_cec_client.delivery(topic, value) + + def require_param_validate(self, request, require_params): + """ + 检查请求参数中是否包含所需的所有参数,任何参数缺失将返回错误,错误信息中包含缺失的参数列表 + @param request => HTTP 请求 + @param require_params => 需要验证的参数列表,例如:['hostname', 'ip'] + """ + if request.method in ['POST', 'PATCH', 'DELETE']: + context, missing_param_list = request.data, [] + for require_param in require_params: + if require_param not in context: + missing_param_list.append(require_param) + if len(missing_param_list) > 0: + return { + "success": False, + "message": f"Missing parameters: {', '.join(missing_param_list)}" + } + return { + "success": True, + "message": "" + } + + def extract_specific_params(self, request, params_list, require_all: bool = False): + """ + 从请求参数中提取 params_list 中指定的参数列表 => 最后将不在参数列表中的所有其它参数从参数列表中移除 + @param request => HTTP 请求 + @param params_list => 需要提取的参数列表,例如:['cluster'] + @param require_all => 是否所有参数都是必须的 + True => 任何参数缺失都会返回错误 + False => 对于每一个参数,有则提取,无则忽略 + """ + if require_all: + res = self.require_param_validate(request, params_list) + if not res["success"]: + return res + if request.method in ['POST', 'PATCH', 'DELETE']: + context, not_allow_params = request.data, [] + for key in context: + if key not in params_list: + not_allow_params.append(key) + for param_name in not_allow_params: + context.pop(param_name) + + return { + "success": True, + "message": "", + } + + def get_format_err_msg_for_validation_error(self, data: dict, err: ValidationError) -> str: + """ + 对验证错误的错误信息进行转换,使得提示更易读懂 + """ + results = [] + for k, v in err.detail.items(): + if v[0].code == 'unique': + # 唯一性错误 + results.append(f"{k}({data[k]})已存在") + else: + # 其它错误 + results.append(v[0]) + return "; ".join(results) diff --git a/sysom_api/conf/__init__.py b/sysom_server/sysom_channel/lib/channels/__init__.py similarity index 100% rename from sysom_api/conf/__init__.py rename to sysom_server/sysom_channel/lib/channels/__init__.py diff --git a/sysom_server/sysom_channel/lib/channels/base.py b/sysom_server/sysom_channel/lib/channels/base.py new file mode 100644 index 00000000..b01ab152 --- /dev/null +++ b/sysom_server/sysom_channel/lib/channels/base.py @@ -0,0 +1,31 @@ +""" +通道Base + +多通道是以单文件的方式构成,文件名就是通道名称(例如: ssh.py 为ssh通道), 通道 +文件中实现Channel类, 继承BaseChannel类, 必须实现client方法, run_command方法 +""" +from abc import ABCMeta, abstractmethod + + +class ChannelException(Exception): + def __init__(self, message: str) -> None: + self.message = message + + def __str__(self) -> str: + return self.message + + +class BaseChannel(metaclass=ABCMeta): + + @abstractmethod + def client(self, **kwargs): + raise NotImplemented + + @staticmethod + @abstractmethod + def initial(**kwargs): + raise NotImplemented + + @abstractmethod + def run_command(self, **kwargs): + raise NotImplemented diff --git a/sysom_server/sysom_channel/lib/channels/ssh.py b/sysom_server/sysom_channel/lib/channels/ssh.py new file mode 100644 index 00000000..0934ca31 --- /dev/null +++ b/sysom_server/sysom_channel/lib/channels/ssh.py @@ -0,0 +1,62 @@ +''' +@File: ssh.py +@Time: 2022-08-30 11:13:55 +@Author: DM +@Email: wb-msm261421@alibaba-inc.com +@Desc: ssh通道 +''' + +# SSH args eg: +# "channel": "ssh", 选填 (默认ssh) +# "instance": "xxxxxxxx", 必填 +# "cmd": "xxxx" 必填 + + +import logging +from lib.utils import valid_params +from lib.ssh import SSH + +from .base import BaseChannel, ChannelException + + +logger = logging.getLogger(__name__) + + +class Channel(BaseChannel): + FIELDS = ('instance', 'command') + + def __init__(self, *args, **kwargs) -> None: + self.kwargs = kwargs + self.ssh = None + self.shell_script = None + + self.validate_kwargs() + + def validate_kwargs(self): + for item in filter( + lambda x: not x[1], [(field, self.kwargs.get(field, None)) + for field in self.FIELDS] + ): + raise Exception(f'parameter: {item[0]} not found!') + + if not self.ssh: + self.ssh = SSH(hostname=self.kwargs['instance']) + self.shell_script = self.kwargs['command'] + + @staticmethod + def initial(**kwargs) -> bool: + valid_result = valid_params(["instance", "password"], kwargs) + if len(valid_result) == 0: + host, password = kwargs.pop("instance"), kwargs.pop("password") + success, err_msg = SSH.validate_ssh_host(host, password, **kwargs) + if not success: + raise ChannelException(err_msg) + else: + raise ChannelException(f"Missing parameters: {', '.join(valid_result)}") + return 0, "" + + def client(self): + return self.ssh if self.ssh else SSH(hostname=self.kwargs['instance']) + + def run_command(self): + return self.ssh.run_command(self.shell_script) diff --git a/sysom_server/sysom_channel/lib/exception.py b/sysom_server/sysom_channel/lib/exception.py new file mode 100644 index 00000000..25aa22ea --- /dev/null +++ b/sysom_server/sysom_channel/lib/exception.py @@ -0,0 +1,63 @@ +import logging +import traceback + +from django.db.models import ProtectedError +from rest_framework.views import set_rollback +from rest_framework import exceptions +from rest_framework.exceptions import APIException as DRFAPIException, AuthenticationFailed, NotAuthenticated + +from .response import ErrorResponse + + +logger = logging.getLogger(__name__) + + +class APIException(Exception): + def __init__(self, code=400, message='API异常', args=('API异常',)): + self.code = code + self.message = message + self.args = args + + def __str__(self): + return self.message + + +class FileNotFoundException(Exception): + def __init__(self, code=404, message='文件不存在'): + self.code = code + self.message = message + + def __str__(self): + return self.message + + +def exception_handler(exc, context): + """自定义异常处理""" + msg = '' + code = 400 + + if isinstance(exc, FileNotFoundException): + code = exc.code + msg = exc.message + if isinstance(exc, AuthenticationFailed): + code = 403 + msg = exc.detail + elif isinstance(exc, NotAuthenticated): + code = 402 + msg = exc.detail + elif isinstance(exc, DRFAPIException): + set_rollback() + # print(exc.detail) + # msg = {str(e) for e in exc.detail} + msg = exc.detail + elif isinstance(exc, exceptions.APIException): + set_rollback() + msg = exc.detail + elif isinstance(exc, ProtectedError): + set_rollback() + msg = "删除失败:该条数据与其他数据有相关绑定" + elif isinstance(exc, Exception): + logger.error(traceback.format_exc()) + msg = str(exc) # 原样输出错误 + + return ErrorResponse(msg=msg, code=code, status=code) diff --git a/sysom_server/sysom_channel/lib/paginations.py b/sysom_server/sysom_channel/lib/paginations.py new file mode 100644 index 00000000..b14639bb --- /dev/null +++ b/sysom_server/sysom_channel/lib/paginations.py @@ -0,0 +1,26 @@ +''' +@File: paginations.py +@Time: 2021-12-14 13:46:02 +@Author: DM +@Desc: Local Paginations Class +''' + +from rest_framework.pagination import PageNumberPagination +from lib.response import success + + +class Pagination(PageNumberPagination): + page_query_param = "current" + page_size_query_param = "pageSize" + + def paginate_queryset(self, queryset, request, view=None): + self.max_page_size = queryset.count() + return super().paginate_queryset(queryset, request, view=view) + + def get_paginated_response(self, data): + return success(message="获取成功", result=data, total=self.page.paginator.count) + + def get_page_size(self, request): + if not request.query_params.get(self.page_size_query_param, None): + return self.max_page_size + return super().get_page_size(request) \ No newline at end of file diff --git a/sysom_server/sysom_channel/lib/renderers.py b/sysom_server/sysom_channel/lib/renderers.py new file mode 100644 index 00000000..5e92ce82 --- /dev/null +++ b/sysom_server/sysom_channel/lib/renderers.py @@ -0,0 +1,42 @@ +import logging +from django.contrib.auth import get_user_model +from django.shortcuts import get_object_or_404 +from django.core.handlers.asgi import ASGIRequest +from rest_framework.renderers import JSONRenderer +from rest_framework.request import Request + + +logger = logging.getLogger(__name__) +User = get_user_model() + + +class SysomJsonRender(JSONRenderer): + def render(self, data, accepted_media_type=None, renderer_context=None): + if renderer_context: + request = renderer_context.get('request', None) + view = renderer_context.get('view', None) + response = renderer_context.get('response', None) + self.before_response_save_log(request, view, response) + return super().render(data, accepted_media_type, renderer_context) + + def before_response_save_log(self, request: Request, view, response): + user = getattr(request, 'user') or get_object_or_404(User, pk=1) + request: ASGIRequest = getattr(request, '_request', None) + method = request.method + + result = response.data + kwargs = { + 'request_ip': request.META.get('REMOTE_ADDR', None), + 'request_url': request.path, + 'request_browser_agent': request.headers.get('User-Agent', ''), + 'request_method': method, + 'handler_view': view.__class__.__name__, + 'response_status': getattr(response, 'status_code', 200), + } + if 'auth' in request.path: + kwargs['request_option'] = 0 + if result.get('code') == 200: + kwargs['user_id'] = result['data']['id'] + else: + kwargs['request_option'] = 1 + kwargs['user'] = user diff --git a/sysom_server/sysom_channel/lib/response.py b/sysom_server/sysom_channel/lib/response.py new file mode 100644 index 00000000..a2311a42 --- /dev/null +++ b/sysom_server/sysom_channel/lib/response.py @@ -0,0 +1,68 @@ +from rest_framework.response import Response +from rest_framework import status +from django.http import FileResponse + + +def _response(data=None, status=None): + return Response(data=data, status=status) + + +def success(result, message="success", success=True, code=status.HTTP_200_OK, **kwargs): + data = { + "code": code, + "message": message, + "data": result, + "success": success + } + data.update(kwargs) + return _response(data=data, status=code) + + +def not_found(code=status.HTTP_404_NOT_FOUND, success=False, message="Not Found"): + data = { + "code": code, + "message": message, + "success": success, + } + + return _response(data=data, status=code) + + +def not_permission(code=status.HTTP_403_FORBIDDEN, success=False, message="Not Permission"): + data = { + "code": code, + "success": success, + "message": message + } + return _response(data=data, status=code) + + +def other_response(result=dict(), message="", success=True, code=status.HTTP_200_OK, **kwargs): + data = { + "code": code, + "message": message, + "data": result, + "success": success + } + data.update(kwargs) + return _response(data=data, status=code) + + +class ErrorResponse(Response): + """ + 标准响应错误的返回,ErrorResponse(msg='xxx') + 默认错误码返回400, 也可以指定其他返回码:ErrorResponse(code=xxx) + """ + + def __init__(self, data=None, msg='error', code=400, status=None, template_name=None, headers=None, + exception=False, content_type=None): + std_data = { + "code": code, + "data": data or {}, + "message": msg + } + super().__init__(std_data, status, template_name, headers, exception, content_type) + + +class FileResponseAlter(FileResponse): + pass diff --git a/sysom_server/sysom_channel/lib/ssh.py b/sysom_server/sysom_channel/lib/ssh.py new file mode 100644 index 00000000..9de53fc4 --- /dev/null +++ b/sysom_server/sysom_channel/lib/ssh.py @@ -0,0 +1,100 @@ +import logging +from typing import Callable +import paramiko +from io import StringIO +from paramiko.client import SSHClient, AutoAddPolicy +from paramiko.rsakey import RSAKey + +DEFAULT_CONNENT_TIMEOUT = 5 # 默认ssh链接超时时间 5s +DEFAULT_NODE_USER = 'root' # 默认节点用户名 root + +logger = logging.getLogger(__name__) + + +class SSH: + """A SSH client used to run command in remote node + + args: + hostname(str): Host name + + Keyword Args: + username(str): User name, default 'root' + port(str): SSH communicate port, default 22 + connect_timeout(int): Connection timeout duration, default 5s + password(str) + """ + + # key_pair cached the key pair generated by initialization stage + _key_pair = {} + _private_key_getter: Callable[[], str] = None + _public_key_getter: Callable[[], str] = None + + def __init__(self, hostname: str, **kwargs) -> None: + self.connect_args = { + 'hostname': hostname, + 'username': kwargs.get('username', DEFAULT_NODE_USER), + 'port': kwargs.get('port', 22), + 'timeout': kwargs.get('timeout', DEFAULT_CONNENT_TIMEOUT), + } + if 'password' in kwargs and kwargs['password'] is not None: + self.connect_args['password'] = kwargs.get('password') + else: + if SSH._private_key_getter is None: + raise Exception("_private_key_getter not set") + self.connect_args['pkey'] = RSAKey.from_private_key( + StringIO(SSH._private_key_getter()) + ) + + self._client: SSHClient = self.client() + + def client(self): + try: + client = SSHClient() + client.set_missing_host_key_policy(AutoAddPolicy) + client.connect(**self.connect_args) + return client + except paramiko.AuthenticationException: + raise Exception('authorization fail, password or pkey error!') + except: + raise Exception('authorization fail!') + + @classmethod + def set_private_key_getter(cls, private_key_getter: Callable[[], str]): + cls._private_key_getter = private_key_getter + + @classmethod + def set_public_key_getter(cls, public_key_getter: Callable[[], str]): + cls._public_key_getter = public_key_getter + + def run_command(self, command): + if self._client: + ssh_session = self._client.get_transport().open_session() + ssh_session.set_combine_stderr(True) + ssh_session.exec_command(command) + stdout = ssh_session.makefile("rb", -1) + statue = ssh_session.recv_exit_status() + output = stdout.read().decode() + return statue, output + else: + raise Exception('No client!') + + def add_public_key(self): + if self._public_key_getter is None: + raise Exception("_public_key_getter not set") + public_key = SSH._public_key_getter() + command = f'mkdir -p -m 700 ~/.ssh && \ + echo {public_key!r} >> ~/.ssh/authorized_keys && \ + chmod 600 ~/.ssh/authorized_keys' + statue, _ = self.run_command(command) + if statue != 0: + raise Exception('add public key faild!') + + @staticmethod + def validate_ssh_host(ip: str, password: str, port: int = 22, username: str = 'root'): + try: + ssh = SSH(hostname=ip, password=password, + port=port, username=username, timeout=2) + ssh.add_public_key() + return True, 'authorization success' + except Exception as e: + return False, f'error: {e}' diff --git a/sysom_server/sysom_channel/lib/utils.py b/sysom_server/sysom_channel/lib/utils.py new file mode 100644 index 00000000..be13cc4a --- /dev/null +++ b/sysom_server/sysom_channel/lib/utils.py @@ -0,0 +1,78 @@ + +# -*- encoding: utf-8 -*- +""" +@File : utils.py +@Time : 2021/10/28 11:09 +@Author : DM +@Software: PyCharm +""" +from typing import List +import uuid as UUID +import logging +from datetime import datetime +from paramiko.rsakey import RSAKey +from io import StringIO + + +logger = logging.getLogger(__name__) + + +CHAR_SET = ("a", "b", "c", "d", "e", "f", + "g", "h", "i", "j", "k", "l", "m", "n", "o", "p", "q", "r", "s", + "t", "u", "v", "w", "x", "y", "z", "0", "1", "2", "3", "4", "5", + "6", "7", "8", "9", "A", "B", "C", "D", "E", "F", "G", "H", "I", + "J", "K", "L", "M", "N", "O", "P", "Q", "R", "S", "T", "U", "V", + "W", "X", "Y", "Z") + + +def human_datetime(date=None): + if date: + assert isinstance(date, datetime) + else: + date = datetime.now() + return date.strftime('%Y-%m-%d %H:%M:%S') + + +def uuid_36(): + """ + 返回36字符的UUID字符串(十六进制,含有-) bc5debab-95c3-4430-933f-2e3b6407ac30 + :return: + """ + return str(UUID.uuid4()) + + +def uuid_32(): + """ + 返回32字符的UUID字符串(十六进制) bc5debab95c34430933f2e3b6407ac30 + :return: + """ + return uuid_36().replace('-', '') + + +def uuid_8(): + """ + 返回8字符的UUID字符串(非进制) 3FNWjtlD + :return: + """ + s = uuid_32() + result = '' + for i in range(0, 8): + sub = s[i * 4: i * 4 + 4] + x = int(sub, 16) + result += CHAR_SET[x % 0x3E] + return result + + +def generate_key(): + key_obj = StringIO() + key = RSAKey.generate(2048) + key.write_private_key(key_obj) + return key_obj.getvalue(), 'ssh-rsa ' + key.get_base64() + + +def valid_params(require_params: dict, current_params: dict) -> List[str]: + missing_param_list = [] + for param in require_params: + if param not in current_params: + missing_param_list.append(param) + return missing_param_list diff --git a/sysom_server/sysom_channel/manage.py b/sysom_server/sysom_channel/manage.py new file mode 100755 index 00000000..d545e94b --- /dev/null +++ b/sysom_server/sysom_channel/manage.py @@ -0,0 +1,22 @@ +#!/usr/bin/env python +"""Django's command-line utility for administrative tasks.""" +import os +import sys + + +def main(): + """Run administrative tasks.""" + os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'sysom_channel.settings') + try: + from django.core.management import execute_from_command_line + except ImportError as exc: + raise ImportError( + "Couldn't import Django. Are you sure it's installed and " + "available on your PYTHONPATH environment variable? Did you " + "forget to activate a virtual environment?" + ) from exc + execute_from_command_line(sys.argv) + + +if __name__ == '__main__': + main() diff --git a/sysom_server/sysom_channel/sysom_channel/__init__.py b/sysom_server/sysom_channel/sysom_channel/__init__.py new file mode 100644 index 00000000..aa60bed8 --- /dev/null +++ b/sysom_server/sysom_channel/sysom_channel/__init__.py @@ -0,0 +1,3 @@ +import pymysql + +pymysql.install_as_MySQLdb() \ No newline at end of file diff --git a/sysom_server/sysom_channel/sysom_channel/asgi.py b/sysom_server/sysom_channel/sysom_channel/asgi.py new file mode 100644 index 00000000..c6cd4e69 --- /dev/null +++ b/sysom_server/sysom_channel/sysom_channel/asgi.py @@ -0,0 +1,16 @@ +""" +ASGI config for sysom_channel project. + +It exposes the ASGI callable as a module-level variable named ``application``. + +For more information on this file, see +https://docs.djangoproject.com/en/3.2/howto/deployment/asgi/ +""" + +import os + +from django.core.asgi import get_asgi_application + +os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'sysom_channel.settings') + +application = get_asgi_application() diff --git a/sysom_server/sysom_channel/sysom_channel/settings.py b/sysom_server/sysom_channel/sysom_channel/settings.py new file mode 100644 index 00000000..1a5ff399 --- /dev/null +++ b/sysom_server/sysom_channel/sysom_channel/settings.py @@ -0,0 +1,15 @@ +import os + +env = os.environ.get("env", "product") + + +if env == "develop": + from conf.develop import * +elif env == "testing": + from conf.testing import * +elif env == "product": + from conf.product import * + +# 跨域允许 +if DEBUG: + CORS_ORIGIN_ALLOW_ALL = True \ No newline at end of file diff --git a/sysom_server/sysom_channel/sysom_channel/urls.py b/sysom_server/sysom_channel/sysom_channel/urls.py new file mode 100644 index 00000000..f6c765a4 --- /dev/null +++ b/sysom_server/sysom_channel/sysom_channel/urls.py @@ -0,0 +1,26 @@ +"""sysom_channel URL Configuration + +The `urlpatterns` list routes URLs to views. For more information please see: + https://docs.djangoproject.com/en/3.2/topics/http/urls/ +Examples: +Function views + 1. Add an import: from my_app import views + 2. Add a URL to urlpatterns: path('', views.home, name='home') +Class-based views + 1. Add an import: from other_app.views import Home + 2. Add a URL to urlpatterns: path('', Home.as_view(), name='home') +Including another URLconf + 1. Import the include() function: from django.urls import include, path + 2. Add a URL to urlpatterns: path('blog/', include('blog.urls')) +""" +from django.urls import path +from django.urls.conf import include +from django.conf import settings + +app_urlpatterns = [path('', include( + f'{app}.urls')) for app in settings.INSTALLED_APPS if app.startswith('apps')] + +urlpatterns = [ +] + +urlpatterns += app_urlpatterns diff --git a/sysom_server/sysom_channel/sysom_channel/wsgi.py b/sysom_server/sysom_channel/sysom_channel/wsgi.py new file mode 100644 index 00000000..d1892ed3 --- /dev/null +++ b/sysom_server/sysom_channel/sysom_channel/wsgi.py @@ -0,0 +1,16 @@ +""" +WSGI config for sysom_channel project. + +It exposes the WSGI callable as a module-level variable named ``application``. + +For more information on this file, see +https://docs.djangoproject.com/en/3.2/howto/deployment/wsgi/ +""" + +import os + +from django.core.wsgi import get_wsgi_application + +os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'sysom_channel.settings') + +application = get_wsgi_application() diff --git a/sysom_api/lib/channels/__init__.py b/sysom_server/sysom_diagnosis/apps/task/__init__.py similarity index 100% rename from sysom_api/lib/channels/__init__.py rename to sysom_server/sysom_diagnosis/apps/task/__init__.py diff --git a/sysom_api/apps/vul/admin.py b/sysom_server/sysom_diagnosis/apps/task/admin.py similarity index 100% rename from sysom_api/apps/vul/admin.py rename to sysom_server/sysom_diagnosis/apps/task/admin.py diff --git a/sysom_api/apps/task/apps.py b/sysom_server/sysom_diagnosis/apps/task/apps.py similarity index 82% rename from sysom_api/apps/task/apps.py rename to sysom_server/sysom_diagnosis/apps/task/apps.py index be96bea0..35cc95c0 100644 --- a/sysom_api/apps/task/apps.py +++ b/sysom_server/sysom_diagnosis/apps/task/apps.py @@ -2,18 +2,17 @@ import logging import sys from django.apps import AppConfig - logger = logging.getLogger(__name__) -class JobConfig(AppConfig): +class TaskConfig(AppConfig): default_auto_field = 'django.db.models.BigAutoField' name = 'apps.task' def ready(self): from django.conf import settings - if ('runserver' in sys.argv or 'manage.py' not in sys.argv) and settings.IS_MICRO_SERVICES: - from sdk.cec_base.log import LoggerHelper, LoggerLevel + if ('runserver' in sys.argv or 'manage.py' not in sys.argv): + from cec_base.log import LoggerHelper, LoggerLevel from apps.task.executors import TaskDispatcher LoggerHelper.update_sys_stdout_sink(LoggerLevel.LOGGER_LEVEL_INFO) # 这边微服务正式启动的时候执行一些处理代码 @@ -27,4 +26,4 @@ class JobConfig(AppConfig): else: # 这边执行数据库迁移等操作的时候执行一些处理代码 pass - logger.info(">>> Job module loading success") + logger.info(">>> Diagnosis module loading success") diff --git a/sysom_api/apps/task/executors.py b/sysom_server/sysom_diagnosis/apps/task/executors.py similarity index 72% rename from sysom_api/apps/task/executors.py rename to sysom_server/sysom_diagnosis/apps/task/executors.py index 79761729..89134407 100644 --- a/sysom_api/apps/task/executors.py +++ b/sysom_server/sysom_diagnosis/apps/task/executors.py @@ -1,6 +1,5 @@ import json import os -import socket import subprocess import logging import tempfile @@ -9,9 +8,10 @@ from apps.task.models import JobModel from django.conf import settings from django.db import connection from lib.utils import uuid_8 -from sdk.cec_base.cec_client import CecClient -from sdk.cec_base.event import Event -from sdk.cec_base.consumer import Consumer +from cec_base.cec_client import CecClient +from cec_base.event import Event +from cec_base.consumer import Consumer +from cec_base.producer import Producer from .seriaizer import JobDetailSerializer @@ -32,42 +32,87 @@ class TaskDispatcher(CecClient): def start_dispatcher(self): self.append_group_consume_task( - settings.SYSOM_CEC_TASK_RESULT_PROCESS_TOPIC, - settings.SYSOM_CEC_TASK_RESULT_PROCESS_GROUP, + settings.SYSOM_CEC_CHANNEL_RESULT_TOPIC, + settings.SYSOM_CEC_DIAGNOSIS_CONSUMER_GROUP, Consumer.generate_consumer_id(), ensure_topic_exist=True ) self.append_group_consume_task( - settings.SYSOM_CEC_TASK_GENERATE_TOPIC, - settings.SYSOM_CEC_TASK_GENERATE_TASK_LISTENER, + settings.SYSOM_CEC_PLUGIN_TOPIC, + settings.SYSOM_CEC_DIAGNOSIS_CONSUMER_GROUP, Consumer.generate_consumer_id(), ensure_topic_exist=True ) self.start() - def on_receive_event(self, consumer: Consumer, event: Event, task: dict): + def on_receive_event(self, consumer: Consumer, producer: Producer, event: Event, task: dict): try: topic_name = task.get("topic_name", "unknown_topic") - if topic_name == settings.SYSOM_CEC_TASK_RESULT_PROCESS_TOPIC: + if topic_name == settings.SYSOM_CEC_CHANNEL_RESULT_TOPIC: # Deal task process result self._process_task_result(event) - elif topic_name == settings.SYSOM_CEC_TASK_GENERATE_TOPIC: - # Deal generate task - self._generate_task(event) + elif topic_name == settings.SYSOM_CEC_PLUGIN_TOPIC: + self._process_plugin_event(event) + # elif topic_name == settings.SYSOM_CEC_TASK_GENERATE_TOPIC: + # # Deal generate task + # self._generate_task(event) except Exception as exc: logger.exception(exc) finally: consumer.ack(event) + ################################################################################################ + # 插件系统相关事件处理 + ################################################################################################ + def _process_plugin_event(self, event: Event): + """Process plugin event + + { + "type": "clean", + "params": { + "channel": "ssh", + "host": instance.ip, + "username": instance.username, + "port": instance.port + }, + "echo": { + "instance": params.get("host", "Unknown host"), + "label": "host_init" + } + } + """ + value = event.value + plugin_event_type = value.get("type", "Unknown type") + if plugin_event_type == "init": + params = value.get("params", {}) + token = params.pop("token", "") + self._generate_task({ + "token": token, + "data": { + **params, + "service_name": "node_init" + } + }) + elif plugin_event_type == "clean": + params = value.get("params", {}) + token = params.pop("token", "") + self._generate_task({ + "token": token, + "data": { + **params, + "service_name": "node_delete" + } + }) + ################################################################################################ # 诊断任务下发 ################################################################################################ - def _generate_task(self, event: Event): + def _generate_task(self, params: dict): from lib.authentications import decode_token try: - token = event.value.get("token", "") - data = event.value.get("data", {}) + token = params.get("token", "") + data = params.get("data", {}) data["user"] = decode_token(token) self.delivery_task(data) except Exception as exc: @@ -152,10 +197,18 @@ class TaskDispatcher(CecClient): JobModel.objects.create(**task) except: raise Exception('任务创建失败!') + # 将任务下发到事件中心,异步执行 - self.delivery(settings.SYSOM_CEC_TASK_DELIVERY_TOPIC, { - **task, - "command": resp_scripts + self.delivery(settings.SYSOM_CEC_CHANNEL_TOPIC, { + "channel": data.pop("channel", "ssh"), + "type": "cmd", + "params": { + **data, + "command": resp_scripts[0].get("cmd", "Unknown") + }, + "echo": { + "task_id": task_id + } }) # 任务创建成功,返回任务ID return { @@ -177,12 +230,12 @@ class TaskDispatcher(CecClient): """ 处理任务执行结果回调 { - "status": 0, => 0 表示成功,1表示失败 - "task_id": "xxx", => 任务对应的 Task 实例 - "errMsg": "", => 如果任务执行失败,错误信息可以使用这个字段获得 - "results": [ => 单个任务里面可能包含多个命令,这个字段里面包含了所有命令的执行结果 - "" - ] + "code": 0, => 0 表示成功,1表示失败 + "err_msg": "", => 如果任务执行失败,错误信息可以使用这个字段获得 + "result": "xxx", => 命令执行结果 + "echo": { + "task_id": 0 + } } """ def update_job(instance: JobModel, **kwargs): @@ -194,16 +247,21 @@ class TaskDispatcher(CecClient): finally: connection.close() task_result = event.value - task_id, status, err_msg, results = task_result.get("task_id", ""), task_result.get( - "status", 1), task_result.get("errMsg", ""), task_result.get("results") - result = "" if len(results) == 0 else results[-1] - service_name = "unknown" + code = task_result.get("code", 1) + err_msg = task_result.get("err_msg", "") + result = task_result.get("result", "") + echo = task_result.get("echo", {}) + if "task_id" not in echo or not echo.get("task_id"): + return + task = {} + instance = None try: + task_id = echo.get("task_id") # 获取到 Task 实例 instance = JobModel.objects.get(task_id=task_id) # 如果任务执行失败,更新状态 - if status != 0: + if code != 0: update_job(instance, status="Fail", result=err_msg) return # 如果任务执行成功,则执行后处理脚本 @@ -239,15 +297,15 @@ class TaskDispatcher(CecClient): # 后处理脚本执行结束,更新任务状态 update_job(instance, status="Success", result=result) except Exception as e: - status = 1 + code = 1 err_msg = str(e) logger.error(e) finally: if instance is not None: task = JobDetailSerializer(instance).data # 将诊断任务最后是否执行成功写到事件中心当中 - self.delivery(settings.SYSOM_CEC_TASK_EXECUTE_RESULT_TOPIC, { - "status": status, + self.delivery(settings.SYSOM_CEC_DIAGNOSIS_RESULT_TOPIC, { + "code": code, "err_msg": err_msg, "task": task }) diff --git a/sysom_api/apps/task/filter.py b/sysom_server/sysom_diagnosis/apps/task/filter.py similarity index 100% rename from sysom_api/apps/task/filter.py rename to sysom_server/sysom_diagnosis/apps/task/filter.py diff --git a/sysom_api/apps/task/models.py b/sysom_server/sysom_diagnosis/apps/task/models.py similarity index 100% rename from sysom_api/apps/task/models.py rename to sysom_server/sysom_diagnosis/apps/task/models.py diff --git a/sysom_server/sysom_diagnosis/apps/task/seriaizer.py b/sysom_server/sysom_diagnosis/apps/task/seriaizer.py new file mode 100644 index 00000000..7fd52a5f --- /dev/null +++ b/sysom_server/sysom_diagnosis/apps/task/seriaizer.py @@ -0,0 +1,27 @@ +# -*- encoding: utf-8 -*- +""" +@File : seriaizer.py +@Time : 2021/11/22 17:41 +@Author : DM +@Software: PyCharm +""" +from rest_framework import serializers +from apps.task.models import JobModel + + +class JobListSerializer(serializers.ModelSerializer): + class Meta: + model = JobModel + exclude = ('host_by', 'command', 'result') + + +class JobRetrieveSerializer(serializers.ModelSerializer): + class Meta: + model = JobModel + exclude = ('host_by', 'command') + + +class JobDetailSerializer(serializers.ModelSerializer): + class Meta: + model = JobModel + fields = '__all__' diff --git a/sysom_api/apps/vul/tests.py b/sysom_server/sysom_diagnosis/apps/task/tests.py similarity index 100% rename from sysom_api/apps/vul/tests.py rename to sysom_server/sysom_diagnosis/apps/task/tests.py diff --git a/sysom_api/apps/task/urls.py b/sysom_server/sysom_diagnosis/apps/task/urls.py similarity index 100% rename from sysom_api/apps/task/urls.py rename to sysom_server/sysom_diagnosis/apps/task/urls.py diff --git a/sysom_api/apps/task/views.py b/sysom_server/sysom_diagnosis/apps/task/views.py similarity index 92% rename from sysom_api/apps/task/views.py rename to sysom_server/sysom_diagnosis/apps/task/views.py index 9f66872a..9e553fd5 100644 --- a/sysom_api/apps/task/views.py +++ b/sysom_server/sysom_diagnosis/apps/task/views.py @@ -9,13 +9,12 @@ from django.conf import settings from apps.task import seriaizer from apps.task.models import JobModel from apps.task.filter import TaskFilter -from apps.common.common_model_viewset import CommonModelViewSet -from lib.response import success, other_response, not_found +from lib.base_view import CommonModelViewSet +from lib.response import success, other_response, not_found, ErrorResponse from lib.authentications import TaskAuthentication from .executors import TaskDispatcher logger = logging.getLogger(__name__) -IS_MICRO_SERVICES = settings.IS_MICRO_SERVICES class TaskAPIView(CommonModelViewSet, @@ -32,12 +31,13 @@ class TaskAPIView(CommonModelViewSet, filterset_class = TaskFilter # 精确查询 authentication_classes = [TaskAuthentication] create_requird_fields = ['service_name'] - + def __init__(self, **kwargs) -> None: super().__init__(**kwargs) # 构造任务执行器,并使用它来进行任务下发和结果处理(结果处理在 apps.py) # 【v2 基于事件中心接口相关功能】 - self._task_executor: TaskDispatcher = TaskDispatcher(settings.SYSOM_CEC_URL) + self._task_executor: TaskDispatcher = TaskDispatcher( + settings.SYSOM_CEC_URL) def get_authenticators(self): if self.request.path.endswith("svg/"): @@ -108,4 +108,6 @@ class TaskAPIView(CommonModelViewSet, """ res = self._task_executor.delivery_task(data) if res['success']: - return success(result=res["result"]) \ No newline at end of file + return success(result=res["result"]) + else: + return ErrorResponse(msg=res["message"]) diff --git a/sysom_server/sysom_diagnosis/conf/common.py b/sysom_server/sysom_diagnosis/conf/common.py new file mode 100644 index 00000000..0dceb24a --- /dev/null +++ b/sysom_server/sysom_diagnosis/conf/common.py @@ -0,0 +1,183 @@ +import os +import socket +from pathlib import Path + + +def get_ip_address(): + """ip address""" + hostname = socket.gethostname() + return socket.gethostbyname(hostname) + + +BASE_DIR = Path(__file__).resolve().parent.parent +SECRET_KEY = 'django-insecure-^d8b9di9w&-mmsbpt@)o#e+2^z+^m4nhf+z8304%9@8y#ko46l' + +ALLOWED_HOSTS = ['*'] + +INSTALLED_APPS = [ + 'apps.task', + + 'rest_framework', + 'corsheaders', + 'drf_yasg', # 在线API文档 + 'channels', + 'django_filters', + 'django_apscheduler', +] + +MIDDLEWARE = [ + 'corsheaders.middleware.CorsMiddleware', + 'django.middleware.security.SecurityMiddleware', + 'django.middleware.common.CommonMiddleware', +] + +DEBUG = True + +SCRIPTS_DIR = os.path.join(BASE_DIR, 'service_scripts') + +# Mysql数据库 +DATABASES = { + 'default': { + 'ENGINE': 'django.db.backends.mysql', + 'NAME': 'sysom', + 'USER': 'sysom', + 'PASSWORD': 'sysom_admin', + 'HOST': '127.0.0.1', + 'PORT': '3306', + } +} + +ROOT_URLCONF = 'sysom_diagnosis.urls' + +WSGI_APPLICATION = 'sysom_diagnosis.wsgi.application' +ASGI_APPLICATION = 'sysom_diagnosis.asgi.application' + +DEFAULT_AUTO_FIELD = 'django.db.models.BigAutoField' + +LANGUAGE_CODE = 'zh-hans' +TIME_ZONE = 'Asia/Shanghai' +USE_I18N = True +USE_L10N = True +USE_TZ = True +STATIC_URL = '/static/' +STATIC_ROOT = os.path.join(BASE_DIR, 'index/status') + +# rest_framework settings +REST_FRAMEWORK = { + 'DEFAULT_PERMISSION_CLASSES': ( + # 'rest_framework.permissions.IsAuthenticated' + ), + # 'DEFAULT_AUTHENTICATION_CLASSES': [ + # 'apps.accounts.authentication.Authentication' + # ], + 'UNAUTHENTICATED_USER': None, + 'DEFAULT_VERSIONING_CLASS': "rest_framework.versioning.URLPathVersioning", + 'DEFAULT_VERSION': 'v1', # 默认版本 + 'ALLOWED_VERSIONS': ['v1', 'v2'], # 允许的版本 + 'VERSION_PARAM': 'version', + + # 'DEFAULT_RENDERER_CLASSES': ( + # 'lib.renderers.SysomJsonRender', + # ), + 'DEFAULT_PAGINATION_CLASS': 'lib.paginations.Pagination', + 'UNICODE_JSON': True, + 'EXCEPTION_HANDLER': 'lib.exception.exception_handler' +} + +################################################################## +# Cec settings +################################################################## +SYSOM_CEC_URL = "redis://localhost:6379?cec_default_max_len=1000&cec_auto_mk_topic=true" +SYSOM_CEC_ALARM_TOPIC = "CEC-SYSOM-ALARM" +# 通道模块用于对外开放,投递操作的主题 +SYSOM_CEC_CHANNEL_TOPIC = "SYSOM_CEC_CHANNEL_TOPIC" +# 通道模块用于投递执行结果的主题 +SYSOM_CEC_CHANNEL_RESULT_TOPIC = "SYSOM_CEC_CHANNEL_RESULT_TOPIC" +# 诊断模块消费组 +SYSOM_CEC_DIAGNOSIS_CONSUMER_GROUP = "SYSOM_CEC_DIAGNOSIS_CONSUMER_GROUP" +# 诊断模块用于汇报最终诊断执行结果的主题 +SYSOM_CEC_DIAGNOSIS_RESULT_TOPIC = "SYSOM_CEC_DIAGNOSIS_RESULT_TOPIC" +# 用于分发插件系统相关事件的主题 +SYSOM_CEC_PLUGIN_TOPIC = "SYSOM_CEC_PLUGIN_TOPIC" + +SERVER_LOGS_FILE = os.path.join(BASE_DIR, 'logs', 'sys_om_info.log') +ERROR_LOGS_FILE = os.path.join(BASE_DIR, 'logs', 'sys_om_error.log') +if not os.path.exists(os.path.join(BASE_DIR, 'logs')): + os.makedirs(os.path.join(BASE_DIR, 'logs')) + + +# JWT Token Decode DIR +JWT_TOKEN_DECODE_DIR = os.path.join(BASE_DIR, 'lib', 'decode') +if not os.path.exists(JWT_TOKEN_DECODE_DIR): + os.makedirs(JWT_TOKEN_DECODE_DIR) + +# 格式:[2020-04-22 23:33:01][micoservice.apps.ready():16] [INFO] 这是一条日志: +# 格式:[日期][模块.函数名称():行号] [级别] 信息 +STANDARD_LOG_FORMAT = '[%(levelname).4s] -- %(asctime)s -- P_%(process) -- d_T_%(thread)d ' \ + '- <%(module)s:%(lineno)d>: %(message)s' +CONSOLE_LOG_FORMAT = '[%(levelname).4s] -- %(asctime)s -- P_%(process) -- d_T_%(thread)d ' \ + '- <%(module)s:%(lineno)d>: %(message)s' + +LOGGING = { + 'version': 1, + 'disable_existing_loggers': False, + 'formatters': { + 'standard': { + 'format': STANDARD_LOG_FORMAT + }, + 'console': { + 'format': CONSOLE_LOG_FORMAT, + 'datefmt': '%Y-%m-%d %H:%M:%S', + }, + 'file': { + 'format': CONSOLE_LOG_FORMAT, + 'datefmt': '%Y-%m-%d %H:%M:%S', + }, + }, + 'handlers': { + 'file': { + 'level': 'INFO', + 'class': 'logging.handlers.RotatingFileHandler', + 'filename': SERVER_LOGS_FILE, + 'maxBytes': 1024 * 1024 * 100, # 100 MB + 'backupCount': 5, # 最多备份5个 + 'formatter': 'standard', + 'encoding': 'utf-8', + }, + 'error': { + 'level': 'ERROR', + 'class': 'logging.handlers.RotatingFileHandler', + 'filename': ERROR_LOGS_FILE, + 'maxBytes': 1024 * 1024 * 100, # 100 MB + 'backupCount': 3, # 最多备份3个 + 'formatter': 'standard', + 'encoding': 'utf-8', + }, + 'console': { + 'level': 'INFO', + 'class': 'logging.StreamHandler', + 'formatter': 'console', + } + }, + 'loggers': { + # default日志 + '': { + 'handlers': ['console', 'error', 'file'], + 'level': 'INFO', + }, + 'django': { + 'handlers': ['console', 'error', 'file'], + 'level': 'INFO', + }, + 'scripts': { + 'handlers': ['console', 'error', 'file'], + 'level': 'INFO', + }, + # 数据库相关日志 + 'django.db.backends': { + 'handlers': [], + 'propagate': True, + 'level': 'INFO', + }, + } +} diff --git a/sysom_server/sysom_diagnosis/conf/develop.py b/sysom_server/sysom_diagnosis/conf/develop.py new file mode 100644 index 00000000..55a9b74d --- /dev/null +++ b/sysom_server/sysom_diagnosis/conf/develop.py @@ -0,0 +1,7 @@ +from .common import * + +''' +开发环境配置项 +''' + +DEBUG = True \ No newline at end of file diff --git a/sysom_api/conf/task_gunicorn.py b/sysom_server/sysom_diagnosis/conf/diagnosis_gunicorn.py similarity index 51% rename from sysom_api/conf/task_gunicorn.py rename to sysom_server/sysom_diagnosis/conf/diagnosis_gunicorn.py index 54a91cbc..e7af99d1 100644 --- a/sysom_api/conf/task_gunicorn.py +++ b/sysom_server/sysom_diagnosis/conf/diagnosis_gunicorn.py @@ -1,5 +1,5 @@ ''' -Task Service Gunicorn Settings +Diagnosis Service Gunicorn Settings ''' workers = 2 # 指定工作进程数 @@ -11,12 +11,12 @@ worker_class = 'gevent' # 工作模式线程, 默认为sync模式 max_requests = 2000 # 设置最大并发数量为2000 (每个worker处理请求的工作线程) -accesslog = '/usr/local/sysom/server/logs/task-service.log' +accesslog = '/usr/local/sysom/server/logs/diagnosis-service.log' -errorlog = '/usr/local/sysom/server/logs/task-service-error.log' +errorlog = '/usr/local/sysom/server/logs/diagnosis-service-error.log' loglevel = 'info' -proc_name = 'task_service' +proc_name = 'diagnosis_service' -raw_env = 'DJANGO_SETTINGS_MODULE=sysom.services_conf.task_settings' +raw_env = 'DJANGO_SETTINGS_MODULE=sysom_diagnosis.settings' diff --git a/sysom_server/sysom_diagnosis/conf/product.py b/sysom_server/sysom_diagnosis/conf/product.py new file mode 100644 index 00000000..d286e370 --- /dev/null +++ b/sysom_server/sysom_diagnosis/conf/product.py @@ -0,0 +1,7 @@ +from .common import * + +''' +生产环境配置项 +''' + +DEBUG = False diff --git a/sysom_server/sysom_diagnosis/conf/testing.py b/sysom_server/sysom_diagnosis/conf/testing.py new file mode 100644 index 00000000..3039c80c --- /dev/null +++ b/sysom_server/sysom_diagnosis/conf/testing.py @@ -0,0 +1,6 @@ +from .common import * + +''' +测试环境配置项 +''' +DEBUG = True diff --git a/sysom_server/sysom_diagnosis/lib/authentications.py b/sysom_server/sysom_diagnosis/lib/authentications.py new file mode 100644 index 00000000..e79c9172 --- /dev/null +++ b/sysom_server/sysom_diagnosis/lib/authentications.py @@ -0,0 +1,50 @@ +from importlib import import_module +import logging +import os +from typing import List +from django.conf import settings +from django.utils.translation import ugettext as _ +from rest_framework.exceptions import AuthenticationFailed +from rest_framework.request import Request +from rest_framework.authentication import BaseAuthentication + + +logger = logging.getLogger(__name__) + + +def get_jwt_decode_classes() -> List[BaseAuthentication]: + jwt_decode_classes = [] + import_strings = [ + f'lib.decode.{f.replace(".py", "")}' for f in os.listdir(settings.JWT_TOKEN_DECODE_DIR) + ] + for string in import_strings: + module = import_module(string) + try: + m = getattr(module, 'JWTTokenDecode') + jwt_decode_classes.append(m) + except Exception as exc: + logger.warn(exc) + return jwt_decode_classes + + +def decode_token(token: str) -> dict: + error_message, success, result = "", False, {} + for auth_class in get_jwt_decode_classes(): + result, success = auth_class().decode(token) + if not success: + error_message += result + else: + break + if not success: + raise AuthenticationFailed(error_message) + return result + + +class TaskAuthentication(BaseAuthentication): + def authenticate(self, request: Request): + token = request.META.get('HTTP_AUTHORIZATION') + payload = decode_token(token) + payload['token'] = token + if 'sub' in payload: + payload['id'] = int(payload['sub']) + return payload, _ diff --git a/sysom_server/sysom_diagnosis/lib/base_model.py b/sysom_server/sysom_diagnosis/lib/base_model.py new file mode 100644 index 00000000..863123d4 --- /dev/null +++ b/sysom_server/sysom_diagnosis/lib/base_model.py @@ -0,0 +1,20 @@ +# -*- encoding: utf-8 -*- +""" +@File : base_model.py +@Time : 2021/10/28 11:04 +@Author : DM +@Email : smmic@isoftstone.com +@Software: PyCharm +""" + +from django.db import models +from lib.utils import human_datetime + + +class BaseModel(models.Model): + """abstract model""" + created_at = models.CharField(max_length=20, default=human_datetime, verbose_name="创建时间") + updated_at = models.CharField(max_length=20, default=human_datetime, verbose_name="更新时间") + + class Meta: + abstract = True diff --git a/sysom_server/sysom_diagnosis/lib/base_view.py b/sysom_server/sysom_diagnosis/lib/base_view.py new file mode 100644 index 00000000..36172850 --- /dev/null +++ b/sysom_server/sysom_diagnosis/lib/base_view.py @@ -0,0 +1,90 @@ +# -*- coding: utf-8 -*- # +""" +Time 2022/10/21 10:08 +Author: mingfeng (SunnyQjm) +Email mfeng@linux.alibaba.com +File admin_static.py +Description: +""" +from typing import Any +from rest_framework.viewsets import GenericViewSet +from rest_framework.exceptions import ValidationError +from cec_base.cec_client import CecClient +from django.conf import settings + + +class CommonModelViewSet(GenericViewSet): + """ + 通用 ModelViewSet 实现,提供一些通用工具方法 + """ + + def __init__(self, **kwargs: Any) -> None: + super().__init__(**kwargs) + self._inner_cec_client: CecClient = None + + def produce_event_to_cec(self, topic: str, value: dict): + """Produce one event to specific topic""" + if self._inner_cec_client is None: + self._inner_cec_client = CecClient(settings.SYSOM_CEC_URL) + self._inner_cec_client.delivery(topic, value) + + def require_param_validate(self, request, require_params): + """ + 检查请求参数中是否包含所需的所有参数,任何参数缺失将返回错误,错误信息中包含缺失的参数列表 + @param request => HTTP 请求 + @param require_params => 需要验证的参数列表,例如:['hostname', 'ip'] + """ + if request.method in ['POST', 'PATCH', 'DELETE']: + context, missing_param_list = request.data, [] + for require_param in require_params: + if require_param not in context: + missing_param_list.append(require_param) + if len(missing_param_list) > 0: + return { + "success": False, + "message": f"Missing parameters: {', '.join(missing_param_list)}" + } + return { + "success": True, + "message": "" + } + + def extract_specific_params(self, request, params_list, require_all: bool = False): + """ + 从请求参数中提取 params_list 中指定的参数列表 => 最后将不在参数列表中的所有其它参数从参数列表中移除 + @param request => HTTP 请求 + @param params_list => 需要提取的参数列表,例如:['cluster'] + @param require_all => 是否所有参数都是必须的 + True => 任何参数缺失都会返回错误 + False => 对于每一个参数,有则提取,无则忽略 + """ + if require_all: + res = self.require_param_validate(request, params_list) + if not res["success"]: + return res + if request.method in ['POST', 'PATCH', 'DELETE']: + context, not_allow_params = request.data, [] + for key in context: + if key not in params_list: + not_allow_params.append(key) + for param_name in not_allow_params: + context.pop(param_name) + + return { + "success": True, + "message": "", + } + + def get_format_err_msg_for_validation_error(self, data: dict, err: ValidationError) -> str: + """ + 对验证错误的错误信息进行转换,使得提示更易读懂 + """ + results = [] + for k, v in err.detail.items(): + if v[0].code == 'unique': + # 唯一性错误 + results.append(f"{k}({data[k]})已存在") + else: + # 其它错误 + results.append(v[0]) + return "; ".join(results) diff --git a/sysom_api/sysom/services_conf/__init__.py b/sysom_server/sysom_diagnosis/lib/channels/__init__.py similarity index 100% rename from sysom_api/sysom/services_conf/__init__.py rename to sysom_server/sysom_diagnosis/lib/channels/__init__.py diff --git a/sysom_api/lib/channels/base.py b/sysom_server/sysom_diagnosis/lib/channels/base.py similarity index 100% rename from sysom_api/lib/channels/base.py rename to sysom_server/sysom_diagnosis/lib/channels/base.py diff --git a/sysom_api/lib/channels/ssh.py b/sysom_server/sysom_diagnosis/lib/channels/ssh.py similarity index 93% rename from sysom_api/lib/channels/ssh.py rename to sysom_server/sysom_diagnosis/lib/channels/ssh.py index 42b2ce82..51b57213 100644 --- a/sysom_api/lib/channels/ssh.py +++ b/sysom_server/sysom_diagnosis/lib/channels/ssh.py @@ -8,7 +8,7 @@ # SSH args eg: # "channel": "ssh", 选填 (默认ssh) -# "instance": "xxxxxxxx", 必填 +# "instance": "xxxxxxxx", 必填 # "cmd": "xxxx" 必填 @@ -18,10 +18,10 @@ import paramiko from io import StringIO from paramiko.client import SSHClient, AutoAddPolicy from paramiko.rsakey import RSAKey -from sysom.settings import * +from sysom_diagnosis.settings import KEY_PATH +from lib.utils import uuid_8 from .base import BaseChannel -from lib.utils import uuid_8, HTTP logger = logging.getLogger(__name__) @@ -74,7 +74,6 @@ class SSH: logger.error(e) return {} - def run_command(self, command): if self._client: ssh_session = self._client.get_transport().open_session() @@ -100,7 +99,7 @@ class SSH: def validate_ssh_host(ip: str, password: str, port: int = 22, username: str = 'root'): try: ssh = SSH(hostname=ip, password=password, - port=port, username=username, timeout=2) + port=port, username=username, timeout=2) ssh.add_public_key() return True, 'authorization success' except Exception as e: @@ -123,9 +122,6 @@ class Channel(BaseChannel): for field in self.FIELDS] ): raise Exception(f'parameter: {item[0]} not found!') - - # if not self.check_host_is_exist(self.kwargs['instance']): - # raise Exception(f'IP: {self.kwargs["instance"]} not exist') if not self.ssh: self.ssh = SSH(hostname=self.kwargs['instance']) diff --git a/sysom_server/sysom_diagnosis/lib/decode/sysom_decode.py b/sysom_server/sysom_diagnosis/lib/decode/sysom_decode.py new file mode 100644 index 00000000..12c7c1a0 --- /dev/null +++ b/sysom_server/sysom_diagnosis/lib/decode/sysom_decode.py @@ -0,0 +1,17 @@ +import jwt +from django.conf import settings + + +class JWTTokenDecode: + """SYSOM TOken解析认证""" + def decode(self, token): + r, s = None, False + try: + r, s = jwt.decode(token, key=settings.SECRET_KEY, algorithms='HS256'), True + except jwt.exceptions.ExpiredSignatureError as e: + r = f'令牌失效: {e}' + except jwt.exceptions.DecodeError as e: + r = f'令牌校验失败: {e}' + except jwt.exceptions.InvalidAlgorithmError as e: + r = f'令牌不合法: {e}' + return r, s \ No newline at end of file diff --git a/sysom_server/sysom_diagnosis/lib/exception.py b/sysom_server/sysom_diagnosis/lib/exception.py new file mode 100644 index 00000000..25aa22ea --- /dev/null +++ b/sysom_server/sysom_diagnosis/lib/exception.py @@ -0,0 +1,63 @@ +import logging +import traceback + +from django.db.models import ProtectedError +from rest_framework.views import set_rollback +from rest_framework import exceptions +from rest_framework.exceptions import APIException as DRFAPIException, AuthenticationFailed, NotAuthenticated + +from .response import ErrorResponse + + +logger = logging.getLogger(__name__) + + +class APIException(Exception): + def __init__(self, code=400, message='API异常', args=('API异常',)): + self.code = code + self.message = message + self.args = args + + def __str__(self): + return self.message + + +class FileNotFoundException(Exception): + def __init__(self, code=404, message='文件不存在'): + self.code = code + self.message = message + + def __str__(self): + return self.message + + +def exception_handler(exc, context): + """自定义异常处理""" + msg = '' + code = 400 + + if isinstance(exc, FileNotFoundException): + code = exc.code + msg = exc.message + if isinstance(exc, AuthenticationFailed): + code = 403 + msg = exc.detail + elif isinstance(exc, NotAuthenticated): + code = 402 + msg = exc.detail + elif isinstance(exc, DRFAPIException): + set_rollback() + # print(exc.detail) + # msg = {str(e) for e in exc.detail} + msg = exc.detail + elif isinstance(exc, exceptions.APIException): + set_rollback() + msg = exc.detail + elif isinstance(exc, ProtectedError): + set_rollback() + msg = "删除失败:该条数据与其他数据有相关绑定" + elif isinstance(exc, Exception): + logger.error(traceback.format_exc()) + msg = str(exc) # 原样输出错误 + + return ErrorResponse(msg=msg, code=code, status=code) diff --git a/sysom_server/sysom_diagnosis/lib/paginations.py b/sysom_server/sysom_diagnosis/lib/paginations.py new file mode 100644 index 00000000..b14639bb --- /dev/null +++ b/sysom_server/sysom_diagnosis/lib/paginations.py @@ -0,0 +1,26 @@ +''' +@File: paginations.py +@Time: 2021-12-14 13:46:02 +@Author: DM +@Desc: Local Paginations Class +''' + +from rest_framework.pagination import PageNumberPagination +from lib.response import success + + +class Pagination(PageNumberPagination): + page_query_param = "current" + page_size_query_param = "pageSize" + + def paginate_queryset(self, queryset, request, view=None): + self.max_page_size = queryset.count() + return super().paginate_queryset(queryset, request, view=view) + + def get_paginated_response(self, data): + return success(message="获取成功", result=data, total=self.page.paginator.count) + + def get_page_size(self, request): + if not request.query_params.get(self.page_size_query_param, None): + return self.max_page_size + return super().get_page_size(request) \ No newline at end of file diff --git a/sysom_server/sysom_diagnosis/lib/renderers.py b/sysom_server/sysom_diagnosis/lib/renderers.py new file mode 100644 index 00000000..5e92ce82 --- /dev/null +++ b/sysom_server/sysom_diagnosis/lib/renderers.py @@ -0,0 +1,42 @@ +import logging +from django.contrib.auth import get_user_model +from django.shortcuts import get_object_or_404 +from django.core.handlers.asgi import ASGIRequest +from rest_framework.renderers import JSONRenderer +from rest_framework.request import Request + + +logger = logging.getLogger(__name__) +User = get_user_model() + + +class SysomJsonRender(JSONRenderer): + def render(self, data, accepted_media_type=None, renderer_context=None): + if renderer_context: + request = renderer_context.get('request', None) + view = renderer_context.get('view', None) + response = renderer_context.get('response', None) + self.before_response_save_log(request, view, response) + return super().render(data, accepted_media_type, renderer_context) + + def before_response_save_log(self, request: Request, view, response): + user = getattr(request, 'user') or get_object_or_404(User, pk=1) + request: ASGIRequest = getattr(request, '_request', None) + method = request.method + + result = response.data + kwargs = { + 'request_ip': request.META.get('REMOTE_ADDR', None), + 'request_url': request.path, + 'request_browser_agent': request.headers.get('User-Agent', ''), + 'request_method': method, + 'handler_view': view.__class__.__name__, + 'response_status': getattr(response, 'status_code', 200), + } + if 'auth' in request.path: + kwargs['request_option'] = 0 + if result.get('code') == 200: + kwargs['user_id'] = result['data']['id'] + else: + kwargs['request_option'] = 1 + kwargs['user'] = user diff --git a/sysom_server/sysom_diagnosis/lib/response.py b/sysom_server/sysom_diagnosis/lib/response.py new file mode 100644 index 00000000..a2311a42 --- /dev/null +++ b/sysom_server/sysom_diagnosis/lib/response.py @@ -0,0 +1,68 @@ +from rest_framework.response import Response +from rest_framework import status +from django.http import FileResponse + + +def _response(data=None, status=None): + return Response(data=data, status=status) + + +def success(result, message="success", success=True, code=status.HTTP_200_OK, **kwargs): + data = { + "code": code, + "message": message, + "data": result, + "success": success + } + data.update(kwargs) + return _response(data=data, status=code) + + +def not_found(code=status.HTTP_404_NOT_FOUND, success=False, message="Not Found"): + data = { + "code": code, + "message": message, + "success": success, + } + + return _response(data=data, status=code) + + +def not_permission(code=status.HTTP_403_FORBIDDEN, success=False, message="Not Permission"): + data = { + "code": code, + "success": success, + "message": message + } + return _response(data=data, status=code) + + +def other_response(result=dict(), message="", success=True, code=status.HTTP_200_OK, **kwargs): + data = { + "code": code, + "message": message, + "data": result, + "success": success + } + data.update(kwargs) + return _response(data=data, status=code) + + +class ErrorResponse(Response): + """ + 标准响应错误的返回,ErrorResponse(msg='xxx') + 默认错误码返回400, 也可以指定其他返回码:ErrorResponse(code=xxx) + """ + + def __init__(self, data=None, msg='error', code=400, status=None, template_name=None, headers=None, + exception=False, content_type=None): + std_data = { + "code": code, + "data": data or {}, + "message": msg + } + super().__init__(std_data, status, template_name, headers, exception, content_type) + + +class FileResponseAlter(FileResponse): + pass diff --git a/sysom_server/sysom_diagnosis/lib/utils.py b/sysom_server/sysom_diagnosis/lib/utils.py new file mode 100644 index 00000000..2cf5bf9e --- /dev/null +++ b/sysom_server/sysom_diagnosis/lib/utils.py @@ -0,0 +1,69 @@ + +# -*- encoding: utf-8 -*- +""" +@File : utils.py +@Time : 2021/10/28 11:09 +@Author : DM +@Software: PyCharm +""" +import uuid as UUID +import logging +from datetime import datetime +from paramiko.rsakey import RSAKey +from io import StringIO + + +logger = logging.getLogger(__name__) + + +CHAR_SET = ("a", "b", "c", "d", "e", "f", + "g", "h", "i", "j", "k", "l", "m", "n", "o", "p", "q", "r", "s", + "t", "u", "v", "w", "x", "y", "z", "0", "1", "2", "3", "4", "5", + "6", "7", "8", "9", "A", "B", "C", "D", "E", "F", "G", "H", "I", + "J", "K", "L", "M", "N", "O", "P", "Q", "R", "S", "T", "U", "V", + "W", "X", "Y", "Z") + + +def human_datetime(date=None): + if date: + assert isinstance(date, datetime) + else: + date = datetime.now() + return date.strftime('%Y-%m-%d %H:%M:%S') + + +def uuid_36(): + """ + 返回36字符的UUID字符串(十六进制,含有-) bc5debab-95c3-4430-933f-2e3b6407ac30 + :return: + """ + return str(UUID.uuid4()) + + +def uuid_32(): + """ + 返回32字符的UUID字符串(十六进制) bc5debab95c34430933f2e3b6407ac30 + :return: + """ + return uuid_36().replace('-', '') + + +def uuid_8(): + """ + 返回8字符的UUID字符串(非进制) 3FNWjtlD + :return: + """ + s = uuid_32() + result = '' + for i in range(0, 8): + sub = s[i * 4: i * 4 + 4] + x = int(sub, 16) + result += CHAR_SET[x % 0x3E] + return result + + +def generate_key(): + key_obj = StringIO() + key = RSAKey.generate(2048) + key.write_private_key(key_obj) + return key_obj.getvalue(), 'ssh-rsa ' + key.get_base64() diff --git a/sysom_server/sysom_diagnosis/manage.py b/sysom_server/sysom_diagnosis/manage.py new file mode 100755 index 00000000..c3d6cc87 --- /dev/null +++ b/sysom_server/sysom_diagnosis/manage.py @@ -0,0 +1,22 @@ +#!/usr/bin/env python +"""Django's command-line utility for administrative tasks.""" +import os +import sys + + +def main(): + """Run administrative tasks.""" + os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'sysom_diagnosis.settings') + try: + from django.core.management import execute_from_command_line + except ImportError as exc: + raise ImportError( + "Couldn't import Django. Are you sure it's installed and " + "available on your PYTHONPATH environment variable? Did you " + "forget to activate a virtual environment?" + ) from exc + execute_from_command_line(sys.argv) + + +if __name__ == '__main__': + main() diff --git a/sysom_api/service_scripts/filecache b/sysom_server/sysom_diagnosis/service_scripts/filecache similarity index 100% rename from sysom_api/service_scripts/filecache rename to sysom_server/sysom_diagnosis/service_scripts/filecache diff --git a/sysom_api/service_scripts/filecache_post b/sysom_server/sysom_diagnosis/service_scripts/filecache_post similarity index 100% rename from sysom_api/service_scripts/filecache_post rename to sysom_server/sysom_diagnosis/service_scripts/filecache_post diff --git a/sysom_api/service_scripts/iosdiag_latency b/sysom_server/sysom_diagnosis/service_scripts/iosdiag_latency similarity index 100% rename from sysom_api/service_scripts/iosdiag_latency rename to sysom_server/sysom_diagnosis/service_scripts/iosdiag_latency diff --git a/sysom_api/service_scripts/iosdiag_latency_post b/sysom_server/sysom_diagnosis/service_scripts/iosdiag_latency_post similarity index 100% rename from sysom_api/service_scripts/iosdiag_latency_post rename to sysom_server/sysom_diagnosis/service_scripts/iosdiag_latency_post diff --git a/sysom_api/service_scripts/loadtask b/sysom_server/sysom_diagnosis/service_scripts/loadtask similarity index 100% rename from sysom_api/service_scripts/loadtask rename to sysom_server/sysom_diagnosis/service_scripts/loadtask diff --git a/sysom_api/service_scripts/loadtask_post b/sysom_server/sysom_diagnosis/service_scripts/loadtask_post similarity index 100% rename from sysom_api/service_scripts/loadtask_post rename to sysom_server/sysom_diagnosis/service_scripts/loadtask_post diff --git a/sysom_api/service_scripts/memgraph b/sysom_server/sysom_diagnosis/service_scripts/memgraph similarity index 100% rename from sysom_api/service_scripts/memgraph rename to sysom_server/sysom_diagnosis/service_scripts/memgraph diff --git a/sysom_api/service_scripts/memgraph_post b/sysom_server/sysom_diagnosis/service_scripts/memgraph_post similarity index 100% rename from sysom_api/service_scripts/memgraph_post rename to sysom_server/sysom_diagnosis/service_scripts/memgraph_post diff --git a/sysom_api/service_scripts/node_delete b/sysom_server/sysom_diagnosis/service_scripts/node_delete similarity index 100% rename from sysom_api/service_scripts/node_delete rename to sysom_server/sysom_diagnosis/service_scripts/node_delete diff --git a/sysom_api/service_scripts/node_init b/sysom_server/sysom_diagnosis/service_scripts/node_init similarity index 100% rename from sysom_api/service_scripts/node_init rename to sysom_server/sysom_diagnosis/service_scripts/node_init diff --git a/sysom_api/service_scripts/oomcheck b/sysom_server/sysom_diagnosis/service_scripts/oomcheck similarity index 100% rename from sysom_api/service_scripts/oomcheck rename to sysom_server/sysom_diagnosis/service_scripts/oomcheck diff --git a/sysom_api/service_scripts/oomcheck_post b/sysom_server/sysom_diagnosis/service_scripts/oomcheck_post similarity index 100% rename from sysom_api/service_scripts/oomcheck_post rename to sysom_server/sysom_diagnosis/service_scripts/oomcheck_post diff --git a/sysom_api/service_scripts/ossre b/sysom_server/sysom_diagnosis/service_scripts/ossre similarity index 100% rename from sysom_api/service_scripts/ossre rename to sysom_server/sysom_diagnosis/service_scripts/ossre diff --git a/sysom_api/service_scripts/ossre_post b/sysom_server/sysom_diagnosis/service_scripts/ossre_post similarity index 100% rename from sysom_api/service_scripts/ossre_post rename to sysom_server/sysom_diagnosis/service_scripts/ossre_post diff --git a/sysom_api/service_scripts/pingtrace b/sysom_server/sysom_diagnosis/service_scripts/pingtrace similarity index 100% rename from sysom_api/service_scripts/pingtrace rename to sysom_server/sysom_diagnosis/service_scripts/pingtrace diff --git a/sysom_api/service_scripts/pingtrace_post b/sysom_server/sysom_diagnosis/service_scripts/pingtrace_post similarity index 100% rename from sysom_api/service_scripts/pingtrace_post rename to sysom_server/sysom_diagnosis/service_scripts/pingtrace_post diff --git a/sysom_api/service_scripts/schedmoni b/sysom_server/sysom_diagnosis/service_scripts/schedmoni similarity index 100% rename from sysom_api/service_scripts/schedmoni rename to sysom_server/sysom_diagnosis/service_scripts/schedmoni diff --git a/sysom_api/service_scripts/schedmoni_post b/sysom_server/sysom_diagnosis/service_scripts/schedmoni_post similarity index 100% rename from sysom_api/service_scripts/schedmoni_post rename to sysom_server/sysom_diagnosis/service_scripts/schedmoni_post diff --git a/sysom_api/service_scripts/test b/sysom_server/sysom_diagnosis/service_scripts/test similarity index 100% rename from sysom_api/service_scripts/test rename to sysom_server/sysom_diagnosis/service_scripts/test diff --git a/sysom_api/service_scripts/test.sh b/sysom_server/sysom_diagnosis/service_scripts/test.sh similarity index 100% rename from sysom_api/service_scripts/test.sh rename to sysom_server/sysom_diagnosis/service_scripts/test.sh diff --git a/sysom_api/service_scripts/tpython b/sysom_server/sysom_diagnosis/service_scripts/tpython similarity index 100% rename from sysom_api/service_scripts/tpython rename to sysom_server/sysom_diagnosis/service_scripts/tpython diff --git a/sysom_api/service_scripts/vmcore_analyse b/sysom_server/sysom_diagnosis/service_scripts/vmcore_analyse similarity index 100% rename from sysom_api/service_scripts/vmcore_analyse rename to sysom_server/sysom_diagnosis/service_scripts/vmcore_analyse diff --git a/sysom_server/sysom_diagnosis/sysom_diagnosis/__init__.py b/sysom_server/sysom_diagnosis/sysom_diagnosis/__init__.py new file mode 100644 index 00000000..aa60bed8 --- /dev/null +++ b/sysom_server/sysom_diagnosis/sysom_diagnosis/__init__.py @@ -0,0 +1,3 @@ +import pymysql + +pymysql.install_as_MySQLdb() \ No newline at end of file diff --git a/sysom_server/sysom_diagnosis/sysom_diagnosis/asgi.py b/sysom_server/sysom_diagnosis/sysom_diagnosis/asgi.py new file mode 100644 index 00000000..06096259 --- /dev/null +++ b/sysom_server/sysom_diagnosis/sysom_diagnosis/asgi.py @@ -0,0 +1,16 @@ +""" +ASGI config for sysom_diagnosis project. + +It exposes the ASGI callable as a module-level variable named ``application``. + +For more information on this file, see +https://docs.djangoproject.com/en/3.2/howto/deployment/asgi/ +""" + +import os + +from django.core.asgi import get_asgi_application + +os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'sysom_diagnosis.settings') + +application = get_asgi_application() diff --git a/sysom_server/sysom_diagnosis/sysom_diagnosis/settings.py b/sysom_server/sysom_diagnosis/sysom_diagnosis/settings.py new file mode 100644 index 00000000..1a5ff399 --- /dev/null +++ b/sysom_server/sysom_diagnosis/sysom_diagnosis/settings.py @@ -0,0 +1,15 @@ +import os + +env = os.environ.get("env", "product") + + +if env == "develop": + from conf.develop import * +elif env == "testing": + from conf.testing import * +elif env == "product": + from conf.product import * + +# 跨域允许 +if DEBUG: + CORS_ORIGIN_ALLOW_ALL = True \ No newline at end of file diff --git a/sysom_server/sysom_diagnosis/sysom_diagnosis/urls.py b/sysom_server/sysom_diagnosis/sysom_diagnosis/urls.py new file mode 100644 index 00000000..5ebd9201 --- /dev/null +++ b/sysom_server/sysom_diagnosis/sysom_diagnosis/urls.py @@ -0,0 +1,26 @@ +"""sysom_diagnosis URL Configuration + +The `urlpatterns` list routes URLs to views. For more information please see: + https://docs.djangoproject.com/en/3.2/topics/http/urls/ +Examples: +Function views + 1. Add an import: from my_app import views + 2. Add a URL to urlpatterns: path('', views.home, name='home') +Class-based views + 1. Add an import: from other_app.views import Home + 2. Add a URL to urlpatterns: path('', Home.as_view(), name='home') +Including another URLconf + 1. Import the include() function: from django.urls import include, path + 2. Add a URL to urlpatterns: path('blog/', include('blog.urls')) +""" +from django.urls import path +from django.urls.conf import include +from django.conf import settings + +app_urlpatterns = [path('', include( + f'{app}.urls')) for app in settings.INSTALLED_APPS if app.startswith('apps')] + +urlpatterns = [ +] + +urlpatterns += app_urlpatterns diff --git a/sysom_server/sysom_diagnosis/sysom_diagnosis/wsgi.py b/sysom_server/sysom_diagnosis/sysom_diagnosis/wsgi.py new file mode 100644 index 00000000..cab7249d --- /dev/null +++ b/sysom_server/sysom_diagnosis/sysom_diagnosis/wsgi.py @@ -0,0 +1,16 @@ +""" +WSGI config for sysom_diagnosis project. + +It exposes the WSGI callable as a module-level variable named ``application``. + +For more information on this file, see +https://docs.djangoproject.com/en/3.2/howto/deployment/wsgi/ +""" + +import os + +from django.core.wsgi import get_wsgi_application + +os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'sysom_diagnosis.settings') + +application = get_wsgi_application() diff --git a/tools/deploy/channel-service.ini b/tools/deploy/channel-service.ini index 1974da64..74534066 100644 --- a/tools/deploy/channel-service.ini +++ b/tools/deploy/channel-service.ini @@ -1,6 +1,6 @@ [program:channel-service] -directory = /usr/local/sysom/server/target/sysom_api -command=/usr/local/sysom/server/virtualenv/bin/gunicorn -c ./conf/channel_gunicorn.py sysom.wsgi:application +directory = /usr/local/sysom/server/target/sysom_server/sysom_channel +command=/usr/local/sysom/server/virtualenv/bin/gunicorn -c ./conf/channel_gunicorn.py sysom_channel.wsgi:application startsecs=3 autostart=true autorestart=true diff --git a/tools/deploy/deploy.sh b/tools/deploy/deploy.sh index bb1db53e..23371cc1 100755 --- a/tools/deploy/deploy.sh +++ b/tools/deploy/deploy.sh @@ -9,7 +9,11 @@ ALIYUN_MIRROR="https://mirrors.aliyun.com/pypi/simple/" APP_NAME="sysom" -API_DIR="sysom_api" +SERVER_DIR="sysom_server" +API_DIR=$SERVER_DIR/sysom_api +DIAGNOSIS_DIR=$SERVER_DIR/sysom_diagnosis +CHANNEL_DIR=$SERVER_DIR/sysom_channel +SDK_DIR=$SERVER_DIR/sdk WEB_DIR="sysom_web" SCRIPT_DIR="script" APP_HOME=/usr/local/sysom @@ -72,7 +76,7 @@ update_target() { fi mkdir -p ${TARGET_PATH} echo "INFO: copy project file..." - cp -r ${API_DIR} ${WEB_DIR} ${TARGET_PATH} + cp -r ${SERVER_DIR} ${WEB_DIR} ${TARGET_PATH} cp -r ${SCRIPT_DIR} ${APP_HOME}/init_scripts } @@ -82,19 +86,17 @@ init_conf() { cp tools/deploy/nginx.conf /etc/nginx/ cp tools/deploy/sysom.conf /etc/nginx/conf.d/ cp tools/deploy/sysom.ini /etc/supervisord.d/ - cp tools/deploy/task-service.ini /etc/supervisord.d/ - # cp tools/deploy/channel-service.ini /etc/supervisord.d/ - cp tools/deploy/task-executor-service.ini /etc/supervisord.d/ + cp tools/deploy/diagnosis-service.ini /etc/supervisord.d/ + cp tools/deploy/channel-service.ini /etc/supervisord.d/ ###change the install dir base on param $1### sed -i "s;/usr/local/sysom;${APP_HOME};g" /etc/nginx/conf.d/sysom.conf sed -i "s;/usr/local/sysom;${APP_HOME};g" /etc/supervisord.d/sysom.ini - sed -i "s;/usr/local/sysom;${APP_HOME};g" /etc/supervisord.d/task-service.ini - # sed -i "s;/usr/local/sysom;${APP_HOME};g" /etc/supervisord.d/channel-service.ini - sed -i "s;/usr/local/sysom;${APP_HOME};g" /etc/supervisord.d/task-executor-service.ini + sed -i "s;/usr/local/sysom;${APP_HOME};g" /etc/supervisord.d/diagnosis-service.ini + sed -i "s;/usr/local/sysom;${APP_HOME};g" /etc/supervisord.d/channel-service.ini cp tools/deploy/sysom-server.service /usr/lib/systemd/system/ cpu_num=`cat /proc/cpuinfo | grep processor | wc -l` - sed -i "s/threads = 3/threads = $cpu_num/g" ${TARGET_PATH}/${API_DIR}/conf/task_gunicorn.py - # sed -i "s/threads = 3/threads = $cpu_num/g" ${TARGET_PATH}/${API_DIR}/conf/channel_gunicorn.py + sed -i "s/threads = 3/threads = $cpu_num/g" ${TARGET_PATH}/${DIAGNOSIS_DIR}/conf/diagnosis_gunicorn.py + sed -i "s/threads = 3/threads = $cpu_num/g" ${TARGET_PATH}/${CHANNEL_DIR}/conf/channel_gunicorn.py } start_script_server() { diff --git a/tools/deploy/task-service.ini b/tools/deploy/diagnosis-service.ini similarity index 35% rename from tools/deploy/task-service.ini rename to tools/deploy/diagnosis-service.ini index 412415ba..4dcf4753 100644 --- a/tools/deploy/task-service.ini +++ b/tools/deploy/diagnosis-service.ini @@ -1,9 +1,9 @@ -[program:task-service] -directory = /usr/local/sysom/server/target/sysom_api -command=/usr/local/sysom/server/virtualenv/bin/gunicorn -c ./conf/task_gunicorn.py sysom.wsgi:application +[program:diagnosis-service] +directory = /usr/local/sysom/server/target/sysom_server/sysom_diagnosis +command=/usr/local/sysom/server/virtualenv/bin/gunicorn -c ./conf/diagnosis_gunicorn.py sysom_diagnosis.wsgi:application startsecs=3 autostart=true autorestart=true environment=PATH="/usr/local/sysom/server/virtualenv/bin/" -stderr_logfile=/usr/local/sysom/server/logs/supervisord-task-error.log -stdout_logfile=/usr/local/sysom/server/logs/supervisord-task.log \ No newline at end of file +stderr_logfile=/usr/local/sysom/server/logs/supervisord-diagnosis-error.log +stdout_logfile=/usr/local/sysom/server/logs/supervisord-diagnosis.log diff --git a/tools/deploy/sysom.conf b/tools/deploy/sysom.conf index ff82e93f..b89be938 100644 --- a/tools/deploy/sysom.conf +++ b/tools/deploy/sysom.conf @@ -38,14 +38,14 @@ server { proxy_redirect off; proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; } - - location /api/v1/channel/ { - proxy_pass http://127.0.0.1:7003; + + location /api/v2/tasks/ { + proxy_pass http://127.0.0.1:7002; proxy_read_timeout 180s; proxy_redirect off; proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; } - + location /api/ { proxy_pass http://127.0.0.1:7001; proxy_read_timeout 180s; @@ -53,6 +53,7 @@ server { proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; } + location /checkpreload.htm { proxy_pass http://127.0.0.1:7001/checkpreload.htm; proxy_read_timeout 180s; diff --git a/tools/deploy/sysom.ini b/tools/deploy/sysom.ini index f3547bc8..f218576a 100644 --- a/tools/deploy/sysom.ini +++ b/tools/deploy/sysom.ini @@ -1,6 +1,6 @@ [fcgi-program:sysom] socket=tcp://localhost:7001 -directory=/usr/local/sysom/server/target/sysom_api +directory=/usr/local/sysom/server/target/sysom_server/sysom_api command=/usr/local/sysom/server/virtualenv/bin/daphne -u /run/daphne%(process_num)d.sock --fd 0 --access-log - --proxy-headers sysom.asgi:application numprocs=4 process_name=asgi%(process_num)d diff --git a/tools/deploy/task-executor-service.ini b/tools/deploy/task-executor-service.ini deleted file mode 100644 index 45b6b787..00000000 --- a/tools/deploy/task-executor-service.ini +++ /dev/null @@ -1,9 +0,0 @@ -[program:task-executor-service] -directory = /usr/local/sysom/server/target/sysom_api -command=/usr/local/sysom/server/virtualenv/bin/python -m microservices.task_executor.app -startsecs=3 -autostart=true -autorestart=true -environment=PATH="/usr/local/sysom/server/virtualenv/bin/" -stderr_logfile=/usr/local/sysom/server/logs/supervisord-task-executor-error.log -stdout_logfile=/usr/local/sysom/server/logs/supervisord-task-executor.log -- Gitee