diff --git a/0001-fix-cve-list-sort.patch b/0001-fix-cve-list-sort.patch deleted file mode 100644 index 41b17aaa6bf8eeaad3a40635b2d2e1fd0f0ab198..0000000000000000000000000000000000000000 --- a/0001-fix-cve-list-sort.patch +++ /dev/null @@ -1,30 +0,0 @@ -From b87c51761b3143aabd39a3e9c91596e540325172 Mon Sep 17 00:00:00 2001 -From: rearcher <123781007@qq.com> -Date: Mon, 18 Dec 2023 22:24:51 +0800 -Subject: [PATCH] fix cve_list sort - ---- - apollo/database/proxy/cve.py | 5 +++-- - 1 file changed, 3 insertions(+), 2 deletions(-) - -diff --git a/apollo/database/proxy/cve.py b/apollo/database/proxy/cve.py -index e92cb14..6210156 100644 ---- a/apollo/database/proxy/cve.py -+++ b/apollo/database/proxy/cve.py -@@ -669,10 +669,11 @@ class CveProxy(CveMysqlProxy, CveEsProxy): - sort_page['limt_size'] = int(per_page) - - # sort by host num by default -+ sort_page["order_by"] = data.get("direction", "asc") - sort_page["order_by_filed"] = data.get('sort', "cve_id") - if sort_page["order_by_filed"] == "host_num": -- sort_page["order_by_filed"] = "cve_id, host_num" -- sort_page["order_by"] = data.get("direction", "asc") -+ sort_page["order_by_filed"] = f"host_num {sort_page['order_by']},cve_id " -+ - return sort_page - - def _query_cve_list(self, data): --- -Gitee - diff --git a/0001-fix-security-advisory-parsing-error.patch b/0001-fix-security-advisory-parsing-error.patch new file mode 100644 index 0000000000000000000000000000000000000000..bd9121015f545f999b15e747f9927c277f4b7701 --- /dev/null +++ b/0001-fix-security-advisory-parsing-error.patch @@ -0,0 +1,85 @@ +From 25e79499a5c578579c1112bfcbdfb7137748fa84 Mon Sep 17 00:00:00 2001 +From: rabbitali +Date: Fri, 19 Jul 2024 10:28:52 +0800 +Subject: [PATCH 1/1] fix security advisory parsing error + +--- + apollo/cron/download_advisory.py | 6 ++++-- + apollo/handler/cve_handler/view.py | 16 ++++++++-------- + 2 files changed, 12 insertions(+), 10 deletions(-) + +diff --git a/apollo/cron/download_advisory.py b/apollo/cron/download_advisory.py +index 6ba2f01..5bec2a8 100644 +--- a/apollo/cron/download_advisory.py ++++ b/apollo/cron/download_advisory.py +@@ -115,14 +115,16 @@ class DownloadSATask: + file_path = os.path.join(advisory_dir, file_name) + advisory_year, advisory_serial_number = re.findall("\d+", file_name) + try: +- cve_rows, cve_pkg_rows, cve_pkg_docs, _, _ = parse_security_advisory(file_path) ++ security_cvrf_info = parse_security_advisory(file_path) ++ security_cvrf_info.sa_year = None ++ security_cvrf_info.sa_number = None + except (KeyError, ParseAdvisoryError) as error: + LOGGER.error(error) + LOGGER.error("Some error occurred when parse advisory '%s'." % file_name) + self._record_download_result(advisory_year, advisory_serial_number, False) + continue + +- save_status_code = proxy.save_security_advisory(file_name, cve_rows, cve_pkg_rows, cve_pkg_docs) ++ save_status_code = proxy.save_security_advisory(file_name, security_cvrf_info) + status = True if save_status_code == SUCCEED else False + self._record_download_result(advisory_year, advisory_serial_number, status) + +diff --git a/apollo/handler/cve_handler/view.py b/apollo/handler/cve_handler/view.py +index 58d3bb1..200cc0d 100644 +--- a/apollo/handler/cve_handler/view.py ++++ b/apollo/handler/cve_handler/view.py +@@ -466,9 +466,9 @@ class VulUploadAdvisory(BaseResponse): + def _save_single_advisory(proxy, file_path): + file_name = os.path.basename(file_path) + try: +- cve_rows, cve_pkg_rows, cve_pkg_docs, sa_year, sa_number = parse_security_advisory(file_path) ++ security_cvrf_info = parse_security_advisory(file_path) + os.remove(file_path) +- if not all([cve_rows, cve_pkg_rows, cve_pkg_docs]): ++ if not all([security_cvrf_info.cve_rows, security_cvrf_info.cve_pkg_rows, security_cvrf_info.cve_pkg_docs]): + return WRONG_FILE_FORMAT + except (KeyError, ParseAdvisoryError) as error: + os.remove(file_path) +@@ -476,7 +476,7 @@ class VulUploadAdvisory(BaseResponse): + LOGGER.error(error) + return WRONG_FILE_FORMAT + +- status_code = proxy.save_security_advisory(file_name, cve_rows, cve_pkg_rows, cve_pkg_docs, sa_year, sa_number) ++ status_code = proxy.save_security_advisory(file_name, security_cvrf_info) + + return status_code + +@@ -504,8 +504,10 @@ class VulUploadAdvisory(BaseResponse): + shutil.rmtree(folder_path) + return WRONG_FILE_FORMAT + try: +- cve_rows, cve_pkg_rows, cve_pkg_docs, sa_year, sa_number = parse_security_advisory(file_path) +- if not all([cve_rows, cve_pkg_rows, cve_pkg_docs]): ++ security_cvrf_info = parse_security_advisory(file_path) ++ if not all( ++ [security_cvrf_info.cve_rows, security_cvrf_info.cve_pkg_rows, security_cvrf_info.cve_pkg_docs] ++ ): + shutil.rmtree(folder_path) + return WRONG_FILE_FORMAT + except (KeyError, ParseAdvisoryError) as error: +@@ -519,9 +521,7 @@ class VulUploadAdvisory(BaseResponse): + LOGGER.error(error) + continue + # elasticsearch need 1 second to update doc +- status_code = proxy.save_security_advisory( +- file_name, cve_rows, cve_pkg_rows, cve_pkg_docs, sa_year, sa_number +- ) ++ status_code = proxy.save_security_advisory(file_name, security_cvrf_info) + if status_code != SUCCEED: + fail_list.append(file_name) + else: +-- +2.33.0 + diff --git a/0006-fix-bug-with-host-count-in-cve-fix-task.patch b/0002-fix-bug-with-host-count-in-cve-fix-task.patch similarity index 76% rename from 0006-fix-bug-with-host-count-in-cve-fix-task.patch rename to 0002-fix-bug-with-host-count-in-cve-fix-task.patch index ba01569591b3ee889ec55ea908d50af293e0bde0..9a8b106537f8680c07514737ba0fbb6e8b12216f 100644 --- a/0006-fix-bug-with-host-count-in-cve-fix-task.patch +++ b/0002-fix-bug-with-host-count-in-cve-fix-task.patch @@ -1,6 +1,6 @@ -From dffeb775d77e1c608eed8dc9553fcc46ebbfc450 Mon Sep 17 00:00:00 2001 -From: root -Date: Wed, 24 Jul 2024 16:46:56 +0800 +From a82cceada0df66cf48d646a3cd6a55556ebf9962 Mon Sep 17 00:00:00 2001 +From: rabbitali +Date: Tue, 23 Jul 2024 18:55:50 +0800 Subject: [PATCH 1/1] fix bug with host count in cve fix task --- @@ -8,11 +8,11 @@ Subject: [PATCH 1/1] fix bug with host count in cve fix task 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/apollo/database/proxy/task/cve_fix.py b/apollo/database/proxy/task/cve_fix.py -index efdc91e..468c03a 100644 +index fa3f2a9..1aa5546 100644 --- a/apollo/database/proxy/task/cve_fix.py +++ b/apollo/database/proxy/task/cve_fix.py -@@ -133,7 +133,8 @@ class CveFixTaskProxy(TaskProxy): - return PARAM_ERROR, dict() +@@ -121,7 +121,8 @@ class CveFixTaskProxy(TaskProxy): + wait_fix_rpms = dict() for task_info in fix_host_rpm_info: - wait_fix_rpms[task_info["cve_id"]] = dict(rpms=task_info.get("rpms", []), hosts=list(host_dict.keys())) @@ -23,3 +23,4 @@ index efdc91e..468c03a 100644 fix_tasks = [] -- 2.33.0 + diff --git a/0002-fix-the-query-error-of-cve-associated-host.patch b/0002-fix-the-query-error-of-cve-associated-host.patch deleted file mode 100644 index ff863c16618ef55bd899baeb9ed42349c6bd9681..0000000000000000000000000000000000000000 --- a/0002-fix-the-query-error-of-cve-associated-host.patch +++ /dev/null @@ -1,27 +0,0 @@ -From f8cab408f39bb8f6b793021cdfbf93338fb6ed0a Mon Sep 17 00:00:00 2001 -From: gongzt -Date: Wed, 20 Dec 2023 15:13:07 +0800 -Subject: [PATCH] Fix the query error of cve associated host -MIME-Version: 1.0 -Content-Type: text/plain; charset=UTF-8 -Content-Transfer-Encoding: 8bit - ---- - apollo/database/proxy/cve.py | 1 + - 1 file changed, 1 insertion(+) - -diff --git a/apollo/database/proxy/cve.py b/apollo/database/proxy/cve.py -index 6210156..b5a61b9 100644 ---- a/apollo/database/proxy/cve.py -+++ b/apollo/database/proxy/cve.py -@@ -1532,6 +1532,7 @@ class CveProxy(CveMysqlProxy, CveEsProxy): - def _get_processed_cve_packages_host(self, data): - result = {"total_count": 0, "total_page": 0, "result": []} - filters = { -+ CveHostAssociation.host_user == data["username"], - CveHostAssociation.cve_id == data["cve_id"], - CveHostAssociation.installed_rpm == data["installed_rpm"], - CveHostAssociation.fixed == data["fixed"], --- -2.33.0 - diff --git a/0003-fix-issue-with-language-display-in-task-generation.patch b/0003-fix-issue-with-language-display-in-task-generation.patch new file mode 100644 index 0000000000000000000000000000000000000000..20e107fa4f1581288b86808d3b183ca0cc776929 --- /dev/null +++ b/0003-fix-issue-with-language-display-in-task-generation.patch @@ -0,0 +1,149 @@ +From 7ef8931b94ce3a8801e00413b35a37691456e800 Mon Sep 17 00:00:00 2001 +From: rabbitali +Date: Wed, 28 Aug 2024 09:34:49 +0800 +Subject: [PATCH 1/1] fix issue with language display in task generation api + +--- + apollo/database/proxy/task/cve_fix.py | 18 ++++++++++++++++-- + apollo/database/proxy/task/cve_rollback.py | 20 +++++++++++++++++--- + apollo/function/schema/task.py | 6 +++--- + database/aops-apollo.sql | 2 +- + 4 files changed, 37 insertions(+), 9 deletions(-) + +diff --git a/apollo/database/proxy/task/cve_fix.py b/apollo/database/proxy/task/cve_fix.py +index 1aa5546..fc98633 100644 +--- a/apollo/database/proxy/task/cve_fix.py ++++ b/apollo/database/proxy/task/cve_fix.py +@@ -23,7 +23,7 @@ from typing import Dict, Tuple + + import sqlalchemy.orm + from elasticsearch import ElasticsearchException +-from flask import g ++from flask import request + from sqlalchemy import func, case + from sqlalchemy.exc import SQLAlchemyError + from sqlalchemy.sql import or_ +@@ -462,6 +462,11 @@ class CveFixTaskProxy(TaskProxy): + } + + """ ++ lang_info = request.headers.get("Accept-Language") ++ if lang_info: ++ lang = lang_info.split(',')[0].split(';')[0] ++ else: ++ lang = "en" + task_id = str(uuid.uuid1()).replace('-', '') + task_info = copy.deepcopy(data) + task_info['task_id'] = task_id +@@ -470,8 +475,17 @@ class CveFixTaskProxy(TaskProxy): + task_info["check_items"] = ",".join(task_info["check_items"]) + task_info["host_num"] = len(wait_fix_rpms.keys()) + task_info["fix_type"] = fix_way ++ ++ prefix_map = { ++ "zh": {"hotpatch": "热补丁修复", "coldpatch": "冷补丁修复"}, ++ "en": {"hotpatch": "Livepatch Upgrade", "coldpatch": "Normal Upgrade"}, ++ } + if subtask: +- task_prefix = "冷补丁修复：" if fix_way == "coldpatch" else "热补丁修复：" ++ task_prefix = ( ++ f"{prefix_map['en'].get(fix_way,'coldpatch')}：" ++ if lang.startswith("en") ++ else f"{prefix_map['zh'].get(fix_way,'coldpatch')}：" ++ ) + task_info["description"] = task_prefix + task_info["description"] + task_info["task_name"] = task_prefix + task_info["task_name"] + task_info["takeover"] = False if fix_way == "coldpatch" else task_info["takeover"] +diff --git a/apollo/database/proxy/task/cve_rollback.py b/apollo/database/proxy/task/cve_rollback.py +index e93be15..0525602 100644 +--- a/apollo/database/proxy/task/cve_rollback.py ++++ b/apollo/database/proxy/task/cve_rollback.py +@@ -14,7 +14,7 @@ from typing import Tuple, Optional + + import sqlalchemy.orm + from elasticsearch import ElasticsearchException +-from flask import g ++from flask import request + from sqlalchemy import or_, func, case + from sqlalchemy.exc import SQLAlchemyError + +@@ -106,16 +106,30 @@ class CveRollbackTaskProxy(TaskProxy): + + @staticmethod + def _gen_task_row(data: dict, cve_fix_task_info: sqlalchemy.orm.Query) -> dict: ++ lang_info = request.headers.get("Accept-Language") ++ if lang_info: ++ lang = lang_info.split(',')[0].split(';')[0] ++ else: ++ lang = "en" ++ + fix_task_description = cve_fix_task_info.description + fix_task_name = cve_fix_task_info.task_name + host_num = cve_fix_task_info.host_num ++ ++ if lang.startswith("en"): ++ task_name = "ROLLBACK_TASK: %s" % fix_task_name ++ description = "ORIGIN_TASK_DESCRIPTION: %s" % fix_task_description ++ else: ++ task_name = "回滚: %s" % fix_task_name ++ description = "原CVE修复任务描述: %s" % fix_task_description ++ + task_data = { + "cluster_id": data["cluster_id"], + "task_id": data["task_id"], + "task_type": data["task_type"], + "create_time": data["create_time"], +- "task_name": "回滚: %s" % fix_task_name, +- "description": "原CVE修复任务描述: %s" % fix_task_description, ++ "task_name": task_name, ++ "description": description, + "host_num": host_num, + "username": data.get("username"), + } +diff --git a/apollo/function/schema/task.py b/apollo/function/schema/task.py +index a494703..f306cea 100644 +--- a/apollo/function/schema/task.py ++++ b/apollo/function/schema/task.py +@@ -91,7 +91,7 @@ class GenerateCveTaskSchema(Schema): + """ + + task_name = fields.String(required=True, validate=lambda s: 0 < len(s) <= 20) +- description = fields.String(required=True, validate=lambda s: 0 < len(s) <= 50) ++ description = fields.String(required=True, validate=lambda s: 0 < len(s) <= 100) + accepted = fields.Boolean(required=True, validate=validate.OneOf([True, False])) + check_items = fields.List(fields.String(required=True, validate=lambda s: 0 < len(s) <= 32), required=False) + takeover = fields.Boolean(required=True, validate=validate.OneOf([True, False])) +@@ -178,7 +178,7 @@ class GenerateRepoTaskSchema(Schema): + """ + + task_name = fields.String(required=True, validate=lambda s: 0 < len(s) <= 20) +- description = fields.String(required=True, validate=lambda s: 0 < len(s) <= 50) ++ description = fields.String(required=True, validate=lambda s: 0 < len(s) <= 100) + repo_id = fields.String(required=True, validate=lambda s: 0 < len(s) <= 36) + host_list = fields.List(fields.String(required=True, validate=lambda s: 0 < len(s) <= 36), required=True) + +@@ -367,7 +367,7 @@ class HotpatchRemoveInfoSchema(Schema): + + class GenerateHotpatchRemoveTaskSchema(Schema): + task_name = fields.String(required=True, validate=lambda s: 0 < len(s) <= 20) +- description = fields.String(required=True, validate=lambda s: 0 < len(s) <= 50) ++ description = fields.String(required=True, validate=lambda s: 0 < len(s) <= 100) + info = fields.List(fields.Nested(HotpatchRemoveInfoSchema), required=True, validate=lambda s: len(s) > 0) + + +diff --git a/database/aops-apollo.sql b/database/aops-apollo.sql +index 4e99d7a..b78bb3a 100644 +--- a/database/aops-apollo.sql ++++ b/database/aops-apollo.sql +@@ -19,7 +19,7 @@ CREATE TABLE IF NOT EXISTS `cve` ( + CREATE TABLE IF NOT EXISTS `vul_task` ( + `task_id` varchar(32) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL, + `task_type` varchar(20) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL, +- `description` varchar(100) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL, ++ `description` varchar(150) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL, + `task_name` varchar(50) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL, + `latest_execute_time` int(11) NULL DEFAULT NULL, + `create_time` int(11) NULL DEFAULT NULL, +-- +2.33.0 + diff --git a/0003-update-verification-method-for-host-ip-fieldl.patch b/0003-update-verification-method-for-host-ip-fieldl.patch deleted file mode 100644 index c40bab761e32ffc8f33231786b271cfde65fc54c..0000000000000000000000000000000000000000 --- a/0003-update-verification-method-for-host-ip-fieldl.patch +++ /dev/null @@ -1,85 +0,0 @@ -From 47a4c1d6488f07aa55621454fefedb559fc1bbf8 Mon Sep 17 00:00:00 2001 -From: rabbitali -Date: Wed, 20 Dec 2023 16:26:24 +0800 -Subject: [PATCH] update verification method for host ip fieldl;fix repo - field filter error - ---- - apollo/database/proxy/cve.py | 8 ++++++-- - apollo/function/schema/cve.py | 2 +- - apollo/function/schema/task.py | 6 +++--- - 3 files changed, 10 insertions(+), 6 deletions(-) - -diff --git a/apollo/database/proxy/cve.py b/apollo/database/proxy/cve.py -index 6210156..5d29544 100644 ---- a/apollo/database/proxy/cve.py -+++ b/apollo/database/proxy/cve.py -@@ -20,7 +20,7 @@ import copy - from collections import defaultdict - - from elasticsearch import ElasticsearchException --from sqlalchemy import func, tuple_, case -+from sqlalchemy import func, tuple_, case, or_ - from sqlalchemy.exc import SQLAlchemyError - from vulcanus.database.helper import sort_and_page, judge_return_code - from vulcanus.database.proxy import MysqlProxy, ElasticsearchProxy -@@ -200,7 +200,11 @@ class CveMysqlProxy(MysqlProxy): - if filter_dict.get("host_group"): - filters.add(Host.host_group_name.in_(filter_dict["host_group"])) - if filter_dict.get("repo"): -- filters.add(Host.repo_name.in_(filter_dict["repo"])) -+ if all(filter_dict.get("repo")): -+ filters.add(Host.repo_name.in_(filter_dict["repo"])) -+ else: -+ repo_names = list(filter(None, filter_dict["repo"])) -+ filters.add(or_(Host.repo_name.in_(repo_names), Host.repo_name == None)) - return filters - - def _query_cve_hosts(self, username: str, cve_id: str, filters: set): -diff --git a/apollo/function/schema/cve.py b/apollo/function/schema/cve.py -index 178672e..56d76ed 100644 ---- a/apollo/function/schema/cve.py -+++ b/apollo/function/schema/cve.py -@@ -56,7 +56,7 @@ class CveHostFilterSchema(Schema): - - host_name = fields.String(required=False, validate=lambda s: len(s) != 0) - host_group = fields.List(fields.String(validate=lambda s: len(s) != 0), required=False) -- repo = fields.List(fields.String(validate=lambda s: len(s) != 0), required=False) -+ repo = fields.List(fields.String(validate=lambda s: len(s) != 0, allow_none=True), required=False) - fixed = fields.Boolean(required=True, validate=validate.OneOf([True, False])) - - -diff --git a/apollo/function/schema/task.py b/apollo/function/schema/task.py -index de86194..e136182 100644 ---- a/apollo/function/schema/task.py -+++ b/apollo/function/schema/task.py -@@ -18,7 +18,7 @@ Description: For task related restful interfaces schema - from marshmallow import Schema - from marshmallow import fields - from marshmallow import validate --from vulcanus.restful.serialize.validate import PaginationSchema -+from vulcanus.restful.serialize.validate import PaginationSchema, ValidateRules - - from apollo.conf.constant import TaskType, TaskStatus - -@@ -248,7 +248,7 @@ class CveFixResultCallbackSchema(Schema): - class CallbackSchma(Schema): - task_id = fields.String(required=True, validate=lambda s: 0 < len(s) <= 32) - host_id = fields.Integer(required=True, validate=lambda s: s > 0) -- host_ip = fields.IP(required=True) -+ host_ip = fields.String(required=True, validate=ValidateRules.ipv4_address_check) - host_name = fields.String(required=True, validate=lambda s: 0 < len(s) <= 50) - status = fields.String(required=True, validate=lambda s: len(s) != 0) - execution_time = fields.Integer(required=True) -@@ -270,7 +270,7 @@ class CheckItemsSchema(Schema): - class RepoSetCallbackSchema(Schema): - task_id = fields.String(required=True, validate=lambda s: 0 < len(s) <= 32) - host_id = fields.Integer(required=True, validate=lambda s: s > 0) -- host_ip = fields.IP(required=True) -+ host_ip = fields.String(required=True, validate=ValidateRules.ipv4_address_check) - host_name = fields.String(required=True, validate=lambda s: 0 < len(s) <= 50) - status = fields.String(required=True, validate=lambda s: len(s) != 0) - execution_time = fields.Integer(required=True) --- -2.33.0 - diff --git a/0004-fix-TimedCorrectTask.patch b/0004-fix-TimedCorrectTask.patch deleted file mode 100644 index 2ecce9b969999bb252f0e49cd497118249902acc..0000000000000000000000000000000000000000 --- a/0004-fix-TimedCorrectTask.patch +++ /dev/null @@ -1,168 +0,0 @@ -From 8e19d92b6a484ddcf7ca7bf666ce21baa56ab326 Mon Sep 17 00:00:00 2001 -From: rearcher <123781007@qq.com> -Date: Wed, 20 Dec 2023 17:21:22 +0800 -Subject: [PATCH] fix TimedCorrectTask - ---- - apollo/cron/timed_correct_manager.py | 12 ++++-- - apollo/database/proxy/task/base.py | 50 ++++++++++++++++++----- - apollo/database/proxy/task/timed_proxy.py | 8 ++++ - apollo/tests/database/test_task.py | 2 +- - 4 files changed, 57 insertions(+), 15 deletions(-) - -diff --git a/apollo/cron/timed_correct_manager.py b/apollo/cron/timed_correct_manager.py -index db0b4c1..ae3a1e3 100644 ---- a/apollo/cron/timed_correct_manager.py -+++ b/apollo/cron/timed_correct_manager.py -@@ -41,11 +41,15 @@ class TimedCorrectTask(TimedTask): - """ - Start the correct after the specified time of day. - """ -- LOGGER.info("Begin to correct the whole host in %s.", str(datetime.datetime.now())) -+ LOGGER.info( -+ "Begin to correct the status of timeout tasks and scan timeout host in %s.", -+ str(datetime.datetime.now())) - abnormal_task_ids, abnormal_host_ids = self.get_abnormal_task() -- self._update_host_status(abnormal_host_ids) -- with TimedProxy() as proxy: -- proxy.timed_correct_error_task_status(abnormal_task_ids) -+ if len(abnormal_host_ids) != 0: -+ self._update_host_status(abnormal_host_ids) -+ if len(abnormal_task_ids) != 0: -+ with TimedProxy() as proxy: -+ proxy.timed_correct_error_task_status(abnormal_task_ids) - - @staticmethod - def _abnormal_task(tasks): -diff --git a/apollo/database/proxy/task/base.py b/apollo/database/proxy/task/base.py -index 840c140..a5ddede 100644 ---- a/apollo/database/proxy/task/base.py -+++ b/apollo/database/proxy/task/base.py -@@ -861,17 +861,17 @@ class TaskProxy(TaskMysqlProxy, TaskEsProxy): - - raise EsOperationError("Delete task from elasticsearch failed due to internal error.") - -- def get_running_task_form_task_cve_host(self) -> list: -+ def get_running_task_form_hotpatch_remove_task(self) -> list: - """ -- Get all CVE repair tasks with running status under Username -+ Get all hotpatch remove tasks with running status under Username - - Returns: - list: task id list - """ -- task_cve_query = ( -- self.session.query(HotpatchRemoveTask).filter(HotpatchRemoveTask.status == TaskStatus.RUNNING).all() -+ hotpatch_remove_query = ( -+ self.session.query(HotpatchRemoveTask.task_id).filter(HotpatchRemoveTask.status == TaskStatus.RUNNING).all() - ) -- task_id_list = [task.task_id for task in task_cve_query] -+ task_id_list = [task.task_id for task in hotpatch_remove_query] - return task_id_list - - def get_running_task_form_task_host_repo(self) -> list: -@@ -882,13 +882,39 @@ class TaskProxy(TaskMysqlProxy, TaskEsProxy): - list: task id list - """ - host_repo_query = ( -- self.session.query(TaskHostRepoAssociation) -+ self.session.query(TaskHostRepoAssociation.task_id) - .filter(TaskHostRepoAssociation.status == TaskStatus.RUNNING) - .all() - ) - task_id_list = [task.task_id for task in host_repo_query] - return task_id_list - -+ def get_running_task_form_cve_fix_task(self) -> list: -+ """ -+ Get all CVE fix tasks with running status -+ -+ Returns: -+ list: task id list -+ """ -+ cve_fix_query = ( -+ self.session.query(CveFixTask.task_id).filter(CveFixTask.status == TaskStatus.RUNNING).all() -+ ) -+ task_id_list = [task.task_id for task in cve_fix_query] -+ return task_id_list -+ -+ def get_running_task_form_cve_rollback_task(self) -> list: -+ """ -+ Get all CVE rollback tasks with running status -+ -+ Returns: -+ list: task id list -+ """ -+ cve_rollback_query = ( -+ self.session.query(CveRollbackTask.task_id).filter(CveRollbackTask.status == TaskStatus.RUNNING).all() -+ ) -+ task_id_list = [task.task_id for task in cve_rollback_query] -+ return task_id_list -+ - def get_scanning_status_and_time_from_host(self) -> list: - """ - Get all host id and time with scanning status from the host table -@@ -907,13 +933,17 @@ class TaskProxy(TaskMysqlProxy, TaskEsProxy): - Returns: - list: Each element is a task information, including the task ID, task type, creation time - """ -- task_cve_id_list = self.get_running_task_form_task_cve_host() -- task_repo_id_list = self.get_running_task_form_task_host_repo() - host_info_list = self.get_scanning_status_and_time_from_host() -- task_id_list = task_cve_id_list + task_repo_id_list -+ -+ task_cve_id_list = self.get_running_task_form_hotpatch_remove_task() -+ task_repo_id_list = self.get_running_task_form_task_host_repo() -+ task_cve_fix_list = self.get_running_task_form_cve_fix_task() -+ task_cve_rollback_list = self.get_running_task_form_cve_rollback_task() -+ -+ task_id_list = task_cve_id_list + task_repo_id_list + task_cve_fix_list + task_cve_rollback_list - - task_query = self.session.query(Task).filter(Task.task_id.in_(task_id_list)).all() -- running_task_list = [(task.task_id, task.create_time) for task in task_query] -+ running_task_list = [(task.task_id, task.latest_execute_time) for task in task_query] - return running_task_list, host_info_list - - def validate_cves(self, cve_id: list) -> bool: -diff --git a/apollo/database/proxy/task/timed_proxy.py b/apollo/database/proxy/task/timed_proxy.py -index 436c3bd..fd396d1 100644 ---- a/apollo/database/proxy/task/timed_proxy.py -+++ b/apollo/database/proxy/task/timed_proxy.py -@@ -22,6 +22,8 @@ from apollo.conf.constant import TaskStatus - from apollo.database.table import ( - HotpatchRemoveTask, - TaskHostRepoAssociation, -+ CveFixTask, -+ CveRollbackTask, - ) - - -@@ -42,6 +44,12 @@ class TimedProxy(MysqlProxy): - self.session.query(TaskHostRepoAssociation).filter(TaskHostRepoAssociation.task_id.in_(task_ids)).update( - {TaskHostRepoAssociation.status: TaskStatus.UNKNOWN}, synchronize_session=False - ) -+ self.session.query(CveFixTask).filter(CveFixTask.task_id.in_(task_ids)).update( -+ {CveFixTask.status: TaskStatus.UNKNOWN}, synchronize_session=False -+ ) -+ self.session.query(CveRollbackTask).filter(CveRollbackTask.task_id.in_(task_ids)).update( -+ {CveRollbackTask.status: TaskStatus.UNKNOWN}, synchronize_session=False -+ ) - self.session.commit() - except SQLAlchemyError as error: - self.session.rollback() -diff --git a/apollo/tests/database/test_task.py b/apollo/tests/database/test_task.py -index 35d923b..ceb84ab 100644 ---- a/apollo/tests/database/test_task.py -+++ b/apollo/tests/database/test_task.py -@@ -401,7 +401,7 @@ class TestTaskMysqlFirst(DatabaseTestCase): - - def test_get_running_task_form_task_cve_host(self): - self.assertEqual( -- self.task_database.get_running_task_form_task_cve_host(), -+ self.task_database.get_running_task_form_hotpatch_remove_task(), - ["1111111111poiuytrewqasdfghjklmnb"], - ) - --- -2.33.0 - diff --git a/0004-fix-repo-query-error-and-adjust-schema.patch b/0004-fix-repo-query-error-and-adjust-schema.patch new file mode 100644 index 0000000000000000000000000000000000000000..d128d1c3bd2c41e36bad7db8594f2ca04b2d1778 --- /dev/null +++ b/0004-fix-repo-query-error-and-adjust-schema.patch @@ -0,0 +1,115 @@ +From fde9a0d9ed3b3ad99fdde047d9b8928e97e9af2f Mon Sep 17 00:00:00 2001 +From: rabbitali +Date: Sat, 7 Sep 2024 16:52:45 +0800 +Subject: [PATCH 1/1] Fixed the error of the repo query interface; Adaptable to Copilot adjustments +MIME-Version: 1.0 +Content-Type: text/plain; charset=UTF-8 +Content-Transfer-Encoding: 8bit + +--- + apollo/database/proxy/task/base.py | 1 + + apollo/function/schema/cve.py | 2 +- + apollo/function/schema/host.py | 4 +++- + apollo/function/schema/repo.py | 1 - + apollo/function/schema/task.py | 4 +++- + apollo/handler/repo_handler/view.py | 15 ++------------- + 6 files changed, 10 insertions(+), 17 deletions(-) + +diff --git a/apollo/database/proxy/task/base.py b/apollo/database/proxy/task/base.py +index 8d86614..3f9a0ba 100644 +--- a/apollo/database/proxy/task/base.py ++++ b/apollo/database/proxy/task/base.py +@@ -504,6 +504,7 @@ class TaskMysqlProxy(MysqlProxy): + "takeover": row.takeover, + "cluster_id": row.cluster_id, + "cluster_name": cluster_dict_info.get(row.cluster_id), ++ "task_type": row.task_type, + } + return task_info + +diff --git a/apollo/function/schema/cve.py b/apollo/function/schema/cve.py +index b5a0270..4a18916 100644 +--- a/apollo/function/schema/cve.py ++++ b/apollo/function/schema/cve.py +@@ -89,7 +89,7 @@ class CveTaskHostSchemaOfCveInfo(Schema): + """ + + cve_id = fields.String(required=True, validate=lambda s: 0 < len(s) <= 20) +- rpms = fields.List(fields.Nested(PackageInfoSchema), required=True) ++ rpms = fields.List(fields.Nested(PackageInfoSchema), required=False, missing=[]) + + + class GetCveTaskHostSchema(Schema): +diff --git a/apollo/function/schema/host.py b/apollo/function/schema/host.py +index bccdb65..684a55c 100644 +--- a/apollo/function/schema/host.py ++++ b/apollo/function/schema/host.py +@@ -37,7 +37,9 @@ class ScanHostSchema(Schema): + validators for parameter of /vulnerability/host/scan + """ + +- host_list = fields.List(fields.String(validate=lambda s: 0 < len(s) <= 36, required=True), required=True) ++ host_list = fields.List( ++ fields.String(validate=lambda s: 0 < len(s) <= 36, required=True), required=False, missing=[] ++ ) + filter = fields.Nested(ScanHostFilterSchema, required=False) + + +diff --git a/apollo/function/schema/repo.py b/apollo/function/schema/repo.py +index bdd4c3d..618470f 100644 +--- a/apollo/function/schema/repo.py ++++ b/apollo/function/schema/repo.py +@@ -34,7 +34,6 @@ class GetYumRepoSchema(Schema): + """ + + repo_id_list = fields.List(fields.String(validate=lambda s: 0 < len(s) <= 36), required=False) +- search_key = fields.String(required=False, validate=lambda s: 0 < len(s) <= 32) + + + class UpdateYumRepoSchema(Schema): +diff --git a/apollo/function/schema/task.py b/apollo/function/schema/task.py +index f306cea..be116ad 100644 +--- a/apollo/function/schema/task.py ++++ b/apollo/function/schema/task.py +@@ -93,7 +93,9 @@ class GenerateCveTaskSchema(Schema): + task_name = fields.String(required=True, validate=lambda s: 0 < len(s) <= 20) + description = fields.String(required=True, validate=lambda s: 0 < len(s) <= 100) + accepted = fields.Boolean(required=True, validate=validate.OneOf([True, False])) +- check_items = fields.List(fields.String(required=True, validate=lambda s: 0 < len(s) <= 32), required=False) ++ check_items = fields.List( ++ fields.String(required=True, validate=lambda s: 0 < len(s) <= 32), required=False, missing=[] ++ ) + takeover = fields.Boolean(required=True, validate=validate.OneOf([True, False])) + info = fields.List(fields.Nested(CveInfoDictSchema), required=True, validate=lambda s: len(s) > 0) + +diff --git a/apollo/handler/repo_handler/view.py b/apollo/handler/repo_handler/view.py +index 6a06c1d..6909e24 100644 +--- a/apollo/handler/repo_handler/view.py ++++ b/apollo/handler/repo_handler/view.py +@@ -93,21 +93,10 @@ class VulGetYumRepo(BaseResponse): + Query repo info handle + """ + cluster_info_dic = cache.get_user_clusters() +- if cluster_info_dic is None: +- return DATABASE_QUERY_ERROR, [] +- +- cluster_list = [] +- if params.get("search_key"): +- for cluster_id, info in cluster_info_dic.items(): +- if params.get("search_key") in info.get("cluster_name"): +- cluster_list.append(cluster_id) +- else: +- cluster_list = list(cluster_info_dic.keys()) +- +- if not cluster_list: ++ if not cluster_info_dic: + return SUCCEED, [] + +- status_code, result = proxy.get_repo(params.get("repo_id_list", []), cluster_list) ++ status_code, result = proxy.get_repo(params.get("repo_id_list", []), list(cluster_info_dic.keys())) + if status_code != SUCCEED: + return status_code, [] + +-- +2.33.0 + diff --git a/0005-add-reboot-field-to-query-host-info-api.patch b/0005-add-reboot-field-to-query-host-info-api.patch deleted file mode 100644 index de9c2755714f139439fa62c53b9f236c3d2e81ae..0000000000000000000000000000000000000000 --- a/0005-add-reboot-field-to-query-host-info-api.patch +++ /dev/null @@ -1,45 +0,0 @@ -From 5c803953928ad16d56795dcf7158bb9f3d340e1d Mon Sep 17 00:00:00 2001 -From: gongzt -Date: Thu, 21 Dec 2023 17:33:23 +0800 -Subject: [PATCH] add reboot field to query host info api -MIME-Version: 1.0 -Content-Type: text/plain; charset=UTF-8 -Content-Transfer-Encoding: 8bit - ---- - apollo/database/proxy/host.py | 5 ++++- - 1 file changed, 4 insertions(+), 1 deletion(-) - -diff --git a/apollo/database/proxy/host.py b/apollo/database/proxy/host.py -index ca709a8..6015a04 100644 ---- a/apollo/database/proxy/host.py -+++ b/apollo/database/proxy/host.py -@@ -341,7 +341,8 @@ class HostMysqlProxy(MysqlProxy): - "repo": "20.03-update", - "affected_cve_num": 12, - "unaffected_cve_num": 1, -- "last_scan": 1111111111 -+ "last_scan": 1111111111, -+ "reboot": true/false - } - } - """ -@@ -412,6 +413,7 @@ class HostMysqlProxy(MysqlProxy): - Host.host_group_name, - Host.repo_name, - Host.last_scan, -+ Host.reboot, - func.COUNT(func.IF(subquery.c.fixed == True, 1, None)).label("fixed_cve_num"), - func.COUNT(func.IF(and_(subquery.c.fixed == False, subquery.c.affected == True), 1, None)).label( - "affected_cve_num" -@@ -437,6 +439,7 @@ class HostMysqlProxy(MysqlProxy): - "unaffected_cve_num": row.unaffected_cve_num, - "last_scan": row.last_scan, - "fixed_cve_num": row.fixed_cve_num, -+ "reboot": row.reboot, - } - return host_info - --- -Gitee - diff --git a/0005-set-uwsgi-buffer-size.patch b/0005-set-uwsgi-buffer-size.patch new file mode 100644 index 0000000000000000000000000000000000000000..193302efb2cdac4a00b7befe6e1acc055c8dda5c --- /dev/null +++ b/0005-set-uwsgi-buffer-size.patch @@ -0,0 +1,26 @@ +From 8b2b1f8eb4cf6a6b47103acee81d218022e3166a Mon Sep 17 00:00:00 2001 +From: rearcher <123781007@qq.com> +Date: Mon, 18 Nov 2024 15:50:54 +0800 +Subject: [PATCH] =?UTF-8?q?=E8=AE=BE=E7=BD=AEuwsgi=20buffer-size=E4=B8=BA3?= + =?UTF-8?q?2k?= +MIME-Version: 1.0 +Content-Type: text/plain; charset=UTF-8 +Content-Transfer-Encoding: 8bit + +--- + conf/aops-apollo.yml | 1 + + 1 file changed, 1 insertion(+) + +diff --git a/conf/aops-apollo.yml b/conf/aops-apollo.yml +index f7a39c4..1f34aa4 100644 +--- a/conf/aops-apollo.yml ++++ b/conf/aops-apollo.yml +@@ -4,3 +4,4 @@ uwsgi: + processes: 1 + gevent: 100 + port: 11116 ++ buffer_size: 32768 +\ No newline at end of file +-- +Gitee + diff --git a/aops-apollo-v1.4.1.tar.gz b/aops-apollo-v1.4.1.tar.gz deleted file mode 100644 index 872f093206dcea104f101a2c28a86c66c77545a8..0000000000000000000000000000000000000000 Binary files a/aops-apollo-v1.4.1.tar.gz and /dev/null differ diff --git a/aops-apollo-v2.0.0.tar.gz b/aops-apollo-v2.0.0.tar.gz new file mode 100644 index 0000000000000000000000000000000000000000..d5fae3e9be8396fffd4602f79c5614d1228d848c Binary files /dev/null and b/aops-apollo-v2.0.0.tar.gz differ diff --git a/aops-apollo.spec b/aops-apollo.spec index 6273363f230db29d719ce8f5cb84599408cc2711..d75df1e09eef12b9f6b587f081610b8d944a946c 100644 --- a/aops-apollo.spec +++ b/aops-apollo.spec @@ -1,23 +1,19 @@ Name: aops-apollo -Version: v1.4.1 -Release: 4 +Version: v2.0.0 +Release: 6 Summary: Cve management service, monitor machine vulnerabilities and provide fix functions. License: MulanPSL2 URL: https://gitee.com/openeuler/%{name} Source0: %{name}-%{version}.tar.gz -Patch0001: 0001-fix-cve-list-sort.patch -Patch0002: 0002-fix-the-query-error-of-cve-associated-host.patch -Patch0003: 0003-update-verification-method-for-host-ip-fieldl.patch -Patch0004: 0004-fix-TimedCorrectTask.patch -Patch0005: 0005-add-reboot-field-to-query-host-info-api.patch -Patch0006: 0006-fix-bug-with-host-count-in-cve-fix-task.patch +Patch0001: 0001-fix-security-advisory-parsing-error.patch +Patch0002: 0002-fix-bug-with-host-count-in-cve-fix-task.patch +Patch0003: 0003-fix-issue-with-language-display-in-task-generation.patch +Patch0004: 0004-fix-repo-query-error-and-adjust-schema.patch +Patch0005: 0005-set-uwsgi-buffer-size.patch BuildRequires: python3-setuptools -Requires: aops-vulcanus >= v1.3.0 -Requires: python3-elasticsearch >= 7 python3-flask-restful python3-marshmallow >= 3.13.0 -Requires: python3-sqlalchemy python3-PyMySQL python3-Flask-APScheduler >= 1.11.0 -Requires: python3-PyYAML python3-flask python3-gevent python3-uWSGI -Requires: python3-retrying python3-lxml +Requires: aops-vulcanus >= v2.0.0 +Requires: python3-gevent python3-uWSGI python3-celery aops-zeus >= v2.0.0 Provides: aops-apollo @@ -57,10 +53,8 @@ popd %files %doc README.* -%attr(0644,root,root) %{_sysconfdir}/aops/apollo.ini -%attr(0644,root,root) %{_sysconfdir}/aops/apollo_crontab.yml -%attr(0755,root,root) %{_bindir}/aops-apollo -%attr(0755,root,root) /usr/lib/systemd/system/aops-apollo.service +%attr(0644,root,root) %{_sysconfdir}/aops/conf.d/aops-apollo.yml +%attr(0755,root,root) %{_unitdir}/aops-apollo.service %{python3_sitelib}/aops_apollo*.egg-info/* %{python3_sitelib}/apollo/* %attr(0755, root, root) /opt/aops/database/* @@ -73,8 +67,19 @@ popd %{python3_sitelib}/aops_apollo_tool/* %changelog -* Wed Jul 24 2024 wenxin - v1.4.1-4 -- fix bug with host count in cve fix task +* Tue Nov 19 2024 luxuexian - v2.0.0-6 +- set-uwsgi-buffer-size to 32k + +* Mon Sep 09 2024 wenxin - v2.0.0-5 +- Fix issue with querying repo info api +- Adjust some schema validation logic + +* Fri Aug 30 2024 wenxin - v2.0.0-4 +- Fix issue with language display in task generation api + +* Fri Aug 16 2024 wenxin - v2.0.0-3 +- Added support for cluster features. +- Adjusted Task Module logic to use Celery for task management and execution. * Fri Dec 22 2023 wenxin - v1.4.1-3 - fix the query error of cve associated host