From 4901cb39a37166a4b4b6322d5e76b787a83af90d Mon Sep 17 00:00:00 2001 From: rabbitali Date: Thu, 21 Dec 2023 14:13:24 +0800 Subject: [PATCH] fix issue:#I8PC69, #I8PUNT,#I8PZZU (cherry picked from commit 85f3a85f8a43729aad017e67a95201ba66ed6711) --- ...e-query-error-of-cve-associated-host.patch | 27 +++ ...rification-method-for-host-ip-fieldl.patch | 85 +++++++++ 0004-fix-TimedCorrectTask.patch | 168 ++++++++++++++++++ ...-reboot-field-to-query-host-info-api.patch | 45 +++++ aops-apollo.spec | 11 +- 5 files changed, 335 insertions(+), 1 deletion(-) create mode 100644 0002-fix-the-query-error-of-cve-associated-host.patch create mode 100644 0003-update-verification-method-for-host-ip-fieldl.patch create mode 100644 0004-fix-TimedCorrectTask.patch create mode 100644 0005-add-reboot-field-to-query-host-info-api.patch diff --git a/0002-fix-the-query-error-of-cve-associated-host.patch b/0002-fix-the-query-error-of-cve-associated-host.patch new file mode 100644 index 0000000..ff863c1 --- /dev/null +++ b/0002-fix-the-query-error-of-cve-associated-host.patch @@ -0,0 +1,27 @@ +From f8cab408f39bb8f6b793021cdfbf93338fb6ed0a Mon Sep 17 00:00:00 2001 +From: gongzt +Date: Wed, 20 Dec 2023 15:13:07 +0800 +Subject: [PATCH] Fix the query error of cve associated host +MIME-Version: 1.0 +Content-Type: text/plain; charset=UTF-8 +Content-Transfer-Encoding: 8bit + +--- + apollo/database/proxy/cve.py | 1 + + 1 file changed, 1 insertion(+) + +diff --git a/apollo/database/proxy/cve.py b/apollo/database/proxy/cve.py +index 6210156..b5a61b9 100644 +--- a/apollo/database/proxy/cve.py ++++ b/apollo/database/proxy/cve.py +@@ -1532,6 +1532,7 @@ class CveProxy(CveMysqlProxy, CveEsProxy): + def _get_processed_cve_packages_host(self, data): + result = {"total_count": 0, "total_page": 0, "result": []} + filters = { ++ CveHostAssociation.host_user == data["username"], + CveHostAssociation.cve_id == data["cve_id"], + CveHostAssociation.installed_rpm == data["installed_rpm"], + CveHostAssociation.fixed == data["fixed"], +-- +2.33.0 + diff --git a/0003-update-verification-method-for-host-ip-fieldl.patch b/0003-update-verification-method-for-host-ip-fieldl.patch new file mode 100644 index 0000000..c40bab7 --- /dev/null +++ b/0003-update-verification-method-for-host-ip-fieldl.patch @@ -0,0 +1,85 @@ +From 47a4c1d6488f07aa55621454fefedb559fc1bbf8 Mon Sep 17 00:00:00 2001 +From: rabbitali +Date: Wed, 20 Dec 2023 16:26:24 +0800 +Subject: [PATCH] update verification method for host ip fieldl;fix repo + field filter error + +--- + apollo/database/proxy/cve.py | 8 ++++++-- + apollo/function/schema/cve.py | 2 +- + apollo/function/schema/task.py | 6 +++--- + 3 files changed, 10 insertions(+), 6 deletions(-) + +diff --git a/apollo/database/proxy/cve.py b/apollo/database/proxy/cve.py +index 6210156..5d29544 100644 +--- a/apollo/database/proxy/cve.py ++++ b/apollo/database/proxy/cve.py +@@ -20,7 +20,7 @@ import copy + from collections import defaultdict + + from elasticsearch import ElasticsearchException +-from sqlalchemy import func, tuple_, case ++from sqlalchemy import func, tuple_, case, or_ + from sqlalchemy.exc import SQLAlchemyError + from vulcanus.database.helper import sort_and_page, judge_return_code + from vulcanus.database.proxy import MysqlProxy, ElasticsearchProxy +@@ -200,7 +200,11 @@ class CveMysqlProxy(MysqlProxy): + if filter_dict.get("host_group"): + filters.add(Host.host_group_name.in_(filter_dict["host_group"])) + if filter_dict.get("repo"): +- filters.add(Host.repo_name.in_(filter_dict["repo"])) ++ if all(filter_dict.get("repo")): ++ filters.add(Host.repo_name.in_(filter_dict["repo"])) ++ else: ++ repo_names = list(filter(None, filter_dict["repo"])) ++ filters.add(or_(Host.repo_name.in_(repo_names), Host.repo_name == None)) + return filters + + def _query_cve_hosts(self, username: str, cve_id: str, filters: set): +diff --git a/apollo/function/schema/cve.py b/apollo/function/schema/cve.py +index 178672e..56d76ed 100644 +--- a/apollo/function/schema/cve.py ++++ b/apollo/function/schema/cve.py +@@ -56,7 +56,7 @@ class CveHostFilterSchema(Schema): + + host_name = fields.String(required=False, validate=lambda s: len(s) != 0) + host_group = fields.List(fields.String(validate=lambda s: len(s) != 0), required=False) +- repo = fields.List(fields.String(validate=lambda s: len(s) != 0), required=False) ++ repo = fields.List(fields.String(validate=lambda s: len(s) != 0, allow_none=True), required=False) + fixed = fields.Boolean(required=True, validate=validate.OneOf([True, False])) + + +diff --git a/apollo/function/schema/task.py b/apollo/function/schema/task.py +index de86194..e136182 100644 +--- a/apollo/function/schema/task.py ++++ b/apollo/function/schema/task.py +@@ -18,7 +18,7 @@ Description: For task related restful interfaces schema + from marshmallow import Schema + from marshmallow import fields + from marshmallow import validate +-from vulcanus.restful.serialize.validate import PaginationSchema ++from vulcanus.restful.serialize.validate import PaginationSchema, ValidateRules + + from apollo.conf.constant import TaskType, TaskStatus + +@@ -248,7 +248,7 @@ class CveFixResultCallbackSchema(Schema): + class CallbackSchma(Schema): + task_id = fields.String(required=True, validate=lambda s: 0 < len(s) <= 32) + host_id = fields.Integer(required=True, validate=lambda s: s > 0) +- host_ip = fields.IP(required=True) ++ host_ip = fields.String(required=True, validate=ValidateRules.ipv4_address_check) + host_name = fields.String(required=True, validate=lambda s: 0 < len(s) <= 50) + status = fields.String(required=True, validate=lambda s: len(s) != 0) + execution_time = fields.Integer(required=True) +@@ -270,7 +270,7 @@ class CheckItemsSchema(Schema): + class RepoSetCallbackSchema(Schema): + task_id = fields.String(required=True, validate=lambda s: 0 < len(s) <= 32) + host_id = fields.Integer(required=True, validate=lambda s: s > 0) +- host_ip = fields.IP(required=True) ++ host_ip = fields.String(required=True, validate=ValidateRules.ipv4_address_check) + host_name = fields.String(required=True, validate=lambda s: 0 < len(s) <= 50) + status = fields.String(required=True, validate=lambda s: len(s) != 0) + execution_time = fields.Integer(required=True) +-- +2.33.0 + diff --git a/0004-fix-TimedCorrectTask.patch b/0004-fix-TimedCorrectTask.patch new file mode 100644 index 0000000..2ecce9b --- /dev/null +++ b/0004-fix-TimedCorrectTask.patch @@ -0,0 +1,168 @@ +From 8e19d92b6a484ddcf7ca7bf666ce21baa56ab326 Mon Sep 17 00:00:00 2001 +From: rearcher <123781007@qq.com> +Date: Wed, 20 Dec 2023 17:21:22 +0800 +Subject: [PATCH] fix TimedCorrectTask + +--- + apollo/cron/timed_correct_manager.py | 12 ++++-- + apollo/database/proxy/task/base.py | 50 ++++++++++++++++++----- + apollo/database/proxy/task/timed_proxy.py | 8 ++++ + apollo/tests/database/test_task.py | 2 +- + 4 files changed, 57 insertions(+), 15 deletions(-) + +diff --git a/apollo/cron/timed_correct_manager.py b/apollo/cron/timed_correct_manager.py +index db0b4c1..ae3a1e3 100644 +--- a/apollo/cron/timed_correct_manager.py ++++ b/apollo/cron/timed_correct_manager.py +@@ -41,11 +41,15 @@ class TimedCorrectTask(TimedTask): + """ + Start the correct after the specified time of day. + """ +- LOGGER.info("Begin to correct the whole host in %s.", str(datetime.datetime.now())) ++ LOGGER.info( ++ "Begin to correct the status of timeout tasks and scan timeout host in %s.", ++ str(datetime.datetime.now())) + abnormal_task_ids, abnormal_host_ids = self.get_abnormal_task() +- self._update_host_status(abnormal_host_ids) +- with TimedProxy() as proxy: +- proxy.timed_correct_error_task_status(abnormal_task_ids) ++ if len(abnormal_host_ids) != 0: ++ self._update_host_status(abnormal_host_ids) ++ if len(abnormal_task_ids) != 0: ++ with TimedProxy() as proxy: ++ proxy.timed_correct_error_task_status(abnormal_task_ids) + + @staticmethod + def _abnormal_task(tasks): +diff --git a/apollo/database/proxy/task/base.py b/apollo/database/proxy/task/base.py +index 840c140..a5ddede 100644 +--- a/apollo/database/proxy/task/base.py ++++ b/apollo/database/proxy/task/base.py +@@ -861,17 +861,17 @@ class TaskProxy(TaskMysqlProxy, TaskEsProxy): + + raise EsOperationError("Delete task from elasticsearch failed due to internal error.") + +- def get_running_task_form_task_cve_host(self) -> list: ++ def get_running_task_form_hotpatch_remove_task(self) -> list: + """ +- Get all CVE repair tasks with running status under Username ++ Get all hotpatch remove tasks with running status under Username + + Returns: + list: task id list + """ +- task_cve_query = ( +- self.session.query(HotpatchRemoveTask).filter(HotpatchRemoveTask.status == TaskStatus.RUNNING).all() ++ hotpatch_remove_query = ( ++ self.session.query(HotpatchRemoveTask.task_id).filter(HotpatchRemoveTask.status == TaskStatus.RUNNING).all() + ) +- task_id_list = [task.task_id for task in task_cve_query] ++ task_id_list = [task.task_id for task in hotpatch_remove_query] + return task_id_list + + def get_running_task_form_task_host_repo(self) -> list: +@@ -882,13 +882,39 @@ class TaskProxy(TaskMysqlProxy, TaskEsProxy): + list: task id list + """ + host_repo_query = ( +- self.session.query(TaskHostRepoAssociation) ++ self.session.query(TaskHostRepoAssociation.task_id) + .filter(TaskHostRepoAssociation.status == TaskStatus.RUNNING) + .all() + ) + task_id_list = [task.task_id for task in host_repo_query] + return task_id_list + ++ def get_running_task_form_cve_fix_task(self) -> list: ++ """ ++ Get all CVE fix tasks with running status ++ ++ Returns: ++ list: task id list ++ """ ++ cve_fix_query = ( ++ self.session.query(CveFixTask.task_id).filter(CveFixTask.status == TaskStatus.RUNNING).all() ++ ) ++ task_id_list = [task.task_id for task in cve_fix_query] ++ return task_id_list ++ ++ def get_running_task_form_cve_rollback_task(self) -> list: ++ """ ++ Get all CVE rollback tasks with running status ++ ++ Returns: ++ list: task id list ++ """ ++ cve_rollback_query = ( ++ self.session.query(CveRollbackTask.task_id).filter(CveRollbackTask.status == TaskStatus.RUNNING).all() ++ ) ++ task_id_list = [task.task_id for task in cve_rollback_query] ++ return task_id_list ++ + def get_scanning_status_and_time_from_host(self) -> list: + """ + Get all host id and time with scanning status from the host table +@@ -907,13 +933,17 @@ class TaskProxy(TaskMysqlProxy, TaskEsProxy): + Returns: + list: Each element is a task information, including the task ID, task type, creation time + """ +- task_cve_id_list = self.get_running_task_form_task_cve_host() +- task_repo_id_list = self.get_running_task_form_task_host_repo() + host_info_list = self.get_scanning_status_and_time_from_host() +- task_id_list = task_cve_id_list + task_repo_id_list ++ ++ task_cve_id_list = self.get_running_task_form_hotpatch_remove_task() ++ task_repo_id_list = self.get_running_task_form_task_host_repo() ++ task_cve_fix_list = self.get_running_task_form_cve_fix_task() ++ task_cve_rollback_list = self.get_running_task_form_cve_rollback_task() ++ ++ task_id_list = task_cve_id_list + task_repo_id_list + task_cve_fix_list + task_cve_rollback_list + + task_query = self.session.query(Task).filter(Task.task_id.in_(task_id_list)).all() +- running_task_list = [(task.task_id, task.create_time) for task in task_query] ++ running_task_list = [(task.task_id, task.latest_execute_time) for task in task_query] + return running_task_list, host_info_list + + def validate_cves(self, cve_id: list) -> bool: +diff --git a/apollo/database/proxy/task/timed_proxy.py b/apollo/database/proxy/task/timed_proxy.py +index 436c3bd..fd396d1 100644 +--- a/apollo/database/proxy/task/timed_proxy.py ++++ b/apollo/database/proxy/task/timed_proxy.py +@@ -22,6 +22,8 @@ from apollo.conf.constant import TaskStatus + from apollo.database.table import ( + HotpatchRemoveTask, + TaskHostRepoAssociation, ++ CveFixTask, ++ CveRollbackTask, + ) + + +@@ -42,6 +44,12 @@ class TimedProxy(MysqlProxy): + self.session.query(TaskHostRepoAssociation).filter(TaskHostRepoAssociation.task_id.in_(task_ids)).update( + {TaskHostRepoAssociation.status: TaskStatus.UNKNOWN}, synchronize_session=False + ) ++ self.session.query(CveFixTask).filter(CveFixTask.task_id.in_(task_ids)).update( ++ {CveFixTask.status: TaskStatus.UNKNOWN}, synchronize_session=False ++ ) ++ self.session.query(CveRollbackTask).filter(CveRollbackTask.task_id.in_(task_ids)).update( ++ {CveRollbackTask.status: TaskStatus.UNKNOWN}, synchronize_session=False ++ ) + self.session.commit() + except SQLAlchemyError as error: + self.session.rollback() +diff --git a/apollo/tests/database/test_task.py b/apollo/tests/database/test_task.py +index 35d923b..ceb84ab 100644 +--- a/apollo/tests/database/test_task.py ++++ b/apollo/tests/database/test_task.py +@@ -401,7 +401,7 @@ class TestTaskMysqlFirst(DatabaseTestCase): + + def test_get_running_task_form_task_cve_host(self): + self.assertEqual( +- self.task_database.get_running_task_form_task_cve_host(), ++ self.task_database.get_running_task_form_hotpatch_remove_task(), + ["1111111111poiuytrewqasdfghjklmnb"], + ) + +-- +2.33.0 + diff --git a/0005-add-reboot-field-to-query-host-info-api.patch b/0005-add-reboot-field-to-query-host-info-api.patch new file mode 100644 index 0000000..de9c275 --- /dev/null +++ b/0005-add-reboot-field-to-query-host-info-api.patch @@ -0,0 +1,45 @@ +From 5c803953928ad16d56795dcf7158bb9f3d340e1d Mon Sep 17 00:00:00 2001 +From: gongzt +Date: Thu, 21 Dec 2023 17:33:23 +0800 +Subject: [PATCH] add reboot field to query host info api +MIME-Version: 1.0 +Content-Type: text/plain; charset=UTF-8 +Content-Transfer-Encoding: 8bit + +--- + apollo/database/proxy/host.py | 5 ++++- + 1 file changed, 4 insertions(+), 1 deletion(-) + +diff --git a/apollo/database/proxy/host.py b/apollo/database/proxy/host.py +index ca709a8..6015a04 100644 +--- a/apollo/database/proxy/host.py ++++ b/apollo/database/proxy/host.py +@@ -341,7 +341,8 @@ class HostMysqlProxy(MysqlProxy): + "repo": "20.03-update", + "affected_cve_num": 12, + "unaffected_cve_num": 1, +- "last_scan": 1111111111 ++ "last_scan": 1111111111, ++ "reboot": true/false + } + } + """ +@@ -412,6 +413,7 @@ class HostMysqlProxy(MysqlProxy): + Host.host_group_name, + Host.repo_name, + Host.last_scan, ++ Host.reboot, + func.COUNT(func.IF(subquery.c.fixed == True, 1, None)).label("fixed_cve_num"), + func.COUNT(func.IF(and_(subquery.c.fixed == False, subquery.c.affected == True), 1, None)).label( + "affected_cve_num" +@@ -437,6 +439,7 @@ class HostMysqlProxy(MysqlProxy): + "unaffected_cve_num": row.unaffected_cve_num, + "last_scan": row.last_scan, + "fixed_cve_num": row.fixed_cve_num, ++ "reboot": row.reboot, + } + return host_info + +-- +Gitee + diff --git a/aops-apollo.spec b/aops-apollo.spec index feb5062..7a6959e 100644 --- a/aops-apollo.spec +++ b/aops-apollo.spec @@ -1,11 +1,15 @@ Name: aops-apollo Version: v1.4.1 -Release: 2 +Release: 3 Summary: Cve management service, monitor machine vulnerabilities and provide fix functions. License: MulanPSL2 URL: https://gitee.com/openeuler/%{name} Source0: %{name}-%{version}.tar.gz Patch0001: 0001-fix-cve-list-sort.patch +Patch0002: 0002-fix-the-query-error-of-cve-associated-host.patch +Patch0003: 0003-update-verification-method-for-host-ip-fieldl.patch +Patch0004: 0004-fix-TimedCorrectTask.patch +Patch0005: 0005-add-reboot-field-to-query-host-info-api.patch BuildRequires: python3-setuptools Requires: aops-vulcanus >= v1.3.0 @@ -68,6 +72,11 @@ popd %{python3_sitelib}/aops_apollo_tool/* %changelog +* Fri Dec 22 2023 wenxin - v1.4.1-3 +- fix the query error of cve associated host +- update verification method for host ip fieldl;fix repo field filter error +- update TimedCorrectTask method + * Mon Dec 18 2023 luxuexian - v1.4.1-2 - fix cve_list sort order -- Gitee