diff --git a/0001-fix-args-not-effective-bug.patch b/0001-fix-args-not-effective-bug.patch new file mode 100644 index 0000000000000000000000000000000000000000..e14ee0b80b23735d79423e12013c9a396025b8b3 --- /dev/null +++ b/0001-fix-args-not-effective-bug.patch @@ -0,0 +1,208 @@ +From c7b84c97fde88b30f39f01e3345e2649714c2b7e Mon Sep 17 00:00:00 2001 +From: young <954906362@qq.com> +Date: Thu, 27 Apr 2023 17:25:40 +0800 +Subject: [PATCH] bug fix:continuously sending emails when can't read config info +MIME-Version: 1.0 +Content-Type: text/plain; charset=UTF-8 +Content-Transfer-Encoding: 8bit + +--- + apollo/cron/download_sa_manager.py | 43 ++++++------------- + apollo/cron/manager/__init__.py | 11 +++-- + apollo/cron/timed_correct_manager.py | 2 +- + apollo/database/proxy/cve.py | 2 +- + .../task_handler/manager/scan_manager.py | 9 ++-- + conf/apollo_crontab.ini | 3 +- + 6 files changed, 28 insertions(+), 42 deletions(-) + +diff --git a/apollo/cron/download_sa_manager.py b/apollo/cron/download_sa_manager.py +index 7b3cea2..0bb6ef8 100644 +--- a/apollo/cron/download_sa_manager.py ++++ b/apollo/cron/download_sa_manager.py +@@ -17,7 +17,6 @@ import shutil + import sqlalchemy + import requests + import retrying +-from lxml import etree + from retrying import retry + + from apollo.conf import configuration +@@ -37,16 +36,10 @@ class TimedDownloadSATask(TimedTaskBase): + Timed download sa tasks + """ + config_info = get_timed_task_config_info(TIMED_TASK_CONFIG_PATH) +- security_base_url = config_info.get( +- "download_sa", dict()).get("base_url", "") +- if not security_base_url: +- LOGGER.error("Please add base_url in configuration file") +- advisory_years = config_info.get("download_sa", dict()).get("sa_years", "") +- if advisory_years: +- advisory_years = re.findall(r"\d{4}", advisory_years) +- else: +- LOGGER.error("Please add sa_years in configuration file") +- raise KeyError ++ cvrf_url = config_info.get( ++ "download_sa", dict()).get("cvrf_url", "") ++ if not cvrf_url: ++ LOGGER.error("Please add cvrf_url in configuration file") + + save_sa_record = [] + +@@ -152,7 +145,7 @@ class TimedDownloadSATask(TimedTaskBase): + + for sa_name in sa_name_list: + advisory_year, advisory_serial_number = re.findall("\d+", sa_name) +- sa_url = f"{TimedDownloadSATask.security_base_url}/{advisory_year}/{sa_name}" ++ sa_url = f"{TimedDownloadSATask.cvrf_url}/{advisory_year}/{sa_name}" + try: + response = TimedDownloadSATask.get_response(sa_url) + if response: +@@ -170,22 +163,21 @@ class TimedDownloadSATask(TimedTaskBase): + "download_status": False}) + + @staticmethod +- def get_advisory_url_list(url: str) -> list: ++ def get_advisory_url_list() -> list: + """ + Send a request and parse the data on the page to get all the security bulletins url and store them in the list + +- Args: +- url(str): the url of all security announcements +- + Returns: + list: security url list + """ + try: +- response = TimedDownloadSATask.get_response(url) ++ response = TimedDownloadSATask.get_response(TimedDownloadSATask.cvrf_url + "/index.txt") + if response: +- html = etree.HTML(response.text) ++ html = response.text ++ sa_list = html.replace("\r", "").split("\n") + security_files = [ +- file for file in html.xpath("//*[@id='list']/tbody/tr/td[1]/a/@title") ++ # 2021/cvrf-openEuler-SA-2021-1022.xml, we don't need the first five characters ++ sa_name[5:] for sa_name in sa_list + ] + return security_files + else: +@@ -206,18 +198,7 @@ class TimedDownloadSATask(TimedTaskBase): + Returns: + list: The name of the sa that needs to be downloaded + """ +- today = datetime.datetime.today() +- current_year = str(today.year) +- TimedDownloadSATask.advisory_years.append(current_year) +- all_sa_name_list = [] +- +- for year in set(TimedDownloadSATask.advisory_years): +- try: +- sa_name_list = TimedDownloadSATask.get_advisory_url_list( +- TimedDownloadSATask.security_base_url + "/" + str(year) + "/") +- all_sa_name_list.extend(sa_name_list) +- except retrying.RetryError: +- LOGGER.error(f"Failed to get the {year} sa file") ++ all_sa_name_list = TimedDownloadSATask.get_advisory_url_list() + + succeed_sa_name_list = [] + for succeed_record in download_succeed_record: +diff --git a/apollo/cron/manager/__init__.py b/apollo/cron/manager/__init__.py +index 1ec9b72..1f3a855 100644 +--- a/apollo/cron/manager/__init__.py ++++ b/apollo/cron/manager/__init__.py +@@ -91,9 +91,12 @@ class TimedTaskManager(): + return + timed_task_parameters['id'] = task_id + +- if "base_url" in timed_task_parameters and "sa_years" in timed_task_parameters: +- timed_task_parameters.pop("base_url") +- timed_task_parameters.pop("sa_years") ++ if "day_of_week" not in timed_task_parameters or "hour" not in timed_task_parameters: ++ LOGGER.error("Create scheduled task is missing required fields.") ++ return ++ ++ if "cvrf_url" in timed_task_parameters: ++ timed_task_parameters.pop("cvrf_url") + if "service_timeout_threshold_min" in timed_task_parameters: + timed_task_parameters.pop("service_timeout_threshold_min") + +@@ -102,7 +105,7 @@ class TimedTaskManager(): + if "auto_start" in timed_task_parameters: + auto_start = timed_task_parameters['auto_start'] + timed_task_parameters.pop("auto_start") +- if not auto_start: ++ if auto_start == "false": + LOGGER.info(f"{task_id}, This task is configured to not start.") + return + TimedTaskManager._APscheduler.add_job(**timed_task_parameters) +diff --git a/apollo/cron/timed_correct_manager.py b/apollo/cron/timed_correct_manager.py +index 4f1459e..fc2f62a 100644 +--- a/apollo/cron/timed_correct_manager.py ++++ b/apollo/cron/timed_correct_manager.py +@@ -33,7 +33,7 @@ class TimedCorrectTask(TimedTaskBase): + """ + config_info = get_timed_task_config_info(TIMED_TASK_CONFIG_PATH) + SERVICE_TIMEOUT_THRESHOLD_MIN = config_info.get( +- "correct_data").get("service_timeout_threshold_min") ++ "correct_data").get("service_timeout_threshold_min", 15) + + @staticmethod + def task_enter(): +diff --git a/apollo/database/proxy/cve.py b/apollo/database/proxy/cve.py +index 1a77211..75ab71c 100644 +--- a/apollo/database/proxy/cve.py ++++ b/apollo/database/proxy/cve.py +@@ -951,7 +951,7 @@ class CveProxy(CveMysqlProxy, CveEsProxy): + """ + Save the record of download sa + Args: +- sa_record_rows(list): aach element is a record of the AdvisoryDownloadRecord table,e.g. ++ sa_record_rows(list): each element is a record of the AdvisoryDownloadRecord table,e.g. + [{"advisory_year": 2022, + "advisory_serial_number": 1230, + "download_status": 1}] +diff --git a/apollo/handler/task_handler/manager/scan_manager.py b/apollo/handler/task_handler/manager/scan_manager.py +index efd6356..48ee71b 100644 +--- a/apollo/handler/task_handler/manager/scan_manager.py ++++ b/apollo/handler/task_handler/manager/scan_manager.py +@@ -195,16 +195,19 @@ class ScanManager(Manager): + tmp = {} + chart_data = [] + file_content = "序号,CVE_ID,主机IP,主机名称,CVSS评分,评分级别\n" +- for num, row in enumerate(rows, 1): ++ excel_row_num = 1 ++ for row in rows: + if row.host_ip in tmp: + tmp[row.host_ip]["count"] += 1 +- file_content += f"{num},{row.cve_id},{row.host_ip}," \ ++ file_content += f"{excel_row_num},{row.cve_id},{row.host_ip}," \ + f"{row.host_name},{row.cvss_score},{row.severity}\n" ++ excel_row_num += 1 + else: + if row.cve_id is not None: + tmp[row.host_ip] = {"count": 1, "host_name": row.host_name} +- file_content += f"{num},{row.cve_id},{row.host_ip}," \ ++ file_content += f"{excel_row_num},{row.cve_id},{row.host_ip}," \ + f"{row.host_name},{row.cvss_score},{row.severity}\n" ++ excel_row_num += 1 + else: + tmp[row.host_ip] = {"count": 0, "host_name": row.host_name} + +diff --git a/conf/apollo_crontab.ini b/conf/apollo_crontab.ini +index 3ac8eeb..7ab1bfa 100644 +--- a/conf/apollo_crontab.ini ++++ b/conf/apollo_crontab.ini +@@ -13,8 +13,7 @@ id = download sa + day_of_week = 0-6 + hour = 3 + auto_start = true +-base_url = https://repo.openeuler.org/security/data/cvrf +-sa_years = 2021,2022,2023 ++cvrf_url = https://repo.openeuler.org/security/data/cvrf + + [correct_data] + id = correct data +-- +Gitee + + diff --git a/0002-download-SA-collaborative-process.patch b/0002-download-SA-collaborative-process.patch new file mode 100644 index 0000000000000000000000000000000000000000..e486d2394ca85eb14f2bddccee728917c922c3b3 --- /dev/null +++ b/0002-download-SA-collaborative-process.patch @@ -0,0 +1,131 @@ +From d6cd91cf4221f0b4d1b6af3d23a2989e6eb1a4c4 Mon Sep 17 00:00:00 2001 +From: young <954906362@qq.com> +Date: Fri, 28 Apr 2023 10:36:33 +0800 +Subject: [PATCH] download SA using a collaborative process +MIME-Version: 1.0 +Content-Type: text/plain; charset=UTF-8 +Content-Transfer-Encoding: 8bit + +--- + aops-apollo.spec | 2 +- + apollo/cron/download_sa_manager.py | 49 ++++++++++++++++-------------- + setup.py | 1 + + 3 files changed, 29 insertions(+), 23 deletions(-) + +diff --git a/aops-apollo.spec b/aops-apollo.spec +index 0fb2db3..2d3a771 100644 +--- a/aops-apollo.spec ++++ b/aops-apollo.spec +@@ -10,7 +10,7 @@ BuildRequires: python3-setuptools + Requires: aops-vulcanus >= v1.2.0 + Requires: python3-elasticsearch python3-flask-restful python3-marshmallow >= 3.13.0 + Requires: python3-sqlalchemy python3-PyMySQL python3-Flask-APScheduler >= 1.11.0 +-Requires: python3-PyYAML python3-flask ++Requires: python3-PyYAML python3-flask python3-gevent + Requires: python3-retrying python3-lxml + Provides: aops-apollo + +diff --git a/apollo/cron/download_sa_manager.py b/apollo/cron/download_sa_manager.py +index 0bb6ef8..6ce97bb 100644 +--- a/apollo/cron/download_sa_manager.py ++++ b/apollo/cron/download_sa_manager.py +@@ -18,6 +18,8 @@ import sqlalchemy + import requests + import retrying + from retrying import retry ++from gevent import monkey; monkey.patch_all(ssl=False) ++import gevent + + from apollo.conf import configuration + from apollo.conf.constant import ADVISORY_SAVED_PATH, TIMED_TASK_CONFIG_PATH +@@ -52,6 +54,11 @@ class TimedDownloadSATask(TimedTaskBase): + """ + LOGGER.info("Begin to download advisory in %s.", + str(datetime.datetime.now())) ++ ++ if os.path.exists(ADVISORY_SAVED_PATH): ++ shutil.rmtree(ADVISORY_SAVED_PATH) ++ os.makedirs(ADVISORY_SAVED_PATH) ++ + try: + with CveProxy(configuration) as proxy: + ElasticsearchProxy.connect(proxy) +@@ -61,7 +68,10 @@ class TimedDownloadSATask(TimedTaskBase): + sa_name_list = TimedDownloadSATask.get_incremental_sa_name_list( + download_record) + +- TimedDownloadSATask.download_security_advisory(sa_name_list) ++ jobs = [gevent.spawn(TimedDownloadSATask.download_security_advisory, sa_name) ++ for sa_name in sa_name_list] ++ gevent.joinall(jobs) ++ + TimedDownloadSATask.save_security_advisory_to_database(proxy) + + proxy.save_advisory_download_record( +@@ -132,35 +142,30 @@ class TimedDownloadSATask(TimedTaskBase): + return "" + + @staticmethod +- def download_security_advisory(sa_name_list: list): ++ def download_security_advisory(sa_name: str): + """ + Get each url from the list, download and save it locally, and save it to the database if the download fails + + Args: +- sa_name_list: Advisory url list, each element is the url of the security announcement ++ sa_name: sa`s name, e.g. cvrf-openEuler-SA-2021-1022.xml + """ +- if os.path.exists(ADVISORY_SAVED_PATH): +- shutil.rmtree(ADVISORY_SAVED_PATH) +- os.makedirs(ADVISORY_SAVED_PATH) +- +- for sa_name in sa_name_list: +- advisory_year, advisory_serial_number = re.findall("\d+", sa_name) +- sa_url = f"{TimedDownloadSATask.cvrf_url}/{advisory_year}/{sa_name}" +- try: +- response = TimedDownloadSATask.get_response(sa_url) +- if response: +- with open(os.path.join(ADVISORY_SAVED_PATH, sa_name), "wb")as w: +- w.write(response.content) +- else: +- LOGGER.error(f"Download failed request timeout: {sa_name}") +- TimedDownloadSATask.save_sa_record.append({"advisory_year": advisory_year, +- "advisory_serial_number": advisory_serial_number, +- "download_status": False}) +- except retrying.RetryError: +- LOGGER.error(f"Download failed max retries: {sa_name}") ++ advisory_year, advisory_serial_number = re.findall("\d+", sa_name) ++ sa_url = f"{TimedDownloadSATask.cvrf_url}/{advisory_year}/{sa_name}" ++ try: ++ response = TimedDownloadSATask.get_response(sa_url) ++ if response: ++ with open(os.path.join(ADVISORY_SAVED_PATH, sa_name), "wb")as w: ++ w.write(response.content) ++ else: ++ LOGGER.error(f"Download failed request timeout: {sa_name}") + TimedDownloadSATask.save_sa_record.append({"advisory_year": advisory_year, + "advisory_serial_number": advisory_serial_number, + "download_status": False}) ++ except retrying.RetryError: ++ LOGGER.error(f"Download failed max retries: {sa_name}") ++ TimedDownloadSATask.save_sa_record.append({"advisory_year": advisory_year, ++ "advisory_serial_number": advisory_serial_number, ++ "download_status": False}) + + @staticmethod + def get_advisory_url_list() -> list: +diff --git a/setup.py b/setup.py +index 891c23e..30ab44d 100644 +--- a/setup.py ++++ b/setup.py +@@ -23,6 +23,7 @@ REQUIRES = [ + 'PyYAML', + 'retrying', + 'lxml' ++ 'gevent' + ] + + setup( +-- +Gitee + diff --git a/aops-apollo.spec b/aops-apollo.spec index 4c065878f6dc91d85258e9351a3a2af5c437312e..3d05c67df314ff93fe106766a9a1e5ea2c051320 100644 --- a/aops-apollo.spec +++ b/aops-apollo.spec @@ -1,10 +1,12 @@ Name: aops-apollo Version: v1.2.0 -Release: 1 +Release: 2 Summary: Cve management service, monitor machine vulnerabilities and provide fix functions. License: MulanPSL2 URL: https://gitee.com/openeuler/%{name} Source0: %{name}-%{version}.tar.gz +Patch0001: 0001-fix-args-not-effective-bug.patch +Patch0002: 0002-download-SA-collaborative-process.patch BuildRequires: python3-setuptools Requires: aops-vulcanus >= v1.2.0 @@ -26,7 +28,7 @@ Requires: python3-hawkey python3-dnf syscare dnf hotpatch plugin, it's about hotpatch query and fix %prep -%autosetup -n %{name}-%{version} +%autosetup -n %{name}-%{version} -p1 # build for aops-apollo @@ -52,6 +54,10 @@ cp -r hotpatch %{buildroot}/%{python3_sitelib}/dnf-plugins/ %{python3_sitelib}/dnf-plugins/* %changelog +* Thu Apr 27 2023 ptyang<1475324955@qq.com> - v1.2.0-2 +- fix args not effective bug +- download SA using a collaborative process + * Mon Apr 17 2023 gongzhengtang - v1.2.0-1 - add updated security advisory at regular time - add execute the CVE scan command at regular time