From d13de77217f04c39c5b79e5f1ad0b54cbffb5409 Mon Sep 17 00:00:00 2001 From: dongjie110 <17621827400@163.com> Date: Tue, 13 Sep 2022 11:13:35 +0800 Subject: [PATCH 01/10] add compare with release_management --- script/tools/check_rpms_complete.py | 483 +++++++++++++++++++++++ script/tools/daily_build_check.py | 115 +++++- script/tools/standard.yaml | 570 ++++++++++++++++++---------- 3 files changed, 962 insertions(+), 206 deletions(-) create mode 100644 script/tools/check_rpms_complete.py diff --git a/script/tools/check_rpms_complete.py b/script/tools/check_rpms_complete.py new file mode 100644 index 0000000..d8668ea --- /dev/null +++ b/script/tools/check_rpms_complete.py @@ -0,0 +1,483 @@ +#!/bin/env python3 +# -*- encoding=utf8 -*- +#****************************************************************************** +# Copyright (c) Huawei Technologging.es Co., Ltd. 2020-2020. All rights reserved. +# licensed under the Mulan PSL v2. +# You can use this software according to the terms and conditions of the Mulan PSL v2. +# You may obtain a copy of Mulan PSL v2 at: +# http://license.coscl.org.cn/MulanPSL2 +# THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR +# PURPOSE. +# See the Mulan PSL v2 for more details. +# Author: dongjie +# Create: 2022-03-28 +# ****************************************************************************** +import os +import sys +import re +import yaml +import xlwt +import shutil +import requests +import argparse +import subprocess +import logging +from bs4 import BeautifulSoup + +LOG_FORMAT = "%(asctime)s - %(filename)s[line:%(lineno)d] - %(levelname)s: %(message)s" +DATE_FORMAT = "%Y-%m-%d %H:%M:%S" +logging.basicConfig(level=logging.DEBUG, format=LOG_FORMAT, datefmt=DATE_FORMAT) + +class CheckRpmsComplete(object): + """ + The entrance check for rpms complete + """ + + def __init__(self, **kwargs): + """ + kawrgs: dict,init dict by 'a': 'A' style + rooturl: the daily build page url + main_branch:choose which branch you need to check + """ + self.kwargs = kwargs + self.rooturl = self.kwargs['daily_build_url'] + self.main_branch = self.kwargs['main_branch'] + self.dirflag = self.kwargs['dir_flag'] + self.datebranch = self.kwargs['date_branch'] + self.uploadflag = self.kwargs['uploadflag'] + self.obs = self.kwargs['obs'] + self.offical = self.kwargs['offical'] + self.release_management_path = self.kwargs['release_path'] + + def html_downloader(self, url): + """ + download url html content + """ + user_agent = 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/98.0.4758.102 Safari/537.36' + headers = {'User-Agent': user_agent} + r = requests.get(url, headers=headers) + if r.status_code == 200: + r.encoding = 'utf-8' + logging.info("this html page download success！{}".format(url)) + return r.text + else: + logging.error("this html page download failed！{}".format(url)) + return '' + + def rpms_parser(self,html_content): + """ + parse html content and get rpm name + """ + rpms_list = [] + soup = BeautifulSoup(html_content, "html.parser") + tr_content = soup.find_all('tr') + for line in tr_content: + td_content = line.find_all('td') + if td_content: + dir_url = td_content[0].find('a', href=True) + size = td_content[1].text + if dir_url.get('title',''): + if dir_url['title'] != '../' and self.check_is_rpm(dir_url['title']): + rpms_list.append(dir_url['title']) + return rpms_list + + def check_is_rpm(self, filename): + """ + file type filter + """ + if filename.endswith(".rpm"): + return True + + def obs_rpms_parser(self,html_content): + rpms_list = [] + soup = BeautifulSoup(html_content, "html.parser") + for k in soup.find_all('a',href=True): + if k.get('href',''): + if '.rpm' in k['href']: + rpms_list.append(k['href']) + return rpms_list + + def get_everything_rpms(self): + trans_url = self.main_branch.replace('-',':/') + x86_64_dir = "{}/{}/standard_x86_64/src/".format(self.obs,trans_url) + x86_64_content = self.html_downloader(x86_64_dir) + x86_64_rpms = self.obs_rpms_parser(x86_64_content) + aarch64_dir = "{}/{}/standard_aarch64/src/".format(self.obs,trans_url) + aarch64_content = self.html_downloader(aarch64_dir) + aarch64_rpms = self.obs_rpms_parser(aarch64_content) + everything_data = { + "x86-64":x86_64_rpms, + "aarch64":aarch64_rpms + } + return everything_data + + + def get_rpms(self,epol_dir,source_dir,parser_type='general'): + ''' + get rpms name from epoldir or source_dir + ''' + repo_data = {} + epol_content = self.html_downloader(epol_dir) + source_content = self.html_downloader(source_dir) + if parser_type == 'general': + epol_rpms = self.rpms_parser(epol_content) + source_rpms = self.rpms_parser(source_content) + else: + epol_rpms = self.obs_rpms_parser(epol_content) + source_rpms = self.obs_rpms_parser(source_content) + repo_data = { + 'epol_rpms': epol_rpms, + 'source_rpms': source_rpms + } + return repo_data + + def get_rpms_from_yaml(self, branch): + """ + branch: branch name + return: branch pkgs dict + """ + all_branch_pkgs = [] + # branch = self.main_branch + repo_data = {'epol_rpms':[], 'source_rpms':[]} + if os.path.exists(os.path.join(self.release_management_path, branch)): + standard_dirs = os.listdir(os.path.join(self.release_management_path, branch)) + for standard_dir in standard_dirs: + file_path = os.path.join(self.release_management_path, branch, standard_dir) + if not os.path.isdir(file_path) or standard_dir == 'delete': + standard_dirs.remove(standard_dir) + for c_dir in standard_dirs: + release_path = os.path.join(self.release_management_path, branch, c_dir, 'pckg-mgmt.yaml') + if os.path.exists(release_path): + with open(release_path, 'r', encoding='utf-8') as f: + result = yaml.load(f, Loader=yaml.FullLoader) + all_branch_pkgs.extend(result['packages']) + for pkg in all_branch_pkgs: + # project_pkgs.setdefault(pkg['obs_to'], []).append(pkg['name']) + if 'Epol' in pkg['obs_to']: + repo_data['epol_rpms'].append(pkg['name']) + else: + repo_data['source_rpms'].append(pkg['name']) + else: + logging.error("this branch {} not exist in repo release_management".format(branch)) + return repo_data + + def compare_rpms(self,repo_data,daily_repo_data,obs_repo_data): + ''' + get epol and source rpms data and start compare + :param repo_data:dict of offical repo src repo rpms include epol + :param daily_repo_data:dict of daily build repo src repo rpms include epol + :param obs_repo_data:dict of obs repo src repo rpms include epol + ''' + offical_epol_rpms = repo_data.get("epol_rpms",[]) + daily_epol_rpms = daily_repo_data.get("epol_rpms",[]) + obs_epol_rpms = obs_repo_data.get("epol_rpms",[]) + offical_src_rpms = repo_data.get("source_rpms",[]) + daily_src_rpms = daily_repo_data.get("source_rpms",[]) + obs_src_rpms = obs_repo_data.get("source_rpms",[]) + epol_result = self.compare_detail_rpms(daily_epol_rpms,offical_epol_rpms,obs_epol_rpms) + self.write_file(epol_result) + source_result = self.compare_detail_rpms(daily_src_rpms,offical_src_rpms,obs_src_rpms,compare_type='source') + self.write_file(source_result,compare_type='source') + + + def compare_detail_rpms(self,daily,offical,obs,compare_type='epol'): + ''' + src rpms detail compare + :param daily:List of daily build repo rpms + :param offical:List of offical repo rpms + :param obs:List of obs repo rpms + :return:dict of compare result + ''' + datas = {} + if daily and offical: + # daily rpms compare with offical epol rpms + in_daily_offical = list(set(daily).difference(set(offical))) + in_offical_daily = list(set(offical).difference(set(daily))) + logging.info("***************************************SIDEA:daily_build_repo,SIDEB:offical_repo*********************************") + in_daily_offical,in_offical_daily = self.rpm_version_compare(in_daily_offical,in_offical_daily) + logging.info("***************************************SIDEA:daily_build_repo,SIDEB:offical_repo*********************************") + logging.info("{} rpms compare below rpms in daily repo not in offical repo：{}".format(compare_type,in_daily_offical)) + logging.info("{} rpms compare below rpms in offical repo not in daily repo：{}".format(compare_type,in_offical_daily)) + daily_offical = {"rpmname":[self.datebranch,"offical repo"]} + for rpm in in_daily_offical: + daily_offical[rpm] = [1,0] + for rpm in in_offical_daily: + daily_offical[rpm] = [0,1] + datas["daily_offical"] = daily_offical + else: + if not daily and not offical: + daily_offical = {"error info":["{}get repo rpms failed".format(self.datebranch),"offical repo get rpms failed"]} + elif not daily: + daily_offical = {"error info":["{}get repo rpms failed".format(self.datebranch),"offical repo"]} + else: + daily_offical = {"error info":[self.datebranch,"offical repo get rpms failed"]} + datas["daily_offical"] = daily_offical + if daily and obs: + # daily rpms compare with obs epol rpms + in_daily_obs = list(set(daily).difference(set(obs))) + in_obs_daily = list(set(obs).difference(set(daily))) + logging.info("***************************************SIDEA:daily_build_repo,SIDEB:obs_repo*********************************") + in_daily_obs,in_obs_daily = self.rpm_version_compare(in_daily_obs,in_obs_daily) + logging.info("***************************************SIDEA:daily_build_repo,SIDEB:obs_repo*********************************") + logging.info("{} rpms compare below rpms in daily repo not in obs repo：{}".format(compare_type,in_daily_obs)) + logging.info("{} rpms compare below rpms in obs repo not in daily repo：{}".format(compare_type,in_obs_daily)) + daily_obs = {"rpmname":[self.datebranch,"obs repo"]} + for rpm in in_daily_obs: + daily_obs[rpm] = [1,0] + for rpm in in_obs_daily: + daily_obs[rpm] = [0,1] + datas["daily_obs"] = daily_obs + else: + if not daily and not obs: + daily_obs = {"error info":["{}get repo rpms failed".format(self.datebranch),"obs repo get rpms failed"]} + elif not daily: + daily_obs = {"error info":["{}get repo rpms failed".format(self.datebranch),"obs repo"]} + else: + daily_obs = {"error info":[self.datebranch,"obs repo get rpms failed"]} + datas["daily_obs"] = daily_obs + if obs and offical: + # daily rpms compare with offical epol rpms + in_obs_offical = list(set(obs).difference(set(offical))) + in_offical_obs = list(set(offical).difference(set(obs))) + logging.info("***************************************SIDEA:obs_repo,SIDEB:offical_repo*********************************") + in_obs_offical,in_offical_obs = self.rpm_version_compare(in_obs_offical,in_offical_obs) + logging.info("***************************************SIDEA:obs_repo,SIDEB:offical_repo*********************************") + logging.info("{} rpms compare below rpms in obs repo not in offical repo：{}".format(compare_type,in_obs_offical)) + logging.info("{} rpms compare below rpms in offical repo not in obs repo：{}".format(compare_type,in_offical_obs)) + obs_offical = {"rpmname":["obs repo","offical repo"]} + for rpm in in_obs_offical: + obs_offical[rpm] = [1,0] + for rpm in in_offical_obs: + obs_offical[rpm] = [0,1] + datas["obs_offical"] = obs_offical + else: + if not obs and not offical: + obs_offical = {"error info":["obs get repo rpms failed","offical repo get rpms failed"]} + elif not obs: + obs_offical = {"error info":["obs get repo rpms failed","offical repo"]} + else: + obs_offical = {"error info":["obs repo","offical repo get rpms failed"]} + datas["obs_offical"] = obs_offical + return datas + + def compare_everything_rpms(self,data,source_data): + ''' + compare obs x86_64 and aarch64 with daily build source rpms + :param data:pre compare obs x86_64 and aarch64 rpms + :param source_data:daily build source rpms + ''' + datas = {} + if source_data.get('source_rpms',[]): + source_rpms = source_data['source_rpms'] + for key,value in data.items(): + if value: + in_value_source = list(set(value).difference(set(source_rpms))) + in_source_value = list(set(source_rpms).difference(set(value))) + in_value_source,in_source_value = self.rpm_version_compare(in_value_source,in_source_value) + value_source = {"rpmname":["obs-{}-src".format(key),"{}-source".format(self.datebranch)]} + for rpm in in_value_source: + value_source[rpm] = [1,0] + for rpm in in_source_value: + value_source[rpm] = [0,1] + now_arc = "{}-daily".format(key) + datas[now_arc] = value_source + return datas + + def compare_release_rpms(self, release_data, daily_data): + resultdatas = {} + release_pkgs = [] + version_daily_pkgs = [] + daily_pkgs = [] + for arch, datas in daily_data.items(): + release_datas = release_data.get(arch, []) + release_pkgs.extend(release_datas) + version_daily_pkgs.extend(datas) + for pkg in version_daily_pkgs: + real_pkg = self.rpm_name(pkg) + daily_pkgs.append(real_pkg) + in_daily_release = list(set(daily_pkgs).difference(set(release_pkgs))) + in_release_daily = list(set(release_pkgs).difference(set(daily_pkgs))) + value_source = {"rpmname":[self.datebranch, "release_management"]} + for rpm in in_daily_release: + value_source[rpm] = [1,0] + for rpm in in_release_daily: + value_source[rpm] = [0,1] + print (value_source) + resultdatas['relase_daily'] = value_source + return resultdatas + + + def write_file(self,data,compare_type='epol'): + ''' + out put compare result to excel + :param data:pre compare result data + ''' + if data: + book = xlwt.Workbook(encoding='utf-8') + for key,value in data.items(): + sheet = book.add_sheet(key,cell_overwrite_ok=True) + r = 0 + for i, j in value.items(): + le = len(j) + sheet.write(r, 0, i,) + for c in range(1, le + 1): + sheet.write(r, c, j[c - 1]) + r += 1 + book.save("./{}-{}-src-rpm-compare-result.xls".format(self.main_branch,compare_type)) + + def _add_to_obs(self): + """ + add result file to obs project + """ + filenames = [] + path=os.getcwd() + f_list = os.listdir(path) + for xlsfile in f_list: + if os.path.splitext(xlsfile)[1] == '.xls': + filenames.append(xlsfile) + cmd = "cd %s && osc co home:Admin:ISO/%s" % (path,self.main_branch) + ret = os.popen(cmd).read() + targer_dir = os.path.join(path, "home:Admin:ISO/{}".format(self.main_branch)) + if os.path.exists(targer_dir): + for file in filenames: + src_file = os.path.join(path,file) + dist_file = os.path.join(path,targer_dir,file) + shutil.copyfile(src_file, dist_file) + cmd = "cd %s && osc add %s && osc ci -m1" % (targer_dir,file) + ret = os.popen(cmd).read() + logging.info("compare result already upload to OBS project:{}".format(self.main_branch)) + else: + logging.warning("this {} not in obs project".format(self.main_branch)) + + + def rpm_version_compare(self,paira,pairb): + ''' + check rpm version and compare + :param paira:List of sidea rpms that needs to be compared + :param pairb:List of sideb rpms that needs to be compared + ''' + for rpma in paira[:]: + rpm_namea = self.rpm_name(rpma) + for rpmb in pairb[:]: + rpm_nameb = self.rpm_name(rpmb) + if rpm_nameb == rpm_namea: + logging.info("SIDEA rpm version:{} SIDEB rpm version：{}".format(rpma,rpmb)) + pairb.remove(rpmb) + paira.remove(rpma) + break + return paira,pairb + + def rpm_n_v_r_d_a(self,rpm): + """ + parse rpm package name,version,release,publisher + :param rpm:complete rpm name + :return:split rpm + """ + # eg: grpc-1.31.0-6.oe1.x86_64.rpm + name = self.rpm_name(rpm) + + rpm_split = re.match(r"(.+)-(.+)\.(.+)?\.(.+)\.rpm", rpm.replace(name, "", 1)) + if rpm_split: + return name, rpm_split.group(1), rpm_split.group(2), rpm_split.group(3), rpm_split.group(4) + return name, rpm_split.group(1), rpm_split.group(2), None, rpm_split.group(3) + + def rpm_name(self,rpm): + """ + :param rpm:complete rpm name + :return:only rpm name + """ + m = re.match(r"^(.+)-.+-.+", rpm) + + if m: + return m.group(1) + else: + return rpm + + def shell_cmd(self,cmd_list): + """ + :param cmd_list: + :return: + """ + p = subprocess.Popen(cmd_list, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE) + out, err = p.communicate() + return p.returncode, out.decode("utf-8", errors="ignore"), err.decode("utf-8", errors="ignore") + + + def _check_src_rpm(self): + """ + Check the src rpm entry function + """ + # repo_data = self.get_repo_rpms() + # daily_repo_data = self.get_dailybuild_repo_rpms() + # obs_repo_data = self.get_obs_repo_rpms() + offical_epol_dir = "{}/{}/EPOL/main/source/Packages/".format(self.offical,self.main_branch) + offical_source_dir = "{}/{}/source/Packages/".format(self.offical,self.main_branch) + repo_data = self.get_rpms(offical_epol_dir,offical_source_dir) + daily_epol_dir = "{}/{}/{}/EPOL/main/source/Packages/".format(self.rooturl,self.main_branch,self.datebranch) + daily_source_dir = "{}/{}/{}/source/Packages/".format(self.rooturl,self.main_branch,self.datebranch) + daily_repo_data = self.get_rpms(daily_epol_dir,daily_source_dir) + trans_url = self.main_branch.replace('-',':/') + obs_epol_dir = "{}/{}:/Epol/standard_x86_64/src/".format(self.obs,trans_url) + obs_source_dir = "{}/{}/standard_x86_64/src/".format(self.obs,trans_url) + obs_repo_data = self.get_rpms(obs_epol_dir,obs_source_dir,parser_type='obs') + self.compare_rpms(repo_data,daily_repo_data,obs_repo_data) + + def _check_everything_rpm(self): + """ + Check the everything x86-64 and aarch64 rpms entry function + """ + everything_data = self.get_everything_rpms() + release_data = self.get_rpms_from_yaml(self.main_branch) + daily_epol_dir = "{}/{}/{}/EPOL/main/source/Packages/".format(self.rooturl,self.main_branch,self.datebranch) + daily_source_dir = "{}/{}/{}/source/Packages/".format(self.rooturl,self.main_branch,self.datebranch) + source_data = self.get_rpms(daily_epol_dir,daily_source_dir) + release_result = self.compare_release_rpms(release_data, source_data) + self.write_file(release_result,compare_type='release') + result = self.compare_everything_rpms(everything_data, source_data) + self.write_file(result,compare_type='everything') + + def run(self): + self._check_src_rpm() + self._check_everything_rpm() + if self.uploadflag == '1': + self._add_to_obs() + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument( + "--dirflag", + default="openeuler-2022-", + help="which date branch you want to check,eg:openeuler-2022-03-04-09-22-07") + parser.add_argument( + "--rooturl", + help="daily build page root url,eg: http://ip/dailybuild") + parser.add_argument( + "--branch", + help="which branch you want to check,eg:openEuler-22.03-LTS") + parser.add_argument( + "--datebranch", + help="which date branch you want to check,eg:openeuler-2022-03-04-09-22-07") + parser.add_argument( + "--obs", + help="obs repo ip") + parser.add_argument( + "--offical", + help="offical repo ip") + parser.add_argument( + "--release_path", + help="repo release_management path") + parser.add_argument("--uploadflag",default="1") + args = parser.parse_args() + kw = { + "daily_build_url": args.rooturl, + "main_branch": args.branch, + "date_branch": args.datebranch, + "dir_flag": args.dirflag, + "uploadflag": args.uploadflag, + "obs":args.obs, + "offical":args.offical, + "release_path":args.release_path + } + check = CheckRpmsComplete(**kw) + check.run() \ No newline at end of file diff --git a/script/tools/daily_build_check.py b/script/tools/daily_build_check.py index a5557b0..ebd0a87 100644 --- a/script/tools/daily_build_check.py +++ b/script/tools/daily_build_check.py @@ -15,7 +15,9 @@ # ****************************************************************************** import os import sys +import re import yaml +import shutil import requests import argparse import logging @@ -24,6 +26,7 @@ from bs4 import BeautifulSoup LOG_FORMAT = "%(asctime)s - %(filename)s[line:%(lineno)d] - %(levelname)s: %(message)s" DATE_FORMAT = "%Y-%m-%d %H:%M:%S" logging.basicConfig(level=logging.DEBUG, format=LOG_FORMAT, datefmt=DATE_FORMAT) +UNMOUNT_DIRS = ['debuginfo','OS','source'] class CheckDailyBuild(object): @@ -42,6 +45,7 @@ class CheckDailyBuild(object): self.main_branch = self.kwargs['main_branch'] self.dirflag = self.kwargs['dir_flag'] self.datebranch = self.kwargs['date_branch'] + self.mountflag = self.kwargs['mountflag'] self.standard_dir = self.load_standard() def html_downloader(self, url): @@ -92,8 +96,8 @@ class CheckDailyBuild(object): """ dir_map = {} html_content = self.html_downloader(main_branch_url) - if html_content: - current_dir = self.html_parser(main_branch_url, html_content) + current_dir = self.html_parser(main_branch_url, html_content) + if current_dir: dir_map[main_branch_url] = [] for first_dir in current_dir: dir_split = first_dir.split('/')[5] @@ -129,8 +133,8 @@ class CheckDailyBuild(object): if item.split('|')[-1] != 'ISFILE/': self.check_current_dir(item, origin_dir, temp_list) except Exception as e: - logging.info("error url can not open:{}".format(c_dir)) - logging.info("error url can not open:{}".format(e)) + logging.warning("error url can not open:{}".format(c_dir)) + # logging.info("error url can not open:{}".format(e)) return temp_list def load_standard(self): @@ -139,24 +143,101 @@ class CheckDailyBuild(object): """ try: with open('./standard.yaml', 'r', encoding='utf-8') as f: - result = yaml.load(f, Loader=yaml.FullLoader) + yaml_result = yaml.load(f, Loader=yaml.FullLoader) + if yaml_result.get(self.main_branch,''): + result = yaml_result[self.main_branch] + else: + result = yaml_result['standard'] + if self.mountflag == '0': + for delitem in UNMOUNT_DIRS: + del result[delitem] return result except Exception as e: + logging.info(e) logging.info("error read standard.yaml,please check") + # def compare_standard(self, dir_list, current_branch, c_dir): + # """ + # branch dir compare with standard dir + # """ + # logging.info("********************************************CHECK RESULT*********************************************************") + # standard_dir = self.standard_dir + # error_flag = False + # for key, c_standard in standard_dir.items(): + # for current_dir in c_standard: + # if '*' in current_dir: + # current_dir = current_dir.replace('*', current_branch) + # if current_dir not in dir_list: + # error_flag = True + # logging.error( + # 'this dir or file not found,link url:{}{}'.format( + # c_dir, current_dir)) + # logging.info("********************************************CHECK RESULT*********************************************************") + # if error_flag: + # logging.error("some files or dirs not found in：{}".format(self.datebranch)) + # raise SystemExit("*****************************************PLEASE CHECK*****************************************************") + def compare_standard(self, dir_list, current_branch, c_dir): """ branch dir compare with standard dir """ + logging.info("********************************************CHECK RESULT*********************************************************") + print (dir_list) + compare_results = {"compare_results":[]} standard_dir = self.standard_dir + error_flag = False for key, c_standard in standard_dir.items(): for current_dir in c_standard: - if '*' in current_dir: - current_dir = current_dir.replace('*', current_branch) + # if '*' in current_dir: + current_dir = current_dir.replace('*', current_branch) + temp_dic = {} if current_dir not in dir_list: + error_flag = True + temp_dic[current_dir] = False logging.error( - 'this dir not found,link url:{}{}'.format( + 'this dir or file not found,link url:{}{}'.format( c_dir, current_dir)) + else: + temp_dic[current_dir] = True + compare_results["compare_results"].append(temp_dic) + # logging.info("********************************************CHECK RESULT*********************************************************") + # if error_flag: + # logging.error("some files or dirs not found in：{}".format(self.datebranch)) + # raise SystemExit("*****************************************PLEASE CHECK*****************************************************") + self.write_yaml(compare_results) + logging.info("********************************************CHECK RESULT*********************************************************") + self._add_to_obs() + + + def check_input_args(self, complete_url): + html_content = self.html_downloader(complete_url) + current_dir = self.html_parser(complete_url, html_content) + if not current_dir: + logging.error("error url can not open,please check your input：{}".format(complete_url)) + raise SystemExit("*******PLEASE CHECK YOUR INPUT ARGS*******") + else: + return True + + def _add_to_obs(self): + """ + add result file to obs project + """ + # src_file = os.path.join(os.getcwd(),"{}.yaml".format(self.datebranch)) + file = "{}.yaml".format(self.datebranch) + path=os.getcwd() + cmd = "cd %s && osc co home:Admin:ISO/%s" % (path,self.main_branch) + ret = os.popen(cmd).read() + targer_dir = os.path.join(path, "home:Admin:ISO/{}".format(self.main_branch)) + if os.path.exists(targer_dir): + src_file = os.path.join(path, file) + dist_file = os.path.join(path, targer_dir, file) + shutil.copyfile(src_file, dist_file) + cmd = "cd %s && osc add %s && osc ci -m1" % (targer_dir, file) + ret = os.popen(cmd).read() + logging.info("compare result already upload to OBS project:{}".format(self.main_branch)) + else: + logging.warning("this {} not in obs project".format(self.main_branch)) + def _get_main_branch(self): if self.datebranch: @@ -164,11 +245,20 @@ class CheckDailyBuild(object): complete_key = "{}/{}/".format(self.rooturl, self.main_branch) complete_value = "{}/{}/{}/".format( self.rooturl, self.main_branch, self.datebranch) - dir_map[complete_key] = [complete_value] + check_args_result = self.check_input_args(complete_value) + if check_args_result: + dir_map[complete_key] = [complete_value] + self.check_every_dir(dir_map) else: main_branch_url = "{}/{}/".format(self.rooturl, self.main_branch) dir_map = self.start_check(main_branch_url) - self.check_every_dir(dir_map) + self.check_every_dir(dir_map) + + def write_yaml(self, dict_msg): + file_path = os.path.join(os.getcwd(),"{}.yaml".format(self.datebranch)) + with open(file_path, "w", encoding='utf-8') as f: + yaml.dump(dict_msg, f, default_flow_style=False, sort_keys=False) + def run(self): self._get_main_branch() @@ -189,11 +279,14 @@ if __name__ == "__main__": parser.add_argument( "--datebranch", help="which date branch you want to check,eg:openeuler-2022-03-04-09-22-07") + parser.add_argument("--mountflag",default="1") + args = parser.parse_args() kw = { "daily_build_url": args.rooturl, "main_branch": args.branch, "date_branch": args.datebranch, - "dir_flag": args.dirflag + "dir_flag": args.dirflag, + "mountflag": args.mountflag } check = CheckDailyBuild(**kw) check.run() diff --git a/script/tools/standard.yaml b/script/tools/standard.yaml index 6c5fdbc..50ad513 100644 --- a/script/tools/standard.yaml +++ b/script/tools/standard.yaml @@ -1,196 +1,376 @@ -MAIN_DIR: - - EPOL/ - - ISO/ - - OS/ - - debuginfo/ - - docker_img/ - - everything/ - - raspi_img/ - - source/ - - stratovirt_img/ - - update/ - - virtual_machine_img/ -EPOL: - - EPOL/main/ - - EPOL/update/ - - EPOL/multi_version/ - - EPOL/multi_version/OpenStack/ - - EPOL/main/source/ - - EPOL/main/aarch64/ - - EPOL/main/x86_64/ - - EPOL/update/main/ - - EPOL/update/multi_version/ - - EPOL/update/main/aarch64/ - - EPOL/main/source/Packages/ - - EPOL/main/source/repodata/ - - EPOL/main/aarch64/Packages/ - - EPOL/main/aarch64/repodata/ - - EPOL/main/x86_64/repodata/ - - EPOL/update/multi_version/source/ - - EPOL/update/main/aarch64/Packages/ - - EPOL/update/main/aarch64/repodata/ - - EPOL/update/main/source/Packages/ - - EPOL/update/main/source/repodata/ - - EPOL/update/main/x86_64/Packages/ - - EPOL/update/main/x86_64/repodata/ - - EPOL/multi_version/OpenStack/Train/ - - EPOL/multi_version/OpenStack/Wallaby/ - - EPOL/multi_version/OpenStack/Train/aarch64/ - - EPOL/multi_version/OpenStack/Train/source/ - - EPOL/multi_version/OpenStack/Train/x86_64/ - - EPOL/multi_version/OpenStack/Train/aarch64/Packages/ - - EPOL/multi_version/OpenStack/Train/aarch64/repodata/ - - EPOL/multi_version/OpenStack/Train/source/Packages/ - - EPOL/multi_version/OpenStack/Train/source/repodata/ - - EPOL/multi_version/OpenStack/Train/x86_64/Packages/ - - EPOL/multi_version/OpenStack/Train/x86_64/repodata/ - - EPOL/multi_version/OpenStack/Wallaby/aarch64/ - - EPOL/multi_version/OpenStack/Wallaby/source/ - - EPOL/multi_version/OpenStack/Wallaby/x86_64/ - - EPOL/multi_version/OpenStack/Wallaby/aarch64/Packages/ - - EPOL/multi_version/OpenStack/Wallaby/aarch64/repodata/ - - EPOL/multi_version/OpenStack/Wallaby/source/Packages/ - - EPOL/multi_version/OpenStack/Wallaby/source/repodata/ - - EPOL/multi_version/OpenStack/Wallaby/x86_64/Packages/ - - EPOL/multi_version/OpenStack/Wallaby/x86_64/repodata/ - - EPOL/update/multi_version/source/Packages/ - - EPOL/update/multi_version/source/repodata/ - - EPOL/update/multi_version/x86_64/Packages/ - - EPOL/update/multi_version/x86_64/repodata/ - - EPOL/update/multi_version/aarch64/Packages/ - - EPOL/update/multi_version/aarch64/repodata/ -ISO: - - ISO/aarch64/ - - ISO/x86_64/ - - ISO/source/ - - ISO/aarch64/*-aarch64-dvd.iso - - ISO/aarch64/*-aarch64-dvd.iso.sha256sum - - ISO/aarch64/*-aarch64.rpmlist - - ISO/aarch64/*-everything-aarch64-dvd.iso - - ISO/aarch64/*-everything-aarch64-dvd.iso.sha256sum - - ISO/aarch64/*-everything-debug-aarch64-dvd.iso - - ISO/aarch64/*-everything-debug-aarch64-dvd.iso.sha256sum - - ISO/aarch64/*-netinst-aarch64-dvd.iso - - ISO/aarch64/*-netinst-aarch64-dvd.iso.sha256sum - - ISO/source/*-source-dvd.iso - - ISO/source/*-source-dvd.iso.sha256sum - - ISO/x86_64/*-everything-debug-x86_64-dvd.iso - - ISO/x86_64/*-everything-debug-x86_64-dvd.iso.sha256sum - - ISO/x86_64/*-netinst-x86_64-dvd.iso - - ISO/x86_64/*-netinst-x86_64-dvd.iso.sha256sum - - ISO/x86_64/*-x86_64-dvd.iso - - ISO/x86_64/*-x86_64-dvd.iso.sha256sum - - ISO/x86_64/*-x86_64.rpmlist -OS: - - OS/aarch64/ - - OS/x86_64/ - - OS/aarch64/EFI/ - - OS/aarch64/Packages/ - - OS/aarch64/docs/ - - OS/aarch64/images/ - - OS/aarch64/repodata/ - - OS/aarch64/RPM-GPG-KEY-openEuler/ - - OS/aarch64/TRANS.TBL - - OS/aarch64/boot.catalog - - OS/x86_64/EFI/ - - OS/x86_64/Packages/ - - OS/x86_64/docs/ - - OS/x86_64/images/ - - OS/x86_64/repodata/ - - OS/x86_64/RPM-GPG-KEY-openEuler/ - - OS/x86_64/TRANS.TBL - - OS/x86_64/boot.catalog -debuginfo: - - debuginfo/aarch64/ - - debuginfo/x86_64/ - - debuginfo/aarch64/EFI/ - - debuginfo/aarch64/Packages/ - - debuginfo/aarch64/docs/ - - debuginfo/aarch64/images/ - - debuginfo/aarch64/repodata/ - - debuginfo/aarch64/RPM-GPG-KEY-openEuler/ - - debuginfo/aarch64/TRANS.TBL - - debuginfo/aarch64/boot.catalog - - debuginfo/x86_64/EFI/ - - debuginfo/x86_64/Packages/ - - debuginfo/x86_64/docs/ - - debuginfo/x86_64/images/ - - debuginfo/x86_64/repodata/ - - debuginfo/x86_64/RPM-GPG-KEY-openEuler/ - - debuginfo/x86_64/TRANS.TBL - - debuginfo/x86_64/boot.catalog -docker_img: - - docker_img/aarch64/ - - docker_img/x86_64/ - - docker_img/aarch64/openEuler-docker.aarch64.tar.xz - - docker_img/aarch64/openEuler-docker.aarch64.tar.xz.sha256sum - - docker_img/x86_64/openEuler-docker.x86_64.tar.xz - - docker_img/x86_64/openEuler-docker.x86_64.tar.xz.sha256sum -everything: - - everything/aarch64/ - - everything/x86_64/ - - everything/aarch64/EFI/ - - everything/aarch64/Packages/ - - everything/aarch64/docs/ - - everything/aarch64/images/ - - everything/aarch64/repodata/ - - everything/aarch64/RPM-GPG-KEY-openEuler/ - - everything/aarch64/TRANS.TBL - - everything/aarch64/boot.catalog - - everything/x86_64/EFI/ - - everything/x86_64/Packages/ - - everything/x86_64/docs/ - - everything/x86_64/images/ - - everything/x86_64/repodata/ - - everything/x86_64/RPM-GPG-KEY-openEuler/ - - everything/x86_64/TRANS.TBL - - everything/x86_64/boot.catalog -raspi_img: - - raspi_img/*-raspi-aarch64.img - - raspi_img/*-raspi-aarch64.img.sha256sum - - raspi_img/*-raspi-aarch64.img.xz - - raspi_img/*-raspi-aarch64.img.xz.sha256sum -source: - - source/aarch64/EFI/ - - source/aarch64/Packages/ - - source/aarch64/docs/ - - source/aarch64/images/ - - source/aarch64/repodata/ - - source/aarch64/RPM-GPG-KEY-openEuler/ - - source/aarch64/TRANS.TBL - - source/aarch64/boot.catalog -stratovirt_img: - - stratovirt_img/aarch64/ - - stratovirt_img/x86_64/ - - stratovirt_img/aarch64/*-stratovirt-aarch64.img.xz - - stratovirt_img/aarch64/*-stratovirt-aarch64.img.xz.sha256sum - - stratovirt_img/aarch64/EFI/ - - stratovirt_img/aarch64/std-vmlinux.bin - - stratovirt_img/aarch64/std-vmlinux.bin.sha256sum - - stratovirt_img/aarch64/vmlinux.bin - - stratovirt_img/aarch64/vmlinux.bin.sha256sum - - stratovirt_img/x86_64/*-stratovirt-x86_64.img.xz - - stratovirt_img/x86_64/EFI/ - - stratovirt_img/x86_64/*-stratovirt-x86_64.img.xz.sha256sum - - stratovirt_img/x86_64/std-vmlinux.bin - - stratovirt_img/x86_64/std-vmlinux.bin.sha256sum - - stratovirt_img/x86_64/vmlinux.bin - - stratovirt_img/x86_64/vmlinux.bin.sha256sum -update: - - update/aarch64/ - - update/source/ - - update/x86_64/ - - update/aarch64/Packages/ - - update/aarch64/repodata/ - - update/source/Packages/ - - update/source/repodata/ - - update/x86_64/Packages/ - - update/x86_64/repodata/ +standard: + MAIN_DIR: + - EPOL/ + - ISO/ + - OS/ + - debuginfo/ + - docker_img/ + - everything/ + - raspi_img/ + - source/ + - stratovirt_img/ + - update/ + - virtual_machine_img/ + EPOL: + - EPOL/main/ + - EPOL/update/ + - EPOL/multi_version/ + - EPOL/multi_version/OpenStack/ + - EPOL/main/source/ + - EPOL/main/aarch64/ + - EPOL/main/x86_64/ + - EPOL/update/main/ + - EPOL/update/multi_version/ + - EPOL/update/main/aarch64/ + - EPOL/update/main/source/ + - EPOL/update/main/x86_64/ + - EPOL/main/source/Packages/ + - EPOL/main/source/repodata/ + - EPOL/main/aarch64/Packages/ + - EPOL/main/aarch64/repodata/ + - EPOL/main/x86_64/repodata/ + - EPOL/update/multi_version/source/ + - EPOL/update/main/aarch64/Packages/ + - EPOL/update/main/aarch64/repodata/ + - EPOL/update/main/source/Packages/ + - EPOL/update/main/source/repodata/ + - EPOL/update/main/x86_64/Packages/ + - EPOL/update/main/x86_64/repodata/ + - EPOL/multi_version/OpenStack/Train/ + - EPOL/multi_version/OpenStack/Wallaby/ + - EPOL/multi_version/OpenStack/Train/aarch64/ + - EPOL/multi_version/OpenStack/Train/source/ + - EPOL/multi_version/OpenStack/Train/x86_64/ + - EPOL/multi_version/OpenStack/Train/aarch64/Packages/ + - EPOL/multi_version/OpenStack/Train/aarch64/repodata/ + - EPOL/multi_version/OpenStack/Train/source/Packages/ + - EPOL/multi_version/OpenStack/Train/source/repodata/ + - EPOL/multi_version/OpenStack/Train/x86_64/Packages/ + - EPOL/multi_version/OpenStack/Train/x86_64/repodata/ + - EPOL/multi_version/OpenStack/Wallaby/aarch64/ + - EPOL/multi_version/OpenStack/Wallaby/source/ + - EPOL/multi_version/OpenStack/Wallaby/x86_64/ + - EPOL/multi_version/OpenStack/Wallaby/aarch64/Packages/ + - EPOL/multi_version/OpenStack/Wallaby/aarch64/repodata/ + - EPOL/multi_version/OpenStack/Wallaby/source/Packages/ + - EPOL/multi_version/OpenStack/Wallaby/source/repodata/ + - EPOL/multi_version/OpenStack/Wallaby/x86_64/Packages/ + - EPOL/multi_version/OpenStack/Wallaby/x86_64/repodata/ + - EPOL/update/multi_version/source/Packages/ + - EPOL/update/multi_version/source/repodata/ + - EPOL/update/multi_version/x86_64/Packages/ + - EPOL/update/multi_version/x86_64/repodata/ + - EPOL/update/multi_version/aarch64/Packages/ + - EPOL/update/multi_version/aarch64/repodata/ + ISO: + - ISO/aarch64/ + - ISO/x86_64/ + - ISO/source/ + - ISO/aarch64/*-aarch64-dvd.iso + - ISO/aarch64/*-aarch64-dvd.iso.sha256sum + - ISO/aarch64/*-aarch64.rpmlist + - ISO/aarch64/*-everything-aarch64-dvd.iso + - ISO/aarch64/*-everything-aarch64-dvd.iso.sha256sum + - ISO/aarch64/*-everything-debug-aarch64-dvd.iso + - ISO/aarch64/*-everything-debug-aarch64-dvd.iso.sha256sum + - ISO/aarch64/*-netinst-aarch64-dvd.iso + - ISO/aarch64/*-netinst-aarch64-dvd.iso.sha256sum + - ISO/source/*-source-dvd.iso + - ISO/source/*-source-dvd.iso.sha256sum + - ISO/x86_64/*-everything-debug-x86_64-dvd.iso + - ISO/x86_64/*-everything-debug-x86_64-dvd.iso.sha256sum + - ISO/x86_64/*-netinst-x86_64-dvd.iso + - ISO/x86_64/*-netinst-x86_64-dvd.iso.sha256sum + - ISO/x86_64/*-x86_64-dvd.iso + - ISO/x86_64/*-x86_64-dvd.iso.sha256sum + - ISO/x86_64/*-x86_64.rpmlist + OS: + - OS/aarch64/ + - OS/x86_64/ + - OS/aarch64/EFI/ + - OS/aarch64/Packages/ + - OS/aarch64/docs/ + - OS/aarch64/images/ + - OS/aarch64/repodata/ + - OS/aarch64/RPM-GPG-KEY-openEuler/ + - OS/aarch64/TRANS.TBL + - OS/aarch64/boot.catalog + - OS/x86_64/EFI/ + - OS/x86_64/Packages/ + - OS/x86_64/docs/ + - OS/x86_64/images/ + - OS/x86_64/repodata/ + - OS/x86_64/RPM-GPG-KEY-openEuler/ + - OS/x86_64/TRANS.TBL + - OS/x86_64/boot.catalog + debuginfo: + - debuginfo/aarch64/ + - debuginfo/x86_64/ + - debuginfo/aarch64/EFI/ + - debuginfo/aarch64/Packages/ + - debuginfo/aarch64/docs/ + - debuginfo/aarch64/images/ + - debuginfo/aarch64/repodata/ + - debuginfo/aarch64/RPM-GPG-KEY-openEuler/ + - debuginfo/aarch64/TRANS.TBL + - debuginfo/aarch64/boot.catalog + - debuginfo/x86_64/EFI/ + - debuginfo/x86_64/Packages/ + - debuginfo/x86_64/docs/ + - debuginfo/x86_64/images/ + - debuginfo/x86_64/repodata/ + - debuginfo/x86_64/RPM-GPG-KEY-openEuler/ + - debuginfo/x86_64/TRANS.TBL + - debuginfo/x86_64/boot.catalog + docker_img: + - docker_img/aarch64/ + - docker_img/x86_64/ + - docker_img/aarch64/openEuler-docker.aarch64.tar.xz + - docker_img/aarch64/openEuler-docker.aarch64.tar.xz.sha256sum + - docker_img/x86_64/openEuler-docker.x86_64.tar.xz + - docker_img/x86_64/openEuler-docker.x86_64.tar.xz.sha256sum + everything: + - everything/aarch64/ + - everything/x86_64/ + - everything/aarch64/EFI/ + - everything/aarch64/Packages/ + - everything/aarch64/docs/ + - everything/aarch64/images/ + - everything/aarch64/repodata/ + - everything/aarch64/RPM-GPG-KEY-openEuler/ + - everything/aarch64/TRANS.TBL + - everything/aarch64/boot.catalog + - everything/x86_64/EFI/ + - everything/x86_64/Packages/ + - everything/x86_64/docs/ + - everything/x86_64/images/ + - everything/x86_64/repodata/ + - everything/x86_64/RPM-GPG-KEY-openEuler/ + - everything/x86_64/TRANS.TBL + - everything/x86_64/boot.catalog + raspi_img: + - raspi_img/*-raspi-aarch64.img + - raspi_img/*-raspi-aarch64.img.sha256sum + - raspi_img/*-raspi-aarch64.img.xz + - raspi_img/*-raspi-aarch64.img.xz.sha256sum + source: + - source/aarch64/EFI/ + - source/aarch64/Packages/ + - source/aarch64/docs/ + - source/aarch64/images/ + - source/aarch64/repodata/ + - source/aarch64/RPM-GPG-KEY-openEuler/ + - source/aarch64/TRANS.TBL + - source/aarch64/boot.catalog + stratovirt_img: + - stratovirt_img/aarch64/ + - stratovirt_img/x86_64/ + - stratovirt_img/aarch64/*-stratovirt-aarch64.img.xz + - stratovirt_img/aarch64/*-stratovirt-aarch64.img.xz.sha256sum + - stratovirt_img/aarch64/EFI/ + - stratovirt_img/aarch64/std-vmlinux.bin + - stratovirt_img/aarch64/std-vmlinux.bin.sha256sum + - stratovirt_img/aarch64/vmlinux.bin + - stratovirt_img/aarch64/vmlinux.bin.sha256sum + - stratovirt_img/x86_64/*-stratovirt-x86_64.img.xz + - stratovirt_img/x86_64/EFI/ + - stratovirt_img/x86_64/*-stratovirt-x86_64.img.xz.sha256sum + - stratovirt_img/x86_64/std-vmlinux.bin + - stratovirt_img/x86_64/std-vmlinux.bin.sha256sum + - stratovirt_img/x86_64/vmlinux.bin + - stratovirt_img/x86_64/vmlinux.bin.sha256sum + update: + - update/aarch64/ + - update/source/ + - update/x86_64/ + - update/aarch64/Packages/ + - update/aarch64/repodata/ + - update/source/Packages/ + - update/source/repodata/ + - update/x86_64/Packages/ + - update/x86_64/repodata/ -virtual_machine_img: - - virtual_machine_img/aarch64/ - - virtual_machine_img/x86_64/ - - virtual_machine_img/aarch64/*-aarch64.qcow2.xz - - virtual_machine_img/aarch64/*-aarch64.qcow2.xz.sha256sum - - virtual_machine_img/x86_64/*-x86_64.qcow2.xz - - virtual_machine_img/x86_64/*-x86_64.qcow2.xz.sha256sum \ No newline at end of file + virtual_machine_img: + - virtual_machine_img/aarch64/ + - virtual_machine_img/x86_64/ + - virtual_machine_img/aarch64/*-aarch64.qcow2.xz + - virtual_machine_img/aarch64/*-aarch64.qcow2.xz.sha256sum + - virtual_machine_img/x86_64/*-x86_64.qcow2.xz + - virtual_machine_img/x86_64/*-x86_64.qcow2.xz.sha256sum +openEuler-22.09-1: + MAIN_DIR: + - EPOL/ + - ISO/ + - OS/ + - debuginfo/ + - docker_img/ + - edge_img/ + - embedded_img/ + - everything/ + - raspi_img/ + - source/ + - stratovirt_img/ + - update/ + - virtual_machine_img/ + EPOL: + - EPOL/main/ + - EPOL/update/ + - EPOL/main/source/ + - EPOL/main/aarch64/ + - EPOL/main/x86_64/ + - EPOL/update/main/ + - EPOL/update/multi_version/ + - EPOL/update/main/aarch64/ + - EPOL/main/source/Packages/ + - EPOL/main/source/repodata/ + - EPOL/main/aarch64/Packages/ + - EPOL/main/aarch64/repodata/ + - EPOL/main/x86_64/repodata/ + - EPOL/update/multi_version/source/ + - EPOL/update/main/aarch64/Packages/ + - EPOL/update/main/aarch64/repodata/ + - EPOL/update/main/source/Packages/ + - EPOL/update/main/source/repodata/ + - EPOL/update/main/x86_64/Packages/ + - EPOL/update/main/x86_64/repodata/ + - EPOL/update/multi_version/source/Packages/ + - EPOL/update/multi_version/source/repodata/ + - EPOL/update/multi_version/x86_64/Packages/ + - EPOL/update/multi_version/x86_64/repodata/ + - EPOL/update/multi_version/aarch64/Packages/ + - EPOL/update/multi_version/aarch64/repodata/ + ISO: + - ISO/aarch64/ + - ISO/x86_64/ + - ISO/source/ + - ISO/aarch64/*-aarch64-dvd.iso + - ISO/aarch64/*-aarch64-dvd.iso.sha256sum + - ISO/aarch64/*-aarch64.rpmlist + - ISO/aarch64/*-everything-aarch64-dvd.iso + - ISO/aarch64/*-everything-aarch64-dvd.iso.sha256sum + - ISO/aarch64/*-everything-debug-aarch64-dvd.iso + - ISO/aarch64/*-everything-debug-aarch64-dvd.iso.sha256sum + - ISO/aarch64/*-netinst-aarch64-dvd.iso + - ISO/aarch64/*-netinst-aarch64-dvd.iso.sha256sum + - ISO/source/*-source-dvd.iso + - ISO/source/*-source-dvd.iso.sha256sum + - ISO/x86_64/*-everything-debug-x86_64-dvd.iso + - ISO/x86_64/*-everything-debug-x86_64-dvd.iso.sha256sum + - ISO/x86_64/*-netinst-x86_64-dvd.iso + - ISO/x86_64/*-netinst-x86_64-dvd.iso.sha256sum + - ISO/x86_64/*-x86_64-dvd.iso + - ISO/x86_64/*-x86_64-dvd.iso.sha256sum + - ISO/x86_64/*-x86_64.rpmlist + OS: + - OS/aarch64/ + - OS/x86_64/ + - OS/aarch64/EFI/ + - OS/aarch64/Packages/ + - OS/aarch64/docs/ + - OS/aarch64/images/ + - OS/aarch64/repodata/ + - OS/aarch64/RPM-GPG-KEY-openEuler/ + - OS/aarch64/TRANS.TBL + - OS/aarch64/boot.catalog + - OS/x86_64/EFI/ + - OS/x86_64/Packages/ + - OS/x86_64/docs/ + - OS/x86_64/images/ + - OS/x86_64/repodata/ + - OS/x86_64/RPM-GPG-KEY-openEuler/ + - OS/x86_64/TRANS.TBL + - OS/x86_64/boot.catalog + debuginfo: + - debuginfo/aarch64/ + - debuginfo/x86_64/ + - debuginfo/aarch64/EFI/ + - debuginfo/aarch64/Packages/ + - debuginfo/aarch64/docs/ + - debuginfo/aarch64/images/ + - debuginfo/aarch64/repodata/ + - debuginfo/aarch64/RPM-GPG-KEY-openEuler/ + - debuginfo/aarch64/TRANS.TBL + - debuginfo/aarch64/boot.catalog + - debuginfo/x86_64/EFI/ + - debuginfo/x86_64/Packages/ + - debuginfo/x86_64/docs/ + - debuginfo/x86_64/images/ + - debuginfo/x86_64/repodata/ + - debuginfo/x86_64/RPM-GPG-KEY-openEuler/ + - debuginfo/x86_64/TRANS.TBL + - debuginfo/x86_64/boot.catalog + docker_img: + - docker_img/aarch64/ + - docker_img/x86_64/ + - docker_img/aarch64/openEuler-docker.aarch64.tar.xz + - docker_img/aarch64/openEuler-docker.aarch64.tar.xz.sha256sum + - docker_img/x86_64/openEuler-docker.x86_64.tar.xz + - docker_img/x86_64/openEuler-docker.x86_64.tar.xz.sha256sum + everything: + - everything/aarch64/ + - everything/x86_64/ + - everything/aarch64/EFI/ + - everything/aarch64/Packages/ + - everything/aarch64/docs/ + - everything/aarch64/images/ + - everything/aarch64/repodata/ + - everything/aarch64/RPM-GPG-KEY-openEuler/ + - everything/aarch64/TRANS.TBL + - everything/aarch64/boot.catalog + - everything/x86_64/EFI/ + - everything/x86_64/Packages/ + - everything/x86_64/docs/ + - everything/x86_64/images/ + - everything/x86_64/repodata/ + - everything/x86_64/RPM-GPG-KEY-openEuler/ + - everything/x86_64/TRANS.TBL + - everything/x86_64/boot.catalog + raspi_img: + - raspi_img/*-raspi-aarch64.img + - raspi_img/*-raspi-aarch64.img.sha256sum + - raspi_img/*-raspi-aarch64.img.xz + - raspi_img/*-raspi-aarch64.img.xz.sha256sum + source: + - source/aarch64/EFI/ + - source/aarch64/Packages/ + - source/aarch64/docs/ + - source/aarch64/images/ + - source/aarch64/repodata/ + - source/aarch64/RPM-GPG-KEY-openEuler/ + - source/aarch64/TRANS.TBL + - source/aarch64/boot.catalog + stratovirt_img: + - stratovirt_img/aarch64/ + - stratovirt_img/x86_64/ + - stratovirt_img/aarch64/*-stratovirt-aarch64.img.xz + - stratovirt_img/aarch64/*-stratovirt-aarch64.img.xz.sha256sum + - stratovirt_img/aarch64/EFI/ + - stratovirt_img/aarch64/std-vmlinux.bin + - stratovirt_img/aarch64/std-vmlinux.bin.sha256sum + - stratovirt_img/aarch64/vmlinux.bin + - stratovirt_img/aarch64/vmlinux.bin.sha256sum + - stratovirt_img/x86_64/*-stratovirt-x86_64.img.xz + - stratovirt_img/x86_64/EFI/ + - stratovirt_img/x86_64/*-stratovirt-x86_64.img.xz.sha256sum + - stratovirt_img/x86_64/std-vmlinux.bin + - stratovirt_img/x86_64/std-vmlinux.bin.sha256sum + - stratovirt_img/x86_64/vmlinux.bin + - stratovirt_img/x86_64/vmlinux.bin.sha256sum + update: + - update/aarch64/ + - update/source/ + - update/x86_64/ + - update/aarch64/Packages/ + - update/aarch64/repodata/ + - update/source/Packages/ + - update/source/repodata/ + - update/x86_64/Packages/ + - update/x86_64/repodata/ + + virtual_machine_img: + - virtual_machine_img/aarch64/ + - virtual_machine_img/x86_64/ + - virtual_machine_img/aarch64/*-aarch64.qcow2.xz + - virtual_machine_img/aarch64/*-aarch64.qcow2.xz.sha256sum + - virtual_machine_img/x86_64/*-x86_64.qcow2.xz + - virtual_machine_img/x86_64/*-x86_64.qcow2.xz.sha256sum \ No newline at end of file -- Gitee From b0e809810b2913417811c2a2f6e1c3c7ea6acc1f Mon Sep 17 00:00:00 2001 From: dongjie110 <17621827400@163.com> Date: Tue, 13 Sep 2022 14:57:10 +0800 Subject: [PATCH 02/10] 2 --- script/tools/daily_build_check.py | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/script/tools/daily_build_check.py b/script/tools/daily_build_check.py index ebd0a87..3d133fe 100644 --- a/script/tools/daily_build_check.py +++ b/script/tools/daily_build_check.py @@ -235,6 +235,8 @@ class CheckDailyBuild(object): cmd = "cd %s && osc add %s && osc ci -m1" % (targer_dir, file) ret = os.popen(cmd).read() logging.info("compare result already upload to OBS project:{}".format(self.main_branch)) + self.post_to_api(dist_file) + logging.info("compare result already post to api:{}".format(self.main_branch)) else: logging.warning("this {} not in obs project".format(self.main_branch)) @@ -259,6 +261,18 @@ class CheckDailyBuild(object): with open(file_path, "w", encoding='utf-8') as f: yaml.dump(dict_msg, f, default_flow_style=False, sort_keys=False) + def post_to_api(self, filepath): + url = "https://radiatest.openeuler.org/api/v1/dailybuild" + files = { + "file": ("{}.yaml".format(self.datebranch), open(filepath, "rb")) + } + data = { + "product": self.main_branch, + "build": "build-{}".format(self.datebranch) + } + response = requests.post(url=url, files=files, data=data) + print(response.json()) + def run(self): self._get_main_branch() -- Gitee From 67cd97bfd12ed52dd87cbf9ba342b445d59a4573 Mon Sep 17 00:00:00 2001 From: dongjie110 <17621827400@163.com> Date: Tue, 13 Sep 2022 15:41:31 +0800 Subject: [PATCH 03/10] 3 --- script/tools/daily_build_check.py | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/script/tools/daily_build_check.py b/script/tools/daily_build_check.py index 3d133fe..c438757 100644 --- a/script/tools/daily_build_check.py +++ b/script/tools/daily_build_check.py @@ -200,10 +200,6 @@ class CheckDailyBuild(object): else: temp_dic[current_dir] = True compare_results["compare_results"].append(temp_dic) - # logging.info("********************************************CHECK RESULT*********************************************************") - # if error_flag: - # logging.error("some files or dirs not found in：{}".format(self.datebranch)) - # raise SystemExit("*****************************************PLEASE CHECK*****************************************************") self.write_yaml(compare_results) logging.info("********************************************CHECK RESULT*********************************************************") self._add_to_obs() @@ -235,8 +231,11 @@ class CheckDailyBuild(object): cmd = "cd %s && osc add %s && osc ci -m1" % (targer_dir, file) ret = os.popen(cmd).read() logging.info("compare result already upload to OBS project:{}".format(self.main_branch)) - self.post_to_api(dist_file) - logging.info("compare result already post to api:{}".format(self.main_branch)) + response_code = self.post_to_api(dist_file) + if response_code == '2000': + logging.info("compare result already post success") + else: + logging.error("compare result already post failed") else: logging.warning("this {} not in obs project".format(self.main_branch)) @@ -271,7 +270,10 @@ class CheckDailyBuild(object): "build": "build-{}".format(self.datebranch) } response = requests.post(url=url, files=files, data=data) - print(response.json()) + response_json = response.json() + logging.info(response_json) + response_code = response_json.get('error_code',0) + return response_code def run(self): -- Gitee From 26f1968afd858083c14f817bcb2b67de86f01add Mon Sep 17 00:00:00 2001 From: dongjie110 <17621827400@163.com> Date: Thu, 15 Sep 2022 16:50:12 +0800 Subject: [PATCH 04/10] 4 --- script/tools/daily_build_check.py | 67 ++-- script/tools/standard.yaml | 598 ++++++++++++++++++++---------- 2 files changed, 431 insertions(+), 234 deletions(-) diff --git a/script/tools/daily_build_check.py b/script/tools/daily_build_check.py index c438757..a69c218 100644 --- a/script/tools/daily_build_check.py +++ b/script/tools/daily_build_check.py @@ -46,7 +46,6 @@ class CheckDailyBuild(object): self.dirflag = self.kwargs['dir_flag'] self.datebranch = self.kwargs['date_branch'] self.mountflag = self.kwargs['mountflag'] - self.standard_dir = self.load_standard() def html_downloader(self, url): """ @@ -148,62 +147,55 @@ class CheckDailyBuild(object): result = yaml_result[self.main_branch] else: result = yaml_result['standard'] + template_result = yaml_result['template'] if self.mountflag == '0': for delitem in UNMOUNT_DIRS: del result[delitem] - return result + return result, template_result except Exception as e: logging.info(e) logging.info("error read standard.yaml,please check") - # def compare_standard(self, dir_list, current_branch, c_dir): - # """ - # branch dir compare with standard dir - # """ - # logging.info("********************************************CHECK RESULT*********************************************************") - # standard_dir = self.standard_dir - # error_flag = False - # for key, c_standard in standard_dir.items(): - # for current_dir in c_standard: - # if '*' in current_dir: - # current_dir = current_dir.replace('*', current_branch) - # if current_dir not in dir_list: - # error_flag = True - # logging.error( - # 'this dir or file not found,link url:{}{}'.format( - # c_dir, current_dir)) - # logging.info("********************************************CHECK RESULT*********************************************************") - # if error_flag: - # logging.error("some files or dirs not found in：{}".format(self.datebranch)) - # raise SystemExit("*****************************************PLEASE CHECK*****************************************************") - def compare_standard(self, dir_list, current_branch, c_dir): """ branch dir compare with standard dir """ - logging.info("********************************************CHECK RESULT*********************************************************") - print (dir_list) - compare_results = {"compare_results":[]} - standard_dir = self.standard_dir - error_flag = False + logging.info("*******************************************CHECK RESULT*********************************************************") + miss_items = [] + standard_dir, template_dir = self.load_standard() for key, c_standard in standard_dir.items(): for current_dir in c_standard: - # if '*' in current_dir: - current_dir = current_dir.replace('*', current_branch) - temp_dic = {} + # current_dir = current_dir.replace('$', current_branch) if current_dir not in dir_list: - error_flag = True - temp_dic[current_dir] = False + miss_items.append(current_dir) logging.error( 'this dir or file not found,link url:{}{}'.format( c_dir, current_dir)) - else: - temp_dic[current_dir] = True - compare_results["compare_results"].append(temp_dic) + compare_results = self.parse_results(template_dir, miss_items, current_branch) self.write_yaml(compare_results) - logging.info("********************************************CHECK RESULT*********************************************************") + logging.info("*******************************************CHECK RESULT*********************************************************") self._add_to_obs() + def parse_results(self, result, miss_items, current_branch): + for line in miss_items: + temp = line.split('/') + if temp[-1] == '': + temp.pop(-1) + num = len(temp) + if num == 1: + result[temp[0]]['value'] = False + elif num ==2: + result[temp[0]][temp[1]]['value'] = False + elif num == 3: + result[temp[0]][temp[1]][temp[2]]['value'] = False + elif num == 4: + result[temp[0]][temp[1]][temp[2]][temp[3]]['value'] = False + elif num == 5: + result[temp[0]][temp[1]][temp[2]][temp[3]][temp[4]]['value'] = False + elif num == 6: + result[temp[0]][temp[1]][temp[2]][temp[3]][temp[4]][temp[5]]['value'] = False + return result + def check_input_args(self, complete_url): html_content = self.html_downloader(complete_url) @@ -218,7 +210,6 @@ class CheckDailyBuild(object): """ add result file to obs project """ - # src_file = os.path.join(os.getcwd(),"{}.yaml".format(self.datebranch)) file = "{}.yaml".format(self.datebranch) path=os.getcwd() cmd = "cd %s && osc co home:Admin:ISO/%s" % (path,self.main_branch) diff --git a/script/tools/standard.yaml b/script/tools/standard.yaml index 50ad513..5eb25a5 100644 --- a/script/tools/standard.yaml +++ b/script/tools/standard.yaml @@ -28,8 +28,11 @@ standard: - EPOL/main/source/repodata/ - EPOL/main/aarch64/Packages/ - EPOL/main/aarch64/repodata/ + - EPOL/main/x86_64/Packages/ - EPOL/main/x86_64/repodata/ - EPOL/update/multi_version/source/ + - EPOL/update/multi_version/aarch64/ + - EPOL/update/multi_version/x86_64/ - EPOL/update/main/aarch64/Packages/ - EPOL/update/main/aarch64/repodata/ - EPOL/update/main/source/Packages/ @@ -66,24 +69,24 @@ standard: - ISO/aarch64/ - ISO/x86_64/ - ISO/source/ - - ISO/aarch64/*-aarch64-dvd.iso - - ISO/aarch64/*-aarch64-dvd.iso.sha256sum - - ISO/aarch64/*-aarch64.rpmlist - - ISO/aarch64/*-everything-aarch64-dvd.iso - - ISO/aarch64/*-everything-aarch64-dvd.iso.sha256sum - - ISO/aarch64/*-everything-debug-aarch64-dvd.iso - - ISO/aarch64/*-everything-debug-aarch64-dvd.iso.sha256sum - - ISO/aarch64/*-netinst-aarch64-dvd.iso - - ISO/aarch64/*-netinst-aarch64-dvd.iso.sha256sum - - ISO/source/*-source-dvd.iso - - ISO/source/*-source-dvd.iso.sha256sum - - ISO/x86_64/*-everything-debug-x86_64-dvd.iso - - ISO/x86_64/*-everything-debug-x86_64-dvd.iso.sha256sum - - ISO/x86_64/*-netinst-x86_64-dvd.iso - - ISO/x86_64/*-netinst-x86_64-dvd.iso.sha256sum - - ISO/x86_64/*-x86_64-dvd.iso - - ISO/x86_64/*-x86_64-dvd.iso.sha256sum - - ISO/x86_64/*-x86_64.rpmlist + - ISO/aarch64/$-aarch64-dvd.iso + - ISO/aarch64/$-aarch64-dvd.iso.sha256sum + - ISO/aarch64/$-aarch64.rpmlist + - ISO/aarch64/$-everything-aarch64-dvd.iso + - ISO/aarch64/$-everything-aarch64-dvd.iso.sha256sum + - ISO/aarch64/$-everything-debug-aarch64-dvd.iso + - ISO/aarch64/$-everything-debug-aarch64-dvd.iso.sha256sum + - ISO/aarch64/$-netinst-aarch64-dvd.iso + - ISO/aarch64/$-netinst-aarch64-dvd.iso.sha256sum + - ISO/source/$-source-dvd.iso + - ISO/source/$-source-dvd.iso.sha256sum + - ISO/x86_64/$-everything-debug-x86_64-dvd.iso + - ISO/x86_64/$-everything-debug-x86_64-dvd.iso.sha256sum + - ISO/x86_64/$-netinst-x86_64-dvd.iso + - ISO/x86_64/$-netinst-x86_64-dvd.iso.sha256sum + - ISO/x86_64/$-x86_64-dvd.iso + - ISO/x86_64/$-x86_64-dvd.iso.sha256sum + - ISO/x86_64/$-x86_64.rpmlist OS: - OS/aarch64/ - OS/x86_64/ @@ -149,11 +152,12 @@ standard: - everything/x86_64/TRANS.TBL - everything/x86_64/boot.catalog raspi_img: - - raspi_img/*-raspi-aarch64.img - - raspi_img/*-raspi-aarch64.img.sha256sum - - raspi_img/*-raspi-aarch64.img.xz - - raspi_img/*-raspi-aarch64.img.xz.sha256sum + - raspi_img/$-raspi-aarch64.img + - raspi_img/$-raspi-aarch64.img.sha256sum + - raspi_img/$-raspi-aarch64.img.xz + - raspi_img/$-raspi-aarch64.img.xz.sha256sum source: + - source/aarch64/ - source/aarch64/EFI/ - source/aarch64/Packages/ - source/aarch64/docs/ @@ -165,16 +169,16 @@ standard: stratovirt_img: - stratovirt_img/aarch64/ - stratovirt_img/x86_64/ - - stratovirt_img/aarch64/*-stratovirt-aarch64.img.xz - - stratovirt_img/aarch64/*-stratovirt-aarch64.img.xz.sha256sum + - stratovirt_img/aarch64/$-stratovirt-aarch64.img.xz + - stratovirt_img/aarch64/$-stratovirt-aarch64.img.xz.sha256sum - stratovirt_img/aarch64/EFI/ - stratovirt_img/aarch64/std-vmlinux.bin - stratovirt_img/aarch64/std-vmlinux.bin.sha256sum - stratovirt_img/aarch64/vmlinux.bin - stratovirt_img/aarch64/vmlinux.bin.sha256sum - - stratovirt_img/x86_64/*-stratovirt-x86_64.img.xz + - stratovirt_img/x86_64/$-stratovirt-x86_64.img.xz - stratovirt_img/x86_64/EFI/ - - stratovirt_img/x86_64/*-stratovirt-x86_64.img.xz.sha256sum + - stratovirt_img/x86_64/$-stratovirt-x86_64.img.xz.sha256sum - stratovirt_img/x86_64/std-vmlinux.bin - stratovirt_img/x86_64/std-vmlinux.bin.sha256sum - stratovirt_img/x86_64/vmlinux.bin @@ -193,184 +197,386 @@ standard: virtual_machine_img: - virtual_machine_img/aarch64/ - virtual_machine_img/x86_64/ - - virtual_machine_img/aarch64/*-aarch64.qcow2.xz - - virtual_machine_img/aarch64/*-aarch64.qcow2.xz.sha256sum - - virtual_machine_img/x86_64/*-x86_64.qcow2.xz - - virtual_machine_img/x86_64/*-x86_64.qcow2.xz.sha256sum -openEuler-22.09-1: - MAIN_DIR: - - EPOL/ - - ISO/ - - OS/ - - debuginfo/ - - docker_img/ - - edge_img/ - - embedded_img/ - - everything/ - - raspi_img/ - - source/ - - stratovirt_img/ - - update/ - - virtual_machine_img/ + - virtual_machine_img/aarch64/$-aarch64.qcow2.xz + - virtual_machine_img/aarch64/$-aarch64.qcow2.xz.sha256sum + - virtual_machine_img/x86_64/$-x86_64.qcow2.xz + - virtual_machine_img/x86_64/$-x86_64.qcow2.xz.sha256sum +template: EPOL: - - EPOL/main/ - - EPOL/update/ - - EPOL/main/source/ - - EPOL/main/aarch64/ - - EPOL/main/x86_64/ - - EPOL/update/main/ - - EPOL/update/multi_version/ - - EPOL/update/main/aarch64/ - - EPOL/main/source/Packages/ - - EPOL/main/source/repodata/ - - EPOL/main/aarch64/Packages/ - - EPOL/main/aarch64/repodata/ - - EPOL/main/x86_64/repodata/ - - EPOL/update/multi_version/source/ - - EPOL/update/main/aarch64/Packages/ - - EPOL/update/main/aarch64/repodata/ - - EPOL/update/main/source/Packages/ - - EPOL/update/main/source/repodata/ - - EPOL/update/main/x86_64/Packages/ - - EPOL/update/main/x86_64/repodata/ - - EPOL/update/multi_version/source/Packages/ - - EPOL/update/multi_version/source/repodata/ - - EPOL/update/multi_version/x86_64/Packages/ - - EPOL/update/multi_version/x86_64/repodata/ - - EPOL/update/multi_version/aarch64/Packages/ - - EPOL/update/multi_version/aarch64/repodata/ + value: true + main: + value: true + source: + value: true + Packages: + value: true + repodata: + value: true + aarch64: + value: true + Packages: + value: true + repodata: + value: true + x86_64: + value: true + Packages: + value: true + repodata: + value: true + update: + value: true + main: + value: true + source: + value: true + Packages: + value: true + repodata: + value: true + aarch64: + value: true + Packages: + value: true + repodata: + value: true + x86_64: + value: true + Packages: + value: true + repodata: + value: true + multi_version: + value: true + source: + value: true + Packages: + value: true + repodata: + value: true + aarch64: + value: true + Packages: + value: true + repodata: + value: true + x86_64: + value: true + Packages: + value: true + repodata: + value: true + multi_version: + value: true + OpenStack: + value: true + Train: + value: true + source: + value: true + Packages: + value: true + repodata: + value: true + aarch64: + value: true + Packages: + value: true + repodata: + value: true + x86_64: + value: true + Packages: + value: true + repodata: + value: true + Wallaby: + value: true + source: + value: true + Packages: + value: true + repodata: + value: true + aarch64: + value: true + Packages: + value: true + repodata: + value: true + x86_64: + value: true + Packages: + value: true + repodata: + value: true ISO: - - ISO/aarch64/ - - ISO/x86_64/ - - ISO/source/ - - ISO/aarch64/*-aarch64-dvd.iso - - ISO/aarch64/*-aarch64-dvd.iso.sha256sum - - ISO/aarch64/*-aarch64.rpmlist - - ISO/aarch64/*-everything-aarch64-dvd.iso - - ISO/aarch64/*-everything-aarch64-dvd.iso.sha256sum - - ISO/aarch64/*-everything-debug-aarch64-dvd.iso - - ISO/aarch64/*-everything-debug-aarch64-dvd.iso.sha256sum - - ISO/aarch64/*-netinst-aarch64-dvd.iso - - ISO/aarch64/*-netinst-aarch64-dvd.iso.sha256sum - - ISO/source/*-source-dvd.iso - - ISO/source/*-source-dvd.iso.sha256sum - - ISO/x86_64/*-everything-debug-x86_64-dvd.iso - - ISO/x86_64/*-everything-debug-x86_64-dvd.iso.sha256sum - - ISO/x86_64/*-netinst-x86_64-dvd.iso - - ISO/x86_64/*-netinst-x86_64-dvd.iso.sha256sum - - ISO/x86_64/*-x86_64-dvd.iso - - ISO/x86_64/*-x86_64-dvd.iso.sha256sum - - ISO/x86_64/*-x86_64.rpmlist + value: true + aarch64: + value: true + $-aarch64-dvd.iso: + value: true + $-aarch64-dvd.iso.sha256sum: + value: true + $-aarch64.rpmlist: + value: true + $-everything-aarch64-dvd.iso: + value: true + $-everything-aarch64-dvd.iso.sha256sum: + value: true + $-everything-debug-aarch64-dvd.iso: + value: true + $-everything-debug-aarch64-dvd.iso.sha256sum: + value: true + $-netinst-aarch64-dvd.iso: + value: true + $-netinst-aarch64-dvd.iso.sha256sum: + value: true + x86_64: + value: true + $-everything-debug-x86_64-dvd.iso: + value: true + $-everything-debug-x86_64-dvd.iso.sha256sum: + value: true + $-netinst-x86_64-dvd.iso: + value: true + $-netinst-x86_64-dvd.iso.sha256sum: + value: true + $-x86_64-dvd.iso: + value: true + $-x86_64-dvd.iso.sha256sum: + value: true + $-x86_64.rpmlist: + value: true + source: + value: true + $-source-dvd.iso: + value: true + $-source-dvd.iso.sha256sum: + value: true OS: - - OS/aarch64/ - - OS/x86_64/ - - OS/aarch64/EFI/ - - OS/aarch64/Packages/ - - OS/aarch64/docs/ - - OS/aarch64/images/ - - OS/aarch64/repodata/ - - OS/aarch64/RPM-GPG-KEY-openEuler/ - - OS/aarch64/TRANS.TBL - - OS/aarch64/boot.catalog - - OS/x86_64/EFI/ - - OS/x86_64/Packages/ - - OS/x86_64/docs/ - - OS/x86_64/images/ - - OS/x86_64/repodata/ - - OS/x86_64/RPM-GPG-KEY-openEuler/ - - OS/x86_64/TRANS.TBL - - OS/x86_64/boot.catalog + value: true + aarch64: + value: true + EFI: + value: true + Packages: + value: true + docs: + value: true + images: + value: true + repodata: + value: true + RPM-GPG-KEY-openEuler: + value: true + TRANS.TBL: + value: true + boot.catalog: + value: true + x86_64: + value: true + EFI: + value: true + Packages: + value: true + docs: + value: true + images: + value: true + repodata: + value: true + RPM-GPG-KEY-openEuler: + value: true + TRANS.TBL: + value: true + boot.catalog: + value: true debuginfo: - - debuginfo/aarch64/ - - debuginfo/x86_64/ - - debuginfo/aarch64/EFI/ - - debuginfo/aarch64/Packages/ - - debuginfo/aarch64/docs/ - - debuginfo/aarch64/images/ - - debuginfo/aarch64/repodata/ - - debuginfo/aarch64/RPM-GPG-KEY-openEuler/ - - debuginfo/aarch64/TRANS.TBL - - debuginfo/aarch64/boot.catalog - - debuginfo/x86_64/EFI/ - - debuginfo/x86_64/Packages/ - - debuginfo/x86_64/docs/ - - debuginfo/x86_64/images/ - - debuginfo/x86_64/repodata/ - - debuginfo/x86_64/RPM-GPG-KEY-openEuler/ - - debuginfo/x86_64/TRANS.TBL - - debuginfo/x86_64/boot.catalog + value: true + aarch64: + value: true + EFI: + value: true + Packages: + value: true + docs: + value: true + images: + value: true + repodata: + value: true + RPM-GPG-KEY-openEuler: + value: true + TRANS.TBL: + value: true + boot.catalog: + value: true + x86_64: + value: true + EFI: + value: true + Packages: + value: true + docs: + value: true + images: + value: true + repodata: + value: true + RPM-GPG-KEY-openEuler: + value: true + TRANS.TBL: + value: true + boot.catalog: + value: true docker_img: - - docker_img/aarch64/ - - docker_img/x86_64/ - - docker_img/aarch64/openEuler-docker.aarch64.tar.xz - - docker_img/aarch64/openEuler-docker.aarch64.tar.xz.sha256sum - - docker_img/x86_64/openEuler-docker.x86_64.tar.xz - - docker_img/x86_64/openEuler-docker.x86_64.tar.xz.sha256sum + value: true + aarch64: + value: true + openEuler-docker.aarch64.tar.xz: + value: true + openEuler-docker.aarch64.tar.xz.sha256sum: + value: true + x86_64: + value: true + openEuler-docker.x86_64.tar.xz: + value: true + openEuler-docker.x86_64.tar.xz.sha256sum: + value: true everything: - - everything/aarch64/ - - everything/x86_64/ - - everything/aarch64/EFI/ - - everything/aarch64/Packages/ - - everything/aarch64/docs/ - - everything/aarch64/images/ - - everything/aarch64/repodata/ - - everything/aarch64/RPM-GPG-KEY-openEuler/ - - everything/aarch64/TRANS.TBL - - everything/aarch64/boot.catalog - - everything/x86_64/EFI/ - - everything/x86_64/Packages/ - - everything/x86_64/docs/ - - everything/x86_64/images/ - - everything/x86_64/repodata/ - - everything/x86_64/RPM-GPG-KEY-openEuler/ - - everything/x86_64/TRANS.TBL - - everything/x86_64/boot.catalog + value: true + aarch64: + value: true + EFI: + value: true + Packages: + value: true + docs: + value: true + images: + value: true + repodata: + value: true + RPM-GPG-KEY-openEuler: + value: true + TRANS.TBL: + value: true + boot.catalog: + value: true + x86_64: + value: true + EFI: + value: true + Packages: + value: true + docs: + value: true + images: + value: true + repodata: + value: true + RPM-GPG-KEY-openEuler: + value: true + TRANS.TBL: + value: true + boot.catalog: + value: true raspi_img: - - raspi_img/*-raspi-aarch64.img - - raspi_img/*-raspi-aarch64.img.sha256sum - - raspi_img/*-raspi-aarch64.img.xz - - raspi_img/*-raspi-aarch64.img.xz.sha256sum + value: true + $-raspi-aarch64.img: + value: true + $-raspi-aarch64.img.sha256sum: + value: true + $-raspi-aarch64.img.xz: + value: true + $-raspi-aarch64.img.xz.sha256sum: + value: true source: - - source/aarch64/EFI/ - - source/aarch64/Packages/ - - source/aarch64/docs/ - - source/aarch64/images/ - - source/aarch64/repodata/ - - source/aarch64/RPM-GPG-KEY-openEuler/ - - source/aarch64/TRANS.TBL - - source/aarch64/boot.catalog + value: true + aarch64: + value: true + EFI: + value: true + Packages: + value: true + docs: + value: true + images: + value: true + repodata: + value: true + RPM-GPG-KEY-openEuler: + value: true + TRANS.TBL: + value: true + boot.catalog: + value: true stratovirt_img: - - stratovirt_img/aarch64/ - - stratovirt_img/x86_64/ - - stratovirt_img/aarch64/*-stratovirt-aarch64.img.xz - - stratovirt_img/aarch64/*-stratovirt-aarch64.img.xz.sha256sum - - stratovirt_img/aarch64/EFI/ - - stratovirt_img/aarch64/std-vmlinux.bin - - stratovirt_img/aarch64/std-vmlinux.bin.sha256sum - - stratovirt_img/aarch64/vmlinux.bin - - stratovirt_img/aarch64/vmlinux.bin.sha256sum - - stratovirt_img/x86_64/*-stratovirt-x86_64.img.xz - - stratovirt_img/x86_64/EFI/ - - stratovirt_img/x86_64/*-stratovirt-x86_64.img.xz.sha256sum - - stratovirt_img/x86_64/std-vmlinux.bin - - stratovirt_img/x86_64/std-vmlinux.bin.sha256sum - - stratovirt_img/x86_64/vmlinux.bin - - stratovirt_img/x86_64/vmlinux.bin.sha256sum + value: true + aarch64: + value: true + $-stratovirt-aarch64.img.xz: + value: true + $-stratovirt-aarch64.img.xz.sha256sum: + value: true + EFI: + value: true + std-vmlinux.bin: + value: true + std-vmlinux.bin.sha256sum: + value: true + vmlinux.bin: + value: true + vmlinux.bin.sha256sum: + value: true + x86_64: + value: true + $-stratovirt-x86_64.img.xz: + value: true + EFI: + value: true + $-stratovirt-x86_64.img.xz.sha256sum: + value: true + std-vmlinux.bin: + value: true + std-vmlinux.bin.sha256sum: + value: true + vmlinux.bin: + value: true + vmlinux.bin.sha256sum: + value: true update: - - update/aarch64/ - - update/source/ - - update/x86_64/ - - update/aarch64/Packages/ - - update/aarch64/repodata/ - - update/source/Packages/ - - update/source/repodata/ - - update/x86_64/Packages/ - - update/x86_64/repodata/ - + value: true + source: + value: true + Packages: + value: true + repodata: + value: true + aarch64: + value: true + Packages: + value: true + repodata: + value: true + x86_64: + value: true + Packages: + value: true + repodata: + value: true virtual_machine_img: - - virtual_machine_img/aarch64/ - - virtual_machine_img/x86_64/ - - virtual_machine_img/aarch64/*-aarch64.qcow2.xz - - virtual_machine_img/aarch64/*-aarch64.qcow2.xz.sha256sum - - virtual_machine_img/x86_64/*-x86_64.qcow2.xz - - virtual_machine_img/x86_64/*-x86_64.qcow2.xz.sha256sum \ No newline at end of file + value: true + aarch64: + value: true + $-aarch64.qcow2.xz: + value: true + $-aarch64.qcow2.xz.sha256sum: + value: true + x86_64: + value: true + $-x86_64.qcow2.xz: + value: true + $-x86_64.qcow2.xz.sha256sum: + value: true \ No newline at end of file -- Gitee From e5afe4087abc43fbceffe6ce678fdfba1753937e Mon Sep 17 00:00:00 2001 From: dongjie110 <17621827400@163.com> Date: Thu, 15 Sep 2022 16:57:56 +0800 Subject: [PATCH 05/10] sd --- script/tools/daily_build_check.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/script/tools/daily_build_check.py b/script/tools/daily_build_check.py index a69c218..41dc744 100644 --- a/script/tools/daily_build_check.py +++ b/script/tools/daily_build_check.py @@ -171,9 +171,9 @@ class CheckDailyBuild(object): logging.error( 'this dir or file not found,link url:{}{}'.format( c_dir, current_dir)) + logging.info("*******************************************CHECK RESULT*********************************************************") compare_results = self.parse_results(template_dir, miss_items, current_branch) self.write_yaml(compare_results) - logging.info("*******************************************CHECK RESULT*********************************************************") self._add_to_obs() def parse_results(self, result, miss_items, current_branch): @@ -184,7 +184,7 @@ class CheckDailyBuild(object): num = len(temp) if num == 1: result[temp[0]]['value'] = False - elif num ==2: + elif num == 2: result[temp[0]][temp[1]]['value'] = False elif num == 3: result[temp[0]][temp[1]][temp[2]]['value'] = False -- Gitee From d00a73fed0a8b9e3585ef2a1f9e34741a957ac53 Mon Sep 17 00:00:00 2001 From: dongjie110 <17621827400@163.com> Date: Wed, 22 Mar 2023 15:14:57 +0800 Subject: [PATCH 06/10] 3232 --- script/tools/daily_build_check.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/script/tools/daily_build_check.py b/script/tools/daily_build_check.py index 41dc744..6860376 100644 --- a/script/tools/daily_build_check.py +++ b/script/tools/daily_build_check.py @@ -216,12 +216,12 @@ class CheckDailyBuild(object): ret = os.popen(cmd).read() targer_dir = os.path.join(path, "home:Admin:ISO/{}".format(self.main_branch)) if os.path.exists(targer_dir): - src_file = os.path.join(path, file) + # src_file = os.path.join(path, file) dist_file = os.path.join(path, targer_dir, file) - shutil.copyfile(src_file, dist_file) - cmd = "cd %s && osc add %s && osc ci -m1" % (targer_dir, file) - ret = os.popen(cmd).read() - logging.info("compare result already upload to OBS project:{}".format(self.main_branch)) + # shutil.copyfile(src_file, dist_file) + # cmd = "cd %s && osc add %s && osc ci -m1" % (targer_dir, file) + # ret = os.popen(cmd).read() + # logging.info("compare result already upload to OBS project:{}".format(self.main_branch)) response_code = self.post_to_api(dist_file) if response_code == '2000': logging.info("compare result already post success") -- Gitee From 91e23d779c9336da814d1beb02d277605d6e983f Mon Sep 17 00:00:00 2001 From: dongjie110 <17621827400@163.com> Date: Wed, 22 Mar 2023 15:23:08 +0800 Subject: [PATCH 07/10] 667 --- script/tools/daily_build_check.py | 1 + 1 file changed, 1 insertion(+) diff --git a/script/tools/daily_build_check.py b/script/tools/daily_build_check.py index 6860376..0270a83 100644 --- a/script/tools/daily_build_check.py +++ b/script/tools/daily_build_check.py @@ -248,6 +248,7 @@ class CheckDailyBuild(object): def write_yaml(self, dict_msg): file_path = os.path.join(os.getcwd(),"{}.yaml".format(self.datebranch)) + logging.info("======================================{}".format(file_path)) with open(file_path, "w", encoding='utf-8') as f: yaml.dump(dict_msg, f, default_flow_style=False, sort_keys=False) -- Gitee From 6ff068c92d4b0dc24d9b1981c194c691e16f0ff9 Mon Sep 17 00:00:00 2001 From: dongjie110 <17621827400@163.com> Date: Wed, 22 Mar 2023 15:37:14 +0800 Subject: [PATCH 08/10] 55 --- script/tools/daily_build_check.py | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/script/tools/daily_build_check.py b/script/tools/daily_build_check.py index 0270a83..85aff71 100644 --- a/script/tools/daily_build_check.py +++ b/script/tools/daily_build_check.py @@ -210,14 +210,16 @@ class CheckDailyBuild(object): """ add result file to obs project """ - file = "{}.yaml".format(self.datebranch) - path=os.getcwd() - cmd = "cd %s && osc co home:Admin:ISO/%s" % (path,self.main_branch) - ret = os.popen(cmd).read() - targer_dir = os.path.join(path, "home:Admin:ISO/{}".format(self.main_branch)) - if os.path.exists(targer_dir): + # file = "{}.yaml".format(self.datebranch) + # path=os.getcwd() + # dist_file = + dist_file = os.path.join(os.getcwd(),"{}.yaml".format(self.datebranch)) + # cmd = "cd %s && osc co home:Admin:ISO/%s" % (path,self.main_branch) + # ret = os.popen(cmd).read() + # targer_dir = os.path.join(path, "home:Admin:ISO/{}".format(self.main_branch)) + if os.path.exists(dist_file): # src_file = os.path.join(path, file) - dist_file = os.path.join(path, targer_dir, file) + # dist_file = os.path.join(path, targer_dir, file) # shutil.copyfile(src_file, dist_file) # cmd = "cd %s && osc add %s && osc ci -m1" % (targer_dir, file) # ret = os.popen(cmd).read() -- Gitee From cdf700bab9aa5e414efa0b0795f29a3bf9dd59ba Mon Sep 17 00:00:00 2001 From: dongjie110 <17621827400@163.com> Date: Wed, 22 Mar 2023 15:42:38 +0800 Subject: [PATCH 09/10] 77 --- script/tools/daily_build_check.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/script/tools/daily_build_check.py b/script/tools/daily_build_check.py index 85aff71..49747c1 100644 --- a/script/tools/daily_build_check.py +++ b/script/tools/daily_build_check.py @@ -250,7 +250,6 @@ class CheckDailyBuild(object): def write_yaml(self, dict_msg): file_path = os.path.join(os.getcwd(),"{}.yaml".format(self.datebranch)) - logging.info("======================================{}".format(file_path)) with open(file_path, "w", encoding='utf-8') as f: yaml.dump(dict_msg, f, default_flow_style=False, sort_keys=False) @@ -263,6 +262,7 @@ class CheckDailyBuild(object): "product": self.main_branch, "build": "build-{}".format(self.datebranch) } + logging.info("post data:{}".format(data)) response = requests.post(url=url, files=files, data=data) response_json = response.json() logging.info(response_json) -- Gitee From 37fe44f8565af67a2467d42d0e973dfcfd3cb215 Mon Sep 17 00:00:00 2001 From: dongjie110 <17621827400@163.com> Date: Wed, 22 Mar 2023 15:48:06 +0800 Subject: [PATCH 10/10] 66 --- script/tools/daily_build_check.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/script/tools/daily_build_check.py b/script/tools/daily_build_check.py index 49747c1..4cc835e 100644 --- a/script/tools/daily_build_check.py +++ b/script/tools/daily_build_check.py @@ -259,7 +259,7 @@ class CheckDailyBuild(object): "file": ("{}.yaml".format(self.datebranch), open(filepath, "rb")) } data = { - "product": self.main_branch, + "product": self.main_branch.replace('EBS-',''), "build": "build-{}".format(self.datebranch) } logging.info("post data:{}".format(data)) -- Gitee