From 866b8ae3698e5467556f552444894da0237c42f5 Mon Sep 17 00:00:00 2001 From: root Date: Fri, 11 Sep 2020 09:13:41 +0800 Subject: [PATCH] support auto create job for openeuler --- src/conf/ignore_repo.yaml | 28 +++--- src/jobs/__init__.py | 0 src/jobs/jenkins_job.py | 95 +++++++++++++++++++-- src/jobs/obs_meta_strategy.py | 0 src/jobs/{repo_buddy.py => repo_mapping.py} | 92 +++++++++++++++----- 5 files changed, 169 insertions(+), 46 deletions(-) mode change 100644 => 100755 src/jobs/__init__.py mode change 100644 => 100755 src/jobs/jenkins_job.py mode change 100644 => 100755 src/jobs/obs_meta_strategy.py rename src/jobs/{repo_buddy.py => repo_mapping.py} (56%) mode change 100644 => 100755 diff --git a/src/conf/ignore_repo.yaml b/src/conf/ignore_repo.yaml index 9348b16..914fb12 100644 --- a/src/conf/ignore_repo.yaml +++ b/src/conf/ignore_repo.yaml @@ -1,14 +1,14 @@ -ignore: -- ci_check -- build -- obs_bin -- ci-bot -- website -- community -- docs -- infrastructure -- obs_meta -- euleros-latest-release -- image-slim -- risc-v-kernel -- opensbi +src-openeuler: + - ci_check + - build + - obs_bin + - ci-bot + - website + - community + - docs + - infrastructure + - obs_meta + - euleros-latest-release + - image-slim + - risc-v-kernel + - opensbi diff --git a/src/jobs/__init__.py b/src/jobs/__init__.py old mode 100644 new mode 100755 diff --git a/src/jobs/jenkins_job.py b/src/jobs/jenkins_job.py old mode 100644 new mode 100755 index ba8c20a..5b87e9e --- a/src/jobs/jenkins_job.py +++ b/src/jobs/jenkins_job.py @@ -5,9 +5,11 @@ monkey.patch_all() from abc import ABCMeta, abstractmethod import os +import stat import logging.config import logging import time +import re import xml.etree.ElementTree as ET import yaml import argparse @@ -181,21 +183,92 @@ class SrcOpenEulerJenkinsJobs(JenkinsJobs): return ET.tostring(root) -class OpenEulerJenkinsJobs(JenkinsJobs): +class OpenEulerJenkinsJobs(SrcOpenEulerJenkinsJobs): """ openEuler 仓库 - TODO... """ - def get_real_target_jobs(self, jobs): - super(OpenEulerJenkinsJobs, self).get_real_target_jobs(jobs) + def guess_build_script(self, job): + """ + 返回仓库对应的jenkins build脚本 + :param job: + :return: + """ + jenkinsfile_dir = os.path.realpath(os.path.join(__file__, "../../jenkinsfile")) + + script = job + for filename in os.listdir(jenkinsfile_dir): + if filename == script: + break + if re.match("{}\..*".format(job), filename): # repo.{sufix} + script = filename + break + + script = os.path.realpath(os.path.join(jenkinsfile_dir, script)) + + # helper chmod +x + if os.path.exists(script): + st = os.stat(script) + os.chmod(script, st.st_mode | stat.S_IXUSR | stat.S_IXGRP | stat.S_IXOTH) + + return script def update_config(self, job): - pass + """ + 根据模板生成目标任务配置信息 + :param job: 目标任务 + :return: xml string + """ + root = ET.fromstring(self._template_job_config.encode("utf-8")) + + buddy = self._buddy_info[job] # + + # triggers + ele = root.find("triggers//regexpFilterExpression") + if ele is not None: + ele.text = ele.text.replace(self._template_job, buddy["repo"]) + + # parameterized trigger + ele = root.find("publishers/hudson.plugins.parameterizedtrigger.BuildTrigger//projects") + if ele is not None: + arches = self._exclusive_arch.get(buddy["repo"]) + if arches: # eg: [x86_64] + projects = [] + for project in ele.text.split(","): + for arch in arches: + if arch in project: + projects.append(project) + ele.text = ",".join(projects).replace(self._template_job, buddy["repo"]) + else: + ele.text = ele.text.replace(self._template_job, buddy["repo"]) + + # build + script = self.guess_build_script(buddy["repo"]) + logger.debug("guess build script: {}".format("script")) + ele = root.findall("buiders/hudson.task.Shell/command") + if ele: + # replace first command + command = ele[0] + command.text = script + + # join trigger + ele = root.find("publishers/join.JoinTrigger//projects") + if ele is not None: + ele.text = ele.text.replace(self._template_job, buddy["repo"]) + + # set repo defaultValue + ele = root.find("properties//*[name=\"repo\"]/defaultValue") + if ele is not None: + ele.text = buddy["repo"] + + return ET.tostring(root) if "__main__" == __name__: args = argparse.ArgumentParser() - args.add_argument("-a", type=str, dest="action", help="workspace where to find source") + + args.add_argument("-f", type=str, dest="community", default="src-openeuler", help="src-openeuler or openeuler") + + args.add_argument("-a", type=str, dest="action", help="create or update") args.add_argument("-c", type=int, dest="concurrency", default=75, help="jobs that send to jenkins server concurrency") args.add_argument("-r", type=int, dest="retry", default=3, help="retry times") args.add_argument("-i", type=int, dest="interval", default=0, help="retry interval") @@ -206,10 +279,10 @@ if "__main__" == __name__: args.add_argument("-d", type=str, dest="target_job_base_url", help="jenkins base url of target jobs") args.add_argument("-o", type=int, dest="jenkins_timeout", default=10, help="jenkins api timeout") - args.add_argument("-u", type=str, dest="jenkins_user", help="repo name") + args.add_argument("-u", type=str, dest="jenkins_user", help="jenkins user name") args.add_argument("-t", type=str, dest="jenkins_api_token", help="jenkins api token") - args.add_argument("-x", type=str, dest="buddy_info_file", help="src-openeuler buddy info file") + args.add_argument("-x", type=str, dest="mapping_info_file", help="mapping info file") args.add_argument("-e", type=str, dest="exclusive_arch_file", help="exclusive arch file") args = args.parse_args() @@ -224,5 +297,9 @@ if "__main__" == __name__: jp_m = JenkinsProxy(args.template_job_base_url, args.jenkins_user, args.jenkins_api_token, args.jenkins_timeout) jp_t = JenkinsProxy(args.target_job_base_url, args.jenkins_user, args.jenkins_api_token, args.jenkins_timeout) - jenkins_jobs = SrcOpenEulerJenkinsJobs(args.template_job, jp_m, args.buddy_info_file, args.exclusive_arch_file) + if args.community == "src-openeuler": + jenkins_jobs = SrcOpenEulerJenkinsJobs(args.template_job, jp_m, args.mapping_info_file, args.exclusive_arch_file) + else: + jenkins_jobs = OpenEulerJenkinsJobs(args.template_job, jp_m, args.mapping_info_file, args.exclusive_arch_file) + jenkins_jobs.run(args.action, args.target_jobs, jp_t, concurrency=args.concurrency, retry=args.retry, interval=args.interval) diff --git a/src/jobs/obs_meta_strategy.py b/src/jobs/obs_meta_strategy.py old mode 100644 new mode 100755 diff --git a/src/jobs/repo_buddy.py b/src/jobs/repo_mapping.py old mode 100644 new mode 100755 similarity index 56% rename from src/jobs/repo_buddy.py rename to src/jobs/repo_mapping.py index 6749588..b07916d --- a/src/jobs/repo_buddy.py +++ b/src/jobs/repo_mapping.py @@ -3,34 +3,39 @@ import logging.config import logging import os import argparse +from abc import ABCMeta, abstractmethod import yaml -class RepoBuddy(object): - def __init__(self, ignored_repos_path, community_path, *repos): +class RepoMapping(object): + __metaclass__ = ABCMeta + + def __init__(self, ignored_repos_path, ignored_repos_key, community_path, *repos): """ :param ignored_repos_path: 忽略的repo配置文件 + :param key: :param repos: 用户输入的仓库 """ self._input_repos = repos - self._repo_buddy = {} # 保存结果 - self._ignored_repos = self._load_ignore_repo(ignored_repos_path) + self._repo_mapping = {} # 保存结果 + self._ignored_repos = self._load_ignore_repo(ignored_repos_path, ignored_repos_key) logger.debug("ignored repos: {}".format(self._ignored_repos)) self._community_repos = self._load_community_repo(community_path) # 社区repos logger.debug("community repos: {}".format(self._community_repos)) @staticmethod - def _load_ignore_repo(conf_file): + def _load_ignore_repo(conf_file, ignored_repos_key): """ 加载不用触发门禁的任务 :param conf_file: + :param ignored_repos_key: :return: """ try: with open(conf_file, "r") as f: handler = yaml.safe_load(f) - return handler.get("ignore", []) + return handler.get(ignored_repos_key, []) except IOError as e: logger.warning("{} not exist".format(conf_file)) return [] @@ -43,14 +48,13 @@ class RepoBuddy(object): :return: """ try: - conf_file = os.path.join(community_path, "repository/src-openeuler.yaml") - with open(conf_file, "r") as f: + with open(community_path, "r") as f: handler = yaml.safe_load(f) repos = {item["name"]: item["type"] for item in handler["repositories"]} logger.info("repos from community: {}".format(len(repos))) return repos except IOError as e: - logger.warning("{} not exist".format(conf_file)) + logger.warning("{} not exist".format(community_path)) return [] def _is_valid_repo(self, repo): @@ -64,33 +68,73 @@ class RepoBuddy(object): return False - def buddy(self, strategy): + @abstractmethod + def mapping(self, strategy): """ 计算仓库关联的package及buddy :param strategy: 策略对象 :return: """ + raise NotImplementedError + + def save(self, output): + """ + 保存结果 + :param output: + :return: + """ + with open(output, "w") as f: + yaml.safe_dump(self._repo_mapping, f) + + +class OERepoMapping(RepoMapping): + """ + openeuler + """ + def __init__(self, ignored_repos_path, community_path, *repos): + super(OERepoMapping, self).__init__(ignored_repos_path, "openeuler", + os.path.join(community_path, "repository/openeuler.yaml"), *repos) + + def mapping(self, strategy): + """ + + :param strategy: + :return: + """ repos = self._community_repos if "all" in self._input_repos else self._input_repos valid_repos = [repo for repo in repos if self._is_valid_repo(repo)] if valid_repos: - strategy.algorithm(*valid_repos) + self._repo_mapping = {repo: {"repo": repo} for repo in valid_repos} # same structure with src-openeuler - self._repo_buddy = {repo: {"repo": repo, "packages": strategy.get_packages_of_repo(repo), - "buddy": strategy.get_buddy_of_repo(repo)} for repo in strategy} - def save(self, output): +class SOERepoMapping(RepoMapping): + """ + src-openeuler + """ + def __init__(self, ignored_repos_path, community_path, *repos): + super(SOERepoMapping, self).__init__(ignored_repos_path, "src-openeuler", + os.path.join(community_path, "repository/src-openeuler.yaml"), *repos) + + def mapping(self, strategy): """ - 保存结果 - :param output: + 计算仓库关联的package及buddy + :param strategy: 策略对象 :return: """ - with open(output, "w") as f: - yaml.safe_dump(self._repo_buddy, f) + repos = self._community_repos if "all" in self._input_repos else self._input_repos + valid_repos = [repo for repo in repos if self._is_valid_repo(repo)] + + if valid_repos: + strategy.algorithm(*valid_repos) + + self._repo_mapping = {repo: {"repo": repo, "packages": strategy.get_packages_of_repo(repo), + "buddy": strategy.get_buddy_of_repo(repo)} for repo in strategy} if "__main__" == __name__: args = argparse.ArgumentParser() + args.add_argument("-f", type=str, dest="community", default="src-openeuler", help="src-openeuler or openeuler") args.add_argument("-j", type=str, dest="jobs", help="jobs name, split by dot") args.add_argument("-o", type=str, dest="mapping_file", help="output file to save buddy info") args.add_argument("-m", type=str, dest="obs_meta_path", help="obs meta path") @@ -103,13 +147,15 @@ if "__main__" == __name__: logger = logging.getLogger("jobs") # import after log initial - from src.proxy.requests_proxy import do_requests - from src.proxy.gitee_proxy import GiteeProxy from src.jobs.obs_meta_strategy import ObsMetaStrategy ignore_repo_path = os.path.realpath(os.path.join(os.path.realpath(__file__), "../../conf/ignore_repo.yaml")) input_repos = [repo.strip() for repo in args.jobs.split(",")] - buddy = RepoBuddy(ignore_repo_path, args.community_path, *input_repos) - buddy.buddy(ObsMetaStrategy(args.obs_meta_path)) - buddy.save(args.mapping_file) + if args.community == "src-openeuler": + rm = SOERepoMapping(ignore_repo_path, args.community_path, *input_repos) + rm.mapping(ObsMetaStrategy(args.obs_meta_path)) + else: + rm = OERepoMapping(ignore_repo_path, args.community_path, *input_repos) + rm.mapping(None) + rm.save(args.mapping_file) -- Gitee