From 925b45d7ec370ee320f390cedbfbe95b3dfd8880 Mon Sep 17 00:00:00 2001 From: tracedeng <56997132@qq.com> Date: Mon, 19 Apr 2021 11:03:26 +0800 Subject: [PATCH 1/9] use python3 stop report to es make address clickable, because jenkins console add special char append each line use local openjdk add libabigail and perl-xml-structured packages to ci common image delete useless code docstring --- src/ac/acl/package_yaml/check_repo.py | 4 +- src/ac/acl/sca/check_sca.py | 2 +- src/ac/acl/spec/check_spec.py | 2 +- src/ac/common/gitee_repo.py | 2 +- src/ac/framework/ac.py | 7 +-- src/ac/framework/ac.yaml | 2 +- src/build/gitee_comment.py | 2 - src/build/osc_build_k8s.py | 18 ++++--- src/build/related_rpm_package.py | 9 +++- src/conf/deny_list.sbom | 12 +++++ src/dockerfile/README.md | 31 ++++++++++++ src/dockerfile/{obs-base => ci-base-ubuntu} | 0 .../{jenkins-openeuler => ci-common} | 11 +++-- src/dockerfile/ci-soe-base | 47 +++++++++++++++++++ src/dockerfile/{obs => ci-ubuntu} | 0 src/dockerfile/openjdk-openeuler | 8 ++-- src/jobs/jenkins_job.py | 4 +- src/proxy/git_proxy.py | 19 +++++++- src/proxy/jenkins_patch.py | 4 +- src/proxy/obs_proxy.py | 2 +- src/requirements | 5 +- src/tools/obs_package_build_report.py | 2 + src/utils/shell_cmd.py | 1 + 23 files changed, 159 insertions(+), 35 deletions(-) create mode 100644 src/dockerfile/README.md rename src/dockerfile/{obs-base => ci-base-ubuntu} (100%) rename src/dockerfile/{jenkins-openeuler => ci-common} (74%) create mode 100644 src/dockerfile/ci-soe-base rename src/dockerfile/{obs => ci-ubuntu} (100%) diff --git a/src/ac/acl/package_yaml/check_repo.py b/src/ac/acl/package_yaml/check_repo.py index 9541ea3..d7be84e 100644 --- a/src/ac/acl/package_yaml/check_repo.py +++ b/src/ac/acl/package_yaml/check_repo.py @@ -22,7 +22,7 @@ import logging import re -import urlparse +import urllib.parse as urlparse import requests import json import subprocess @@ -537,4 +537,4 @@ class ReleaseTagsFactory(object): return: class """ release_tags = ReleaseTagsFactory.VERSION_CTRL_GETTER_MAPPING.get(version_control, DefaultReleaseTags) - return release_tags(version_control) \ No newline at end of file + return release_tags(version_control) diff --git a/src/ac/acl/sca/check_sca.py b/src/ac/acl/sca/check_sca.py index d4d9af9..da04991 100644 --- a/src/ac/acl/sca/check_sca.py +++ b/src/ac/acl/sca/check_sca.py @@ -82,7 +82,7 @@ class CheckSCA(BaseCheck): # 保存详细结果到web server if not result: self.save_scanoss_result(scan.html) - logger.warning("click to view scanoss detail: {}".format(self._scanoss_result_repo_path)) + logger.warning("click {} view scanoss detail".format(self._scanoss_result_repo_path)) return SUCCESS if result else FAILED diff --git a/src/ac/acl/spec/check_spec.py b/src/ac/acl/spec/check_spec.py index 6502e91..289969b 100644 --- a/src/ac/acl/spec/check_spec.py +++ b/src/ac/acl/spec/check_spec.py @@ -123,7 +123,7 @@ class CheckSpec(BaseCheck): if not homepage: return SUCCESS - for _ in xrange(retrying): + for _ in range(retrying): if 0 == do_requests("get", homepage, timeout=timeout): return SUCCESS time.sleep(interval) diff --git a/src/ac/common/gitee_repo.py b/src/ac/common/gitee_repo.py index 3432d4f..0f4527f 100644 --- a/src/ac/common/gitee_repo.py +++ b/src/ac/common/gitee_repo.py @@ -139,7 +139,7 @@ class GiteeRepo(object): if os.path.isdir(os.path.join(self._decompress_dir, filename))] + ["."]: if patch_dir.startswith(".git"): continue - for leading in xrange(max_leading + 1): + for leading in range(max_leading + 1): logger.debug("try dir {} -p{}".format(patch_dir, leading)) if GitProxy.apply_patch_at_dir(os.path.join(self._decompress_dir, patch_dir), os.path.join(self._work_dir, patch), leading): diff --git a/src/ac/framework/ac.py b/src/ac/framework/ac.py index 9beda0b..9d22f22 100644 --- a/src/ac/framework/ac.py +++ b/src/ac/framework/ac.py @@ -136,7 +136,7 @@ class AC(object): :return: """ for filename in os.listdir(acl_dir): - if os.path.isdir(os.path.join(acl_dir, filename)): + if filename != "__pycache__" and os.path.isdir(os.path.join(acl_dir, filename)): self._ac_check_elements[filename] = {} # don't worry, using default when checking def load_check_elements_from_conf(self, conf_file, community): @@ -223,7 +223,7 @@ if "__main__" == __name__: logger = logging.getLogger("ac") logger.info("using credential {}".format(args.account.split(":")[0])) - logger.info("cloning repository https://gitee.com/{}/{}.git".format(args.community, args.repo)) + logger.info("cloning repository https://gitee.com/{}/{}.git ".format(args.community, args.repo)) logger.info("clone depth 4") logger.info("checking out pull request {}".format(args.pr)) @@ -254,7 +254,6 @@ if "__main__" == __name__: logging.getLogger("elasticsearch").setLevel(logging.WARNING) logging.getLogger("kafka").setLevel(logging.WARNING) - ep = ESProxy(os.environ["ESUSERNAME"], os.environ["ESPASSWD"], os.environ["ESURL"], verify_certs=False) kp = KafkaProducerProxy(brokers=os.environ["KAFKAURL"].split(",")) # download repo @@ -267,7 +266,6 @@ if "__main__" == __name__: dd.set_attr_etime("access_control.job.etime") #dd.set_attr("access_control.job.result", "successful") - ep.insert(index="openeuler_statewall_ac", body=dd.to_dict()) kp.send("openeuler_statewall_ci_ac", value=dd.to_dict()) logger.info("fetch finished -") sys.exit(-1) @@ -300,5 +298,4 @@ if "__main__" == __name__: dd.set_attr_etime("access_control.job.etime") #dd.set_attr("access_control.job.result", "successful") - ep.insert(index="openeuler_statewall_ac", body=dd.to_dict()) kp.send("openeuler_statewall_ci_ac", value=dd.to_dict()) diff --git a/src/ac/framework/ac.yaml b/src/ac/framework/ac.yaml index bedfe35..3a12e87 100644 --- a/src/ac/framework/ac.yaml +++ b/src/ac/framework/ac.yaml @@ -34,4 +34,4 @@ openeuler: hint: check_sca module: sca.check_sca entry: CheckSCA - allow_list: ["openeuler-jenkins", "pkgship"] + allow_list: ["openeuler-jenkins", "pkgship", "stratovirt", "secGear", "isula-transform"] diff --git a/src/build/gitee_comment.py b/src/build/gitee_comment.py index 56ef4c0..442ebbb 100755 --- a/src/build/gitee_comment.py +++ b/src/build/gitee_comment.py @@ -315,9 +315,7 @@ if "__main__" == __name__: logging.getLogger("kafka").setLevel(logging.WARNING) # upload to es - ep = ESProxy(os.environ["ESUSERNAME"], os.environ["ESPASSWD"], os.environ["ESURL"], verify_certs=False) kp = KafkaProducerProxy(brokers=os.environ["KAFKAURL"].split(",")) query = {"term": {"id": args.comment_id}} script = {"lang": "painless", "source": "ctx._source.comment = params.comment", "params": dd.to_dict()} - #ep.update_by_query(index="openeuler_statewall_ac", query=query, script=script) kp.send("openeuler_statewall_ci_ac", key=args.comment_id, value=dd.to_dict()) diff --git a/src/build/osc_build_k8s.py b/src/build/osc_build_k8s.py index 52af37c..96c58c6 100755 --- a/src/build/osc_build_k8s.py +++ b/src/build/osc_build_k8s.py @@ -35,8 +35,17 @@ class SinglePackageBuild(object): "openEuler-EPOL-LTS": ["bringInRely"], "openEuler-20.09": ["openEuler:20.09", "openEuler:20.09:Epol"], "mkopeneuler-20.03": ["openEuler:Extras"], - "openEuler-20.03-LTS-SP1": ["openEuler:20.03:LTS:SP1", "openEuler:20.03:LTS:SP1:Epol"], - "openEuler-21.03": ["openEuler:21.03", "openEuler:21.03:Epol", "openEuler:21.03:Extras"] + "openEuler-20.03-LTS-SP1": ["openEuler:20.03:LTS:SP1", "openEuler:20.03:LTS:SP1:Epol", + "openEuler:20.03:LTS:SP1:Extras"], + "openEuler-20.03-LTS-SP2": ["openEuler:20.03:LTS:SP2", "openEuler:20.03:LTS:SP2:Epol", + "openEuler:20.03:LTS:SP2:Extras"], + "openEuler-21.03": ["openEuler:21.03", "openEuler:21.03:Epol", "openEuler:21.03:Extras"], + "oepkg_openstack-common_oe-20.03-LTS-SP2": ["openEuler:20.03:LTS:SP2:oepkg:openstack:common"], + "oepkg_openstack-queens_oe-20.03-LTS-SP2": ["openEuler:20.03:LTS:SP2:oepkg:openstack:queens"], + "oepkg_openstack-rocky_oe-20.03-LTS-SP2": ["openEuler:20.03:LTS:SP2:oepkg:openstack:rocky"], + "oepkg_openstack-common_oe-20.03-LTS-Next": ["openEuler:20.03:LTS:Next:oepkg:openstack:common"], + "oepkg_openstack-queens_oe-20.03-LTS-Next": ["openEuler:20.03:LTS:Next:oepkg:openstack:queens"], + "oepkg_openstack-rocky_oe-20.03-LTS-Next": ["openEuler:20.03:LTS:Next:oepkg:openstack:rocky"] } BUILD_IGNORED_GITEE_BRANCH = ["riscv"] @@ -242,7 +251,7 @@ if "__main__" == __name__: logger = logging.getLogger("build") logger.info("using credential {}".format(args.account.split(":")[0])) - logger.info("cloning repository https://gitee.com/{}/{}.git".format(args.owner, args.repo)) + logger.info("cloning repository https://gitee.com/{}/{}.git ".format(args.owner, args.repo)) logger.info("clone depth 1") logger.info("checking out pull request {}".format(args.pr)) @@ -263,7 +272,6 @@ if "__main__" == __name__: logging.getLogger("elasticsearch").setLevel(logging.WARNING) logging.getLogger("kafka").setLevel(logging.WARNING) - ep = ESProxy(os.environ["ESUSERNAME"], os.environ["ESPASSWD"], os.environ["ESURL"], verify_certs=False) kp = KafkaProducerProxy(brokers=os.environ["KAFKAURL"].split(",")) # download repo @@ -282,7 +290,6 @@ if "__main__" == __name__: query = {"term": {"id": args.comment_id}} script = {"lang": "painless", "source": "ctx._source.spb_{}=params.spb".format(args.arch), "params": dd.to_dict()} - ep.update_by_query(index="openeuler_statewall_ac", query=query, script=script) kp.send("openeuler_statewall_ci_ac", key=args.comment_id, value=dd.to_dict()) sys.exit(-1) else: @@ -302,6 +309,5 @@ if "__main__" == __name__: # upload to es query = {"term": {"id": args.comment_id}} script = {"lang": "painless", "source": "ctx._source.spb_{}=params.spb".format(args.arch), "params": dd.to_dict()} - ep.update_by_query(index="openeuler_statewall_ac", query=query, script=script) kp.send("openeuler_statewall_ci_ac", key=args.comment_id, value=dd.to_dict()) sys.exit(rs) diff --git a/src/build/related_rpm_package.py b/src/build/related_rpm_package.py index 627000a..97b891c 100755 --- a/src/build/related_rpm_package.py +++ b/src/build/related_rpm_package.py @@ -37,7 +37,14 @@ class RelatedRpms(object): "openEuler-EPOL-LTS": ["bringInRely"], "openEuler-20.09": ["openEuler:20.09"], "mkopeneuler-20.03": ["openEuler:Extras"], - "openEuler-20.03-LTS-SP1": ["openEuler:20.03:LTS:SP1"] + "openEuler-20.03-LTS-SP1": ["openEuler:20.03:LTS:SP1", "openEuler:20.03:LTS:SP1:Epol"], + "openEuler-20.03-LTS-SP2": ["openEuler:20.03:LTS:SP2", "openEuler:20.03:LTS:SP2:Epol"], + "oepkg_openstack-common_oe-20.03-LTS-SP2": ["openEuler:20.03:LTS:SP2:oepkg:openstack:common"], + "oepkg_openstack-rocky_oe-20.03-LTS-SP2": ["openEuler:20.03:LTS:SP2:oepkg:openstack:rocky"], + "oepkg_openstack-queens_oe-20.03-LTS-SP2": ["openEuler:20.03:LTS:SP2:oepkg:openstack:queens"], + "oepkg_openstack-common_oe-20.03-LTS-Next": ["openEuler:20.03:LTS:Next:oepkg:openstack:common"], + "oepkg_openstack-rocky_oe-20.03-LTS-Next": ["openEuler:20.03:LTS:Next:oepkg:openstack:rocky"], + "oepkg_openstack-queens_oe-20.03-LTS-Next": ["openEuler:20.03:LTS:Next:oepkg:openstack:queens"] } def __init__(self, obs_addr, obs_repo_url, branch_name, package_arch): diff --git a/src/conf/deny_list.sbom b/src/conf/deny_list.sbom index 907a27b..87e0d67 100644 --- a/src/conf/deny_list.sbom +++ b/src/conf/deny_list.sbom @@ -8,6 +8,18 @@ { "publish": "openeuler", "name": "openeuler-jenkins" + }, + { + "publish": "openeuler", + "name": "stratovirt" + }, + { + "publish": "openeuler", + "name": "secGear" + }, + { + "publish": "openeuler", + "name": "isula-transform" } ] } diff --git a/src/dockerfile/README.md b/src/dockerfile/README.md new file mode 100644 index 0000000..b4af737 --- /dev/null +++ b/src/dockerfile/README.md @@ -0,0 +1,31 @@ +# docker images +## openeuler +- swr.cn-north-4.myhuaweicloud.com/openeuler/openeuler:21.03 +- swr.cn-north-4.myhuaweicloud.com/openeuler/openeuler:20.09 +- swr.cn-north-4.myhuaweicloud.com/openeuler/openeuler:20.03-lts-sp1 +- swr.cn-north-4.myhuaweicloud.com/openeuler/openeuler:20.03-lts + +## openjdk 11-jdk-stretch based on openeuler +- swr.cn-north-4.myhuaweicloud.com/openeuler/openjdk/11-jdk-stretch:21.03 +- swr.cn-north-4.myhuaweicloud.com/openeuler/openjdk/11-jdk-stretch:20.09 +- swr.cn-north-4.myhuaweicloud.com/openeuler/openjdk/11-jdk-stretch:20.03-lts-sp1 +- swr.cn-north-4.myhuaweicloud.com/openeuler/openjdk/11-jdk-stretch:20.03-lts + +> Dockerfile: openjdk-openeuler + +## openeuler ci image based on openjdk +- swr.cn-north-4.myhuaweicloud.com/openeuler/ci/common:21.03 +- swr.cn-north-4.myhuaweicloud.com/openeuler/ci/common:20.09 +- swr.cn-north-4.myhuaweicloud.com/openeuler/ci/common:20.03-lts-sp1 +- swr.cn-north-4.myhuaweicloud.com/openeuler/ci/common:20.03-lts + +> Dockerfile: ci-common + +## src-openeuler ci image based on openjdk +- swr.cn-north-4.myhuaweicloud.com/openeuler/ci/soe:base + +> Dockerfile: ci-soe-base + +> choose an openjdk image that from stable openeuler version + +- swr.cn-north-4.myhuaweicloud.com/openeuler/ci/soe:{version} diff --git a/src/dockerfile/obs-base b/src/dockerfile/ci-base-ubuntu similarity index 100% rename from src/dockerfile/obs-base rename to src/dockerfile/ci-base-ubuntu diff --git a/src/dockerfile/jenkins-openeuler b/src/dockerfile/ci-common similarity index 74% rename from src/dockerfile/jenkins-openeuler rename to src/dockerfile/ci-common index 98ffa41..b99d662 100644 --- a/src/dockerfile/jenkins-openeuler +++ b/src/dockerfile/ci-common @@ -1,5 +1,8 @@ # replace VERSION before run -FROM swr.cn-north-4.myhuaweicloud.com/openeuler/openeuler-VERSION/openjdk:11-jdk-stretch +FROM swr.cn-north-4.myhuaweicloud.com/openeuler/openjdk/OPENJDK:TAG + +RUN set -eux; \ + yum install -y python3 python3-pip cpio bsdtar expect openssh sudo shadow curl wget vim git strace ARG VERSION=4.3 ARG user=jenkins @@ -13,7 +16,9 @@ RUN curl --create-dirs -fsSLo /usr/share/jenkins/agent.jar https://repo.jenkins- && chmod 644 /usr/share/jenkins/agent.jar \ && ln -sf /usr/share/jenkins/agent.jar /usr/share/jenkins/slave.jar -COPY jenkins-agent /usr/local/bin/jenkins-agent +RUN curl --create-dirs -fsSLo /usr/local/bin/jenkins-agent http://121.36.53.23/AdoptOpenJDK/jenkins-agent +#COPY jenkins-agent /usr/local/bin/jenkins-agent + RUN chmod a+rx /usr/local/openjdk-11 \ && chmod a+rx /usr/local/bin/jenkins-agent \ && ln -s /usr/local/bin/jenkins-agent /usr/local/bin/jenkins-slave @@ -33,5 +38,3 @@ WORKDIR ${AGENT_WORKDIR} ENTRYPOINT ["jenkins-agent"] - -# swr.cn-north-4.myhuaweicloud.com/openeuler/jenkins/openeuler:base diff --git a/src/dockerfile/ci-soe-base b/src/dockerfile/ci-soe-base new file mode 100644 index 0000000..b6332ad --- /dev/null +++ b/src/dockerfile/ci-soe-base @@ -0,0 +1,47 @@ +# replace VERSION before run +FROM swr.cn-north-4.myhuaweicloud.com/openeuler/openjdk/OPENJDK:TAG + +# golint not ready yet +RUN set -eux; \ + yum install -y curl sudo cpio bsdtar expect openssh shadow vim git strace python3 python3-pip build osc pylint + +# libabigail perl-XML-Structured not ready yet, use SP2 dailybuild +RUN set -eux; \ + arch="$(arch)"; \ + yum install -y http://121.36.53.23/SP2/libabigail-1.6-4.oe1.$arch.rpm; \ + yum install -y http://121.36.53.23/SP2/perl-XML-Structured-1.3-2.oe1.$arch.rpm + +ARG VERSION=4.3 +ARG user=jenkins +ARG group=jenkins +ARG uid=1000 +ARG gid=1000 +ARG AGENT_WORKDIR=/home/${user}/agent + +RUN curl --create-dirs -fsSLo /usr/share/jenkins/agent.jar https://repo.jenkins-ci.org/public/org/jenkins-ci/main/remoting/${VERSION}/remoting-${VERSION}.jar \ + && chmod 755 /usr/share/jenkins \ + && chmod 644 /usr/share/jenkins/agent.jar \ + && ln -sf /usr/share/jenkins/agent.jar /usr/share/jenkins/slave.jar + +RUN curl --create-dirs -fsSLo /usr/local/bin/jenkins-agent http://121.36.53.23/AdoptOpenJDK/jenkins-agent +#COPY jenkins-agent /usr/local/bin/jenkins-agent + +RUN chmod a+rx /usr/local/openjdk-11 \ + && chmod a+rx /usr/local/bin/jenkins-agent \ + && ln -s /usr/local/bin/jenkins-agent /usr/local/bin/jenkins-slave + +RUN groupadd -g ${gid} ${group} +RUN useradd -c "Jenkins user" -d /home/${user} -u ${uid} -g ${gid} -m ${user} +RUN echo "${user} ALL=(ALL) NOPASSWD:ALL" >> /etc/sudoers + + +USER ${user} +ENV AGENT_WORKDIR=${AGENT_WORKDIR} +RUN mkdir /home/${user}/.jenkins && mkdir -p ${AGENT_WORKDIR} + +VOLUME /home/${user}/.jenkins +VOLUME ${AGENT_WORKDIR} +WORKDIR ${AGENT_WORKDIR} + + +ENTRYPOINT ["jenkins-agent"] diff --git a/src/dockerfile/obs b/src/dockerfile/ci-ubuntu similarity index 100% rename from src/dockerfile/obs rename to src/dockerfile/ci-ubuntu diff --git a/src/dockerfile/openjdk-openeuler b/src/dockerfile/openjdk-openeuler index aaf5914..7ca38a9 100644 --- a/src/dockerfile/openjdk-openeuler +++ b/src/dockerfile/openjdk-openeuler @@ -2,7 +2,7 @@ FROM swr.cn-north-4.myhuaweicloud.com/openeuler/openeuler:VERSION RUN set -eux; \ - yum install -y tar wget python3 expect osc vim openssh shadow git + yum install -y tar wget # Default to UTF-8 file.encoding ENV LANG C.UTF-8 @@ -33,9 +33,11 @@ RUN set -eux; \ # this "case" statement is generated via "update.sh" case "$arch" in \ # arm64v8 - arm64 | aarch64) downloadUrl=https://github.com/AdoptOpenJDK/openjdk11-upstream-binaries/releases/download/jdk-11.0.8%2B10/OpenJDK11U-jdk_aarch64_linux_11.0.8_10.tar.gz ;; \ + arm64 | aarch64) downloadUrl=http://121.36.53.23/AdoptOpenJDK/openjdk_aarch64.tgz ;; \ # amd64 - amd64 | i386:x86-64 | x86_64) downloadUrl=https://github.com/AdoptOpenJDK/openjdk11-upstream-binaries/releases/download/jdk-11.0.8%2B10/OpenJDK11U-jdk_x64_linux_11.0.8_10.tar.gz ;; \ + amd64 | i386:x86-64 | x86_64) downloadUrl=http://121.36.53.23/AdoptOpenJDK/openjdk_x64.tgz ;; \ + #arm64 | aarch64) downloadUrl=https://github.com/AdoptOpenJDK/openjdk11-upstream-binaries/releases/download/jdk-11.0.8%2B10/OpenJDK11U-jdk_aarch64_linux_11.0.8_10.tar.gz ;; \ + #amd64 | i386:x86-64 | x86_64) downloadUrl=https://github.com/AdoptOpenJDK/openjdk11-upstream-binaries/releases/download/jdk-11.0.8%2B10/OpenJDK11U-jdk_x64_linux_11.0.8_10.tar.gz ;; \ # fallback *) echo >&2 "error: unsupported architecture: '$arch'"; exit 1 ;; \ esac; \ diff --git a/src/jobs/jenkins_job.py b/src/jobs/jenkins_job.py index 7ecab8a..3978493 100755 --- a/src/jobs/jenkins_job.py +++ b/src/jobs/jenkins_job.py @@ -68,7 +68,7 @@ class JenkinsJobs(object): """ batch = (len(target_jobs) + concurrency - 1) / concurrency _failed_jobs = [] - for index in xrange(batch): + for index in range(batch): works = [gevent.spawn(self.dispatch, action, job, jenkins_proxy) for job in target_jobs[index * concurrency: (index + 1) * concurrency]] logger.info("{} works, {}/{} ".format(len(works), index + 1, batch)) @@ -86,7 +86,7 @@ class JenkinsJobs(object): failed_jobs = run_once(real_jobs) - for index in xrange(retry): + for index in range(retry): if not failed_jobs: break logger.info("{} jobs failed, retrying {}/{}".format(len(failed_jobs), index + 1, retry)) diff --git a/src/proxy/git_proxy.py b/src/proxy/git_proxy.py index e2c644b..87fe232 100644 --- a/src/proxy/git_proxy.py +++ b/src/proxy/git_proxy.py @@ -1,7 +1,24 @@ # -*- encoding=utf-8 -*- +""" +# ********************************************************************************** +# Copyright (c) Huawei Technologies Co., Ltd. 2020-2020. All rights reserved. +# [openeuler-jenkins] is licensed under the Mulan PSL v1. +# You can use this software according to the terms and conditions of the Mulan PSL v1. +# You may obtain a copy of Mulan PSL v1 at: +# http://license.coscl.org.cn/MulanPSL +# THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR +# PURPOSE. +# See the Mulan PSL v1 for more details. +# Author: +# Create: 2020-09-23 +# Description: git api proxy +# ********************************************************************************** +""" + import os import logging -from cStringIO import StringIO +from io import StringIO import retrying from src.utils.shell_cmd import shell_cmd_live diff --git a/src/proxy/jenkins_patch.py b/src/proxy/jenkins_patch.py index 4fa3ef2..1b73d44 100644 --- a/src/proxy/jenkins_patch.py +++ b/src/proxy/jenkins_patch.py @@ -1,5 +1,5 @@ # -*- encoding=utf-8 -*- -from urllib import quote as urlquote +from urllib.parse import quote from jenkinsapi.jenkinsbase import JenkinsBase @@ -11,7 +11,7 @@ def resolve_job_folders(self, jobs): jobs.remove(job) jobs += self.process_job_folder(job, self.baseurl) else: - job["url"] = '%s/job/%s' % (self.baseurl, urlquote(job['name'])) + job["url"] = '%s/job/%s' % (self.baseurl, quote(job['name'])) return jobs diff --git a/src/proxy/obs_proxy.py b/src/proxy/obs_proxy.py index eee8ad1..6e31653 100644 --- a/src/proxy/obs_proxy.py +++ b/src/proxy/obs_proxy.py @@ -120,7 +120,7 @@ class OBSProxy(object): :return: """ package_path = "{}/{}".format(project, package) - cmd = "cd {}; osc build {} {} {} --no-verify --clean".format( + cmd = "cd {}; osc build {} {} {} --no-verify --clean --noservice".format( package_path, repo, arch, "--disable-debuginfo" if not debug else "") logger.info("osc build {} {} {} --no-verify --clean".format( diff --git a/src/requirements b/src/requirements index bb5b47e..d3fd573 100644 --- a/src/requirements +++ b/src/requirements @@ -3,11 +3,12 @@ jenkinsapi colorlog threadpool PyYAML -gevent==1.2.2 +gevent jsonpath mock tldextract chardet -kafka +kafka-python elasticsearch retrying +scanoss-scanner diff --git a/src/tools/obs_package_build_report.py b/src/tools/obs_package_build_report.py index ae746cc..7841e10 100644 --- a/src/tools/obs_package_build_report.py +++ b/src/tools/obs_package_build_report.py @@ -40,6 +40,8 @@ class ObsPackageBuildReport(object): "openEuler:20.03:LTS:Next:Epol": "openEuler-20.03-LTS-Next", "openEuler:20.03:LTS:SP1": "openEuler-20.03-LTS-SP1", "openEuler:20.03:LTS:SP1:Epol": "openEuler-20.03-LTS-SP1", + "openEuler:20.03:LTS:SP2": "openEuler-20.03-LTS-SP2", + "openEuler:20.03:LTS:SP2:Epol": "openEuler-20.03-LTS-SP2", "openEuler:21.03": "openEuler-21.03", "openEuler:21.03:Epol": "openEuler-21.03", "openEuler:21.03:Extras": "openEuler-21.03" diff --git a/src/utils/shell_cmd.py b/src/utils/shell_cmd.py index d3bc10c..dec8843 100755 --- a/src/utils/shell_cmd.py +++ b/src/utils/shell_cmd.py @@ -42,6 +42,7 @@ def shell_cmd_live(cmd, cap_in=None, cap_out=False, cap_err=False, verbose=False while True: line = p.stdout.readline() if line: + line = line.decode("utf-8") line = line.strip() no_fmt_logger.info(line) if verbose else no_fmt_logger.debug(line) if cap_out and line and line != "\n": -- Gitee From d31cee0f80985ff96309c9e7766791cba704b19c Mon Sep 17 00:00:00 2001 From: tracedeng <56997132@qq.com> Date: Tue, 25 May 2021 14:36:42 +0800 Subject: [PATCH 2/9] kafka producer need serialize string key for python3 --- src/proxy/kafka_proxy.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/proxy/kafka_proxy.py b/src/proxy/kafka_proxy.py index 9ff7ff7..c7dcde4 100644 --- a/src/proxy/kafka_proxy.py +++ b/src/proxy/kafka_proxy.py @@ -37,6 +37,7 @@ class KafkaProducerProxy(object): self._timeout = timeout self._kp = kafka.KafkaProducer(bootstrap_servers=brokers, + key_serializer=str.encode, value_serializer=lambda v:json.dumps(v).encode("utf-8")) def send(self, topic, key=None, value=None): -- Gitee From cb2cc8a0a262bf23027659dbc7043875e86e25a8 Mon Sep 17 00:00:00 2001 From: jiangfeilong Date: Tue, 25 May 2021 20:31:10 +0800 Subject: [PATCH 3/9] add dockerfile for bishengjdk riscv port Signed-off-by: jiangfeilong --- src/dockerfile/ci-bisheng-riscv | 106 ++++++++++++++++++++++++++++++++ 1 file changed, 106 insertions(+) create mode 100644 src/dockerfile/ci-bisheng-riscv diff --git a/src/dockerfile/ci-bisheng-riscv b/src/dockerfile/ci-bisheng-riscv new file mode 100644 index 0000000..e5e3617 --- /dev/null +++ b/src/dockerfile/ci-bisheng-riscv @@ -0,0 +1,106 @@ +# replace VERSION before run +FROM swr.cn-north-4.myhuaweicloud.com/openeuler/openjdk/OPENJDK:TAG + +# Install qemu 5.2.0 +RUM set -eux; \ + yum update; yum clean all; \ + yum -y groupinstall "Development Tools"; \ + yum -y insall pixman-devel zlib-* glib2 glib2-devel perl tar xz zip unzip wget; \ + cd /usr/src; \ + wget -q http://121.36.53.23/bisheng_riscv/ninja-linux.zip; \ + unzip ninja-linux.zip; \ + cp ninja /usr/bin; \ + wget -q http:121.36.53.23/bisheng_riscv/qemu-5.2.0.tar.xz; \ + tar xJf qemu-5.2.0.tar.xz; \ + cd qemu-5.2.0 && mkdir build && cd build; \ + ../configure --target-list=riscv64-linux-user,riscv64-softmmu; \ + make install -j; \ + rm -rf /usr/src/qemu-5.2.0* /usr/src/ninja*; + +# RISCV toolchain +RUN cd /opt; \ + wget -q http://121.36.53.23/bisheng_riscv/riscv_toolchain_linux64.tar.xz; \ + tar xJf riscv_toolchain_linux64.tar.xz; \ + rm riscv_toolchain_linux64.tar.xz; + +# fedora sysroot +RUN cd /opt; \ + wget -q http://121.36.53.23/bisheng_riscv/fedora28_riscv_smlroot.tar.xz; \ + tar xJf fedora28_riscv_smlroot.tar.xz; \ + ln -s /opt/fedora28_riscv_root/usr/lib64/ld-linux-riscv64-lp64d.so.1 /lib/ld-linux-riscv64-lp64d.so.1; \ + rm fedora28_riscv_smlroot.tar.xz; + +# Packages for openjdk build +RUN set -eux; \ + yum -y install java-1.8.0-openjdk-devel cups-devel libXtst-devel libXt-devel libXrender-devel libXi-devel alsa-lib-devel libffi-devel elfutils-libelf-devel libstdc++-static libatomic + +# Test tool packages +RUN set -eux; \ + yum -y install gcc gcc-c++ unzip zlib-devel perl-devel libcurl-devel openssl-devel mercurial perl perl-Test-Simple xorg-x11-server-Xorg xorg-x11-server-Xvfb + +# JTREG +RUN cd /opt; \ + wget -q http://121.36.53.23/bisheng_riscv/jtreg-4.2.0-tip.tar.gz; \ + mkdir jtreg-4; tar -xf jtreg-4.2.0-tip.tar.gz -C jtreg-4 --strip-components 1; \ + wget -q http://121.36.53.23/bisheng_riscv/jtreg-5.1-b01.tar.gz; \ + mkdir jtreg-5; tar -xf jtreg-5.1-b01.tar.gz -c jtreg-5 --strip-components 1; \ + rm jtreg*.tar.gz; + +# AdoptOpenJDK 8/10/11/16 +RUN mkdir -p /usr/lib/jvm && cd /usr/lib/jvm; \ + wget -q http://121.36.53.23/bisheng_riscv/OpenJDK8U-jdk_x64_linux_hotspot_8u292b10.tar.gz -O jdk-8.tar.gz; \ + wget -q http://121.36.53.23/bisheng_riscv/OpenJDK10U-jdk_x64_linux_hotspot_10.0.2_13.tar.gz -O jdk-10.tar.gz; \ + wget -q http://121.36.53.23/bisheng_riscv/OpenJDK11U-jdk_x64_linux_hotspot_11.0.11_9.tar.gz -O jdk-11.tar.gz; \ + wget -q http://121.36.53.23/bisheng_riscv/OpenJDK16U-jdk_x64_linux_hotspot_16.0.1_9.tar.gz -O jdk-16.tar.gz; \ + for tar in *.tar.gz; do tar xf $tar; done; \ + rm *.tar.gz; \ + ln -s jdk8* jdk-8; \ + ln -s jdk-10* jdk-10; \ + ln -s jdk-11* jdk-11; \ + ln -s jdk-16* jdk-16; + +ENV \ + JDK8_BOOT_DIR="/usr/lib/jvm/jdk-8" \ + JDK10_BOOT_DIR="/usr/lib/jvm/jdk-10" \ + JDK11_BOOT_DIR="/usr/lib/jvm/jdk-11" \ + JDK16_BOOT_DIR="/usr/lib/jvm/jdk-16" \ + JTREG4_DIR="/opt/jtreg-4" \ + JTREG5_DIR="/opt/jtreg-5" + +# Jenkins Agent +RUN set -eux; \ + yum install -y python3 python3-pip cpio bsdtar expect openssh sudo shadow curl wget vim git strace + +ARG VERSION=4.3 +ARG user=jenkins +ARG group=jenkins +ARG uid=1000 +ARG gid=1000 +ARG AGENT_WORKDIR=/home/${user}/agent + +RUN curl --create-dirs -fsSLo /usr/share/jenkins/agent.jar https://repo.jenkins-ci.org/public/org/jenkins-ci/main/remoting/${VERSION}/remoting-${VERSION}.jar \ + && chmod 755 /usr/share/jenkins \ + && chmod 644 /usr/share/jenkins/agent.jar \ + && ln -sf /usr/share/jenkins/agent.jar /usr/share/jenkins/slave.jar + +RUN curl --create-dirs -fsSLo /usr/local/bin/jenkins-agent http://121.36.53.23/AdoptOpenJDK/jenkins-agent + +RUN chmod a+rx /usr/local/openjdk-11 \ + && chmod a+rx /usr/local/bin/jenkins-agent \ + && ln -s /usr/local/bin/jenkins-agent /usr/local/bin/jenkins-slave + +RUN groupadd -g ${gid} ${group} +RUN useradd -c "Jenkins user" -d /home/${user} -u ${uid} -g ${gid} -m ${user} +RUN echo "${user} ALL=(ALL) NOPASSWD:ALL" >> /etc/sudoers + + +USER ${user} +ENV AGENT_WORKDIR=${AGENT_WORKDIR} +RUN mkdir /home/${user}/.jenkins && mkdir -p ${AGENT_WORKDIR} + +VOLUME /home/${user}/.jenkins +VOLUME ${AGENT_WORKDIR} +WORKDIR ${AGENT_WORKDIR} + + +ENTRYPOINT ["jenkins-agent"] -- Gitee From 0df69445208ff9a96090de3c2991174288b8b06d Mon Sep 17 00:00:00 2001 From: jiangfeilong Date: Wed, 26 May 2021 09:29:00 +0800 Subject: [PATCH 4/9] Fix typo --- src/dockerfile/ci-bisheng-riscv | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/dockerfile/ci-bisheng-riscv b/src/dockerfile/ci-bisheng-riscv index e5e3617..6f9ee19 100644 --- a/src/dockerfile/ci-bisheng-riscv +++ b/src/dockerfile/ci-bisheng-riscv @@ -2,7 +2,7 @@ FROM swr.cn-north-4.myhuaweicloud.com/openeuler/openjdk/OPENJDK:TAG # Install qemu 5.2.0 -RUM set -eux; \ +RUN set -eux; \ yum update; yum clean all; \ yum -y groupinstall "Development Tools"; \ yum -y insall pixman-devel zlib-* glib2 glib2-devel perl tar xz zip unzip wget; \ -- Gitee From efc43eac4fcb5e598276e13b8b6757f2fd72c3df Mon Sep 17 00:00:00 2001 From: jiangfeilong Date: Wed, 26 May 2021 11:43:46 +0800 Subject: [PATCH 5/9] fix typo insall->install --- src/dockerfile/ci-bisheng-riscv | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/dockerfile/ci-bisheng-riscv b/src/dockerfile/ci-bisheng-riscv index 6f9ee19..a365478 100644 --- a/src/dockerfile/ci-bisheng-riscv +++ b/src/dockerfile/ci-bisheng-riscv @@ -5,7 +5,7 @@ FROM swr.cn-north-4.myhuaweicloud.com/openeuler/openjdk/OPENJDK:TAG RUN set -eux; \ yum update; yum clean all; \ yum -y groupinstall "Development Tools"; \ - yum -y insall pixman-devel zlib-* glib2 glib2-devel perl tar xz zip unzip wget; \ + yum -y install pixman-devel zlib-* glib2 glib2-devel perl tar xz zip unzip wget; \ cd /usr/src; \ wget -q http://121.36.53.23/bisheng_riscv/ninja-linux.zip; \ unzip ninja-linux.zip; \ -- Gitee From 48c2dd03816883238b81efa249069485bbce9c02 Mon Sep 17 00:00:00 2001 From: jiangfeilong Date: Wed, 26 May 2021 12:27:37 +0800 Subject: [PATCH 6/9] fix qemu wget url --- src/dockerfile/ci-bisheng-riscv | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/dockerfile/ci-bisheng-riscv b/src/dockerfile/ci-bisheng-riscv index a365478..e5c6260 100644 --- a/src/dockerfile/ci-bisheng-riscv +++ b/src/dockerfile/ci-bisheng-riscv @@ -10,7 +10,7 @@ RUN set -eux; \ wget -q http://121.36.53.23/bisheng_riscv/ninja-linux.zip; \ unzip ninja-linux.zip; \ cp ninja /usr/bin; \ - wget -q http:121.36.53.23/bisheng_riscv/qemu-5.2.0.tar.xz; \ + wget -q http://121.36.53.23/bisheng_riscv/qemu-5.2.0.tar.xz; \ tar xJf qemu-5.2.0.tar.xz; \ cd qemu-5.2.0 && mkdir build && cd build; \ ../configure --target-list=riscv64-linux-user,riscv64-softmmu; \ -- Gitee From 97e42f24a32c0452d3d94f86a70e9801734a7814 Mon Sep 17 00:00:00 2001 From: jiangfeilong Date: Wed, 26 May 2021 14:05:31 +0800 Subject: [PATCH 7/9] fix jtreg5 tar -C option --- src/dockerfile/ci-bisheng-riscv | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/dockerfile/ci-bisheng-riscv b/src/dockerfile/ci-bisheng-riscv index e5c6260..b4169bb 100644 --- a/src/dockerfile/ci-bisheng-riscv +++ b/src/dockerfile/ci-bisheng-riscv @@ -43,7 +43,7 @@ RUN cd /opt; \ wget -q http://121.36.53.23/bisheng_riscv/jtreg-4.2.0-tip.tar.gz; \ mkdir jtreg-4; tar -xf jtreg-4.2.0-tip.tar.gz -C jtreg-4 --strip-components 1; \ wget -q http://121.36.53.23/bisheng_riscv/jtreg-5.1-b01.tar.gz; \ - mkdir jtreg-5; tar -xf jtreg-5.1-b01.tar.gz -c jtreg-5 --strip-components 1; \ + mkdir jtreg-5; tar -xf jtreg-5.1-b01.tar.gz -C jtreg-5 --strip-components 1; \ rm jtreg*.tar.gz; # AdoptOpenJDK 8/10/11/16 -- Gitee From d3eedbad77827bee64641cdf0655452565e8fe01 Mon Sep 17 00:00:00 2001 From: tracedeng <56997132@qq.com> Date: Wed, 26 May 2021 14:10:10 +0800 Subject: [PATCH 8/9] fix bug, send data to kafka miss param key --- src/ac/framework/ac.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/src/ac/framework/ac.py b/src/ac/framework/ac.py index 9d22f22..da90427 100644 --- a/src/ac/framework/ac.py +++ b/src/ac/framework/ac.py @@ -265,8 +265,7 @@ if "__main__" == __name__: dd.set_attr_etime("access_control.scm.etime") dd.set_attr_etime("access_control.job.etime") - #dd.set_attr("access_control.job.result", "successful") - kp.send("openeuler_statewall_ci_ac", value=dd.to_dict()) + kp.send("openeuler_statewall_ci_ac", key=args.comment_id, value=dd.to_dict()) logger.info("fetch finished -") sys.exit(-1) else: @@ -297,5 +296,4 @@ if "__main__" == __name__: ac.save(args.output) dd.set_attr_etime("access_control.job.etime") - #dd.set_attr("access_control.job.result", "successful") - kp.send("openeuler_statewall_ci_ac", value=dd.to_dict()) + kp.send("openeuler_statewall_ci_ac", key=args.comment_id, value=dd.to_dict()) -- Gitee From 569da087a812fee98007e6d97785536367c5004b Mon Sep 17 00:00:00 2001 From: tracedeng <56997132@qq.com> Date: Fri, 28 May 2021 11:56:12 +0800 Subject: [PATCH 9/9] package at exclude state when init --- src/build/osc_build_k8s.py | 2 +- src/proxy/obs_proxy.py | 5 +++-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/src/build/osc_build_k8s.py b/src/build/osc_build_k8s.py index 96c58c6..eede4c6 100755 --- a/src/build/osc_build_k8s.py +++ b/src/build/osc_build_k8s.py @@ -66,7 +66,7 @@ class SinglePackageBuild(object): 需要构建obs repo列表 :return: list """ - return OBSProxy.list_repos_of_arch(project, self._package, self._arch) + return OBSProxy.list_repos_of_arch(project, self._package, self._arch, show_exclude=True) def build_obs_repos(self, project, repos, work_dir, code_dir): """ diff --git a/src/proxy/obs_proxy.py b/src/proxy/obs_proxy.py index 6e31653..935d944 100644 --- a/src/proxy/obs_proxy.py +++ b/src/proxy/obs_proxy.py @@ -51,14 +51,15 @@ class OBSProxy(object): return rs @staticmethod - def list_repos_of_arch(project, package, arch): + def list_repos_of_arch(project, package, arch, show_exclude=False): """ 获取包的repo列表 :param project: :param package: :return: """ - cmd = "osc results {} {} -a {}".format(project, package, arch) + cmd = "osc results {} {} {} -a {}".format( + "--show-exclude" if show_exclude else "", project, package, arch) ret, out, _ = shell_cmd_live(cmd, cap_out=True) if ret: logger.debug("list obs repos of arch error, {}".format(ret)) -- Gitee