diff --git a/src/build/comment_to_dashboard.py b/src/build/comment_to_dashboard.py
new file mode 100644
index 0000000000000000000000000000000000000000..478857e07cd0a6b036d59eda477eb44a5b1cca0d
--- /dev/null
+++ b/src/build/comment_to_dashboard.py
@@ -0,0 +1,136 @@
+# -*- coding: utf-8 -*-
+"""
+# **********************************************************************************
+# Copyright (c) Huawei Technologies Co., Ltd. 2020-2020. All rights reserved.
+# [openeuler-jenkins] is licensed under the Mulan PSL v1.
+# You can use this software according to the terms and conditions of the Mulan PSL v1.
+# You may obtain a copy of Mulan PSL v1 at:
+# http://license.coscl.org.cn/MulanPSL
+# THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR
+# PURPOSE.
+# See the Mulan PSL v1 for more details.
+# Author:
+# Create: 2020-09-23
+# Description: comment pr with build result to dashboard
+# **********************************************************************************
+"""
+
+import os
+import argparse
+import time
+from datetime import datetime
+
+import yaml
+
+from src.proxy.kafka_proxy import KafkaProducerProxy
+from src.logger import logger
+
+
+class CommentToDashboard(object):
+ """
+ comments process
+ """
+
+ @staticmethod
+ def output_build_num(args_list):
+ """
+ output_build_num
+ :param args_list:
+ :return:
+ """
+ build_num_list = []
+ build_num_file = "{}_{}_{}_build_num.yaml".format(args_list.owner, args_list.repo, args_list.prid)
+ try:
+ if os.path.exists(build_num_file):
+ with open(build_num_file, "r") as f:
+ build_num_list = yaml.safe_load(f)
+ except yaml.MarkedYAMLError:
+ logger.exception("Read trigger build number file exception, yaml format error")
+ if args_list.trigger_build_id not in build_num_list:
+ build_num_list.append(args_list.trigger_build_id)
+ logger.info("build_num_list = %s", build_num_list)
+ try:
+ with open(build_num_file, "w") as f:
+ yaml.safe_dump(build_num_list, f)
+ except IOError:
+ logger.exception("save build number file exception")
+
+ def get_all_result_to_kafka(self, args_list):
+ """
+ 名称 类型 必选 说明
+ pr_url 字符串 是 需要进行上报的pr地址，(pr_url, build_no)共同确定一次门禁结果
+ pr_title 字符串 是 pr标题
+ pr_create_at 数值 是 pr创建时间戳
+ pr_committer 字符串 是 pr提交人
+ pr_branch 字符串 是 pr目标分支
+ build_no 数值 是 门禁评论工程构建编号，区分同一个pr的多次门禁结果
+ build_at 数值 是 门禁触发时间戳
+ update_at 数值 是 当前时间对应的时间戳
+ build_exception 布尔 是 门禁执行是否异常，异常情况部分字段可以为空
+ build_urls 字典 是 包含多个门禁工程链接和显示文本
+ build_time 数值 是 整体构建时间(单位秒) trigger触发时间~comment时间
+ check_total 字符串 是 门禁整体结果
+ check_details 字典 是 门禁各个检查项结果
+ :return:
+ """
+ pr_create_time = round(datetime.timestamp(datetime.strptime(args_list.pr_create_time, '%Y-%m-%dT%H:%M:%S%z')), 1)
+ trigger_time = round(datetime.timestamp(datetime.strptime(args_list.trigger_time, '%Y-%m-%dT%H:%M:%S%z')), 1)
+ current_time = round(time.time(), 1)
+
+ base_dict = {"pr_title": args_list.pr_title,
+ "pr_url": args_list.pr_url,
+ "pr_create_at": pr_create_time,
+ "pr_committer": args_list.committer,
+ "pr_branch": args_list.tbranch,
+ "build_at": trigger_time,
+ "update_at": current_time,
+ "build_no": args_list.trigger_build_id
+ }
+ build_time = round(current_time - trigger_time, 1)
+ base_dict["build_time"] = build_time
+
+ self.output_build_num(args_list)
+ try:
+ build_file = "build_result.yaml"
+ if os.path.exists(build_file):
+ base_dict["build_exception"] = False
+ with open(build_file, "r") as f:
+ comments = yaml.safe_load(f)
+ base_dict.update(comments)
+ else:
+ base_dict["build_exception"] = True
+ except yaml.MarkedYAMLError:
+ logger.exception("Read build result file exception, yaml format error")
+
+ logger.info("base_dict = %s", base_dict)
+ # upload to es
+ kp = KafkaProducerProxy(brokers=os.environ["KAFKAURL"].split(","))
+ kp.send("openeuler_statewall_ci_result", key=args_list.comment_id, value=base_dict)
+
+
+def init_args():
+ """
+ init args
+ :return:
+ """
+ parser = argparse.ArgumentParser()
+ parser.add_argument("-r", type=str, dest="repo", help="repo name")
+ parser.add_argument("-c", type=str, dest="committer", help="commiter")
+ parser.add_argument("-m", type=str, dest="comment_id", help="uniq comment id")
+ parser.add_argument("-g", type=str, dest="trigger_time", help="job trigger time")
+ parser.add_argument("-k", type=str, dest="pr_title", help="pull request title")
+ parser.add_argument("-t", type=str, dest="pr_create_time", help="pull request create time")
+ parser.add_argument("-b", type=str, dest="tbranch", help="target branch")
+ parser.add_argument("-u", type=str, dest="pr_url", help="pull request url")
+ parser.add_argument("-p", type=str, dest="prid", help="pull request id")
+ parser.add_argument("-o", type=str, dest="owner", help="gitee owner")
+ parser.add_argument("-i", type=int, dest="trigger_build_id", help="trigger build id")
+
+ return parser.parse_args()
+
+
+if "__main__" == __name__:
+ args = init_args()
+ comment = CommentToDashboard()
+ comment.get_all_result_to_kafka(args)
diff --git a/src/build/gitee_comment.py b/src/build/gitee_comment.py
index 144957d3ac8903f397b6c1dd88ad986e9d6a3326..233865befe55954db35f4459e89aa90216d45987 100755
--- a/src/build/gitee_comment.py
+++ b/src/build/gitee_comment.py
@@ -17,9 +17,9 @@
"""
import os
+import re
import sys
import logging.config
-import logging
import json
import yaml
import argparse
@@ -47,6 +47,9 @@ class Comment(object):
self._up_builds = []
self._up_up_builds = []
self._get_upstream_builds(jenkins_proxy)
+ self.ac_result = {}
+ self.compare_package_result = {}
+ self.check_item_result = {}
def comment_build(self, gitee_proxy):
"""
@@ -154,7 +157,7 @@ class Comment(object):
else:
comments.append(self.__class__.comment_html_table_tr_rowspan(
item["name"], ac_result.emoji, ac_result.hint))
-
+ self.ac_result[item["name"]] = ac_result.hint
logger.info("ac comment: %s", comments)
return comments
@@ -182,6 +185,7 @@ class Comment(object):
logger.info("%s not exists", result_file)
continue
for build in self._up_builds:
+ arch_cmp_result = "SUCCESS"
name = JenkinsProxy.get_job_path_from_job_url(build["url"])
logger.info("check build %s", name)
arch_result, arch_name = match(name, result_file)
@@ -203,6 +207,8 @@ class Comment(object):
rpm_name = content.get(item)
check_item = item.replace(" ", "_")
result = "FAILED" if rpm_name else "SUCCESS"
+ if result == "FAILED":
+ arch_cmp_result = "FAILED"
compare_result = ACResult.get_instance(result)
if index == 0:
comments.append("| compare_package({}) | {} | {} | "
@@ -213,7 +219,7 @@ class Comment(object):
else:
comments.append("
| {} | {} | {}{} |
".format(
check_item, "
".join(rpm_name), compare_result.emoji, compare_result.hint))
-
+ self.compare_package_result[arch_name] = arch_cmp_result
if comments:
comments = comments_title + comments
comments.append("")
@@ -247,7 +253,7 @@ class Comment(object):
arch = "aarch64"
else:
continue
-
+ arch_dict = {}
check_item_result = {}
for check_item_comment_file in self._check_item_comment_files:
if not os.path.exists(check_item_comment_file):
@@ -267,15 +273,82 @@ class Comment(object):
"#{} | ".format(
1 + len(check_item_result), arch, "check_build", ac_result.emoji, ac_result.hint,
1 + len(check_item_result), "{}{}".format(build_url, "console"), build["number"]))
-
+ arch_dict["check_build"] = ac_result.hint
for check_item, check_result in check_item_result.items():
comments.append("| {} | {}{} | ".format(
check_item, check_result.emoji, check_result.hint))
-
+ arch_dict[check_item] = check_result.hint
+ self.check_item_result[arch] = arch_dict
logger.info("check item comment: %s", comments)
return comments
+ def get_job_url(self, comment_url):
+ """
+ get_job_url
+ :param url:
+ :return:
+ """
+ build_urls = {"trigger": self._up_up_builds[0]["url"],
+ "comment": os.path.join(comment_url, os.environ.get("BUILD_ID"))
+ }
+ for build in self._up_builds:
+ arch = ""
+ try:
+ arch_index = 3
+ list_step = 2
+ if build["url"]:
+ job_path = re.sub(r"http[s]?://", "", build["url"])
+ arch = job_path.split("/")[::list_step][arch_index]
+ except IndexError:
+ logger.info("get arch from job failed, index error.")
+ except KeyError:
+ logger.info("not find build url key")
+ if arch:
+ build_urls[arch] = build["url"]
+
+ return build_urls
+
+ def get_all_job_result(self, check_details):
+ """
+ get_all_job_result
+ :return:
+ """
+
+ check_details["static_code"] = self.ac_result
+ for arch, arch_result in self.check_item_result.items():
+ if self.compare_package_result.get(arch):
+ arch_result["compare_package"] = self.compare_package_result.get(arch)
+ check_details[arch] = arch_result
+
+ return check_details
+
+ def get_all_result_to_kafka(self, comment_url):
+ """
+ 名称 类型 必选 说明
+ build_urls 字典 是 包含多个门禁工程链接和显示文本
+ check_total 字符串 是 门禁整体结果
+ check_details 字典 是 门禁各个检查项结果
+ :return:
+ """
+ check_details = {}
+ build_urls = self.get_job_url(comment_url)
+ self.get_all_job_result(check_details)
+
+ if self.check_build_result() == SUCCESS:
+ check_total = 'SUCCESS'
+ else:
+ check_total = 'FAILED'
+
+ all_dict = {"build_urls": build_urls,
+ "check_total": check_total,
+ "check_details": check_details
+ }
+ logger.info("all_dict = %s", all_dict)
+
+ with open("build_result.yaml", "w") as f:
+ yaml.safe_dump(all_dict, f)
+
@classmethod
def comment_html_table_th(cls):
"""
@@ -369,6 +442,8 @@ if "__main__" == __name__:
gp.create_tags_of_pr(args.pr, "ci_failed")
dd.set_attr("comment.build.tags", ["ci_failed"])
dd.set_attr("comment.build.result", "failed")
+ if args.owner != "openeuler":
+ comment.get_all_result_to_kafka(url)
logger.info("comment: at committer......")
comment.comment_at(args.committer, gp)
diff --git a/src/lib/comment.sh b/src/lib/comment.sh
index 27e131e44b1d1a5b3f0ff5377f2320b4afb1657c..cae8cebd32198ebbdb21710d89984b2bf37ef7a0 100644
--- a/src/lib/comment.sh
+++ b/src/lib/comment.sh
@@ -40,6 +40,7 @@ function clearn_env() {
#cat $compare_package_comment_x86
compare_package_result_aarch64="${repo}_${prid}_aarch64_compare_result"
compare_package_result_x86="${repo}_${prid}_x86_64_compare_result"
+ build_num_file="${repo_owner}_${repo}_${prid}_build_num.yaml"
if [[ -e check_item_comment_aarch64 ]]; then
rm $check_item_comment_aarch64
@@ -53,6 +54,9 @@ function clearn_env() {
if [[ -e $compare_package_result_x86 ]]; then
rm $compare_package_result_x86
fi
+ if [[ -e build_num_file ]]; then
+ rm $build_num_file
+ fi
log_info "***** End to clearn env *****"
}
@@ -66,6 +70,7 @@ function scp_comment_file() {
scp -r -i ${SaveBuildRPM2Repo} -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null root@${repo_server}:$fileserver_tmpfile_path/${compare_package_result_aarch64} . || log_info "file ${compare_package_result_aarch64} not exist"
scp -r -i ${SaveBuildRPM2Repo} -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null root@${repo_server}:$fileserver_tmpfile_path/${compare_package_result_x86} . || log_info "file ${compare_package_result_x86} not exist"
ls $WORKSPACE/${compare_result}
+ scp -r -i ${SaveBuildRPM2Repo} -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null root@${repo_server}:$fileserver_tmpfile_path/${build_num_file} . || log_info "file ${build_num_file} not exist"
log_info "***** End to scp comment file *****"
}
@@ -73,8 +78,14 @@ function scp_comment_file() {
function exec_comment() {
log_info "***** Start to exec comment *****"
export PYTHONPATH=${shell_path}
- python3 ${shell_path}/src/build/gitee_comment.py -o $repo_owner -r $repo -p $prid -c $committer -t ${giteetoken} -b $jenkins_api_host -u $jenkins_user -j $jenkins_api_token -a ${check_item_comment_aarch64} ${check_item_comment_x86} -f ${compare_package_result_x86},${compare_package_result_aarch64} -m ${commentid}
+ python3 ${shell_path}/src/build/gitee_comment.py -o $repo_owner -r $repo -p $prid -c $committer -t ${giteetoken}\
+ -b $jenkins_api_host -u $jenkins_user -j $jenkins_api_token -a ${check_item_comment_aarch64} ${check_item_comment_x86}\
+ -f ${compare_package_result_x86},${compare_package_result_aarch64} -m ${commentid}
log_info "***** End to exec comment *****"
+ log_info "***** Start to exec comment to kafka*****"
+ python3 ${shell_path}/src/build/comment_to_dashboard.py -r $repo -c $committer -m ${commentid} -g $jobtriggertime\
+ -k "${prtitle}" -t $prcreatetime -b $tbranch -u $prurl -i $triggerbuildid -p $prid -o $repo_owner
+ log_info "***** End to exec comment to kafka*****"
}
# 执行入口
@@ -82,4 +93,8 @@ function main() {
clearn_env
scp_comment_file
exec_comment
+ log_info "save build num file"
+ if [[ -e $build_num_file ]]; then
+ scp -r -i ${SaveBuildRPM2Repo} -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -o LogLevel=ERROR ${build_num_file} root@${repo_server}:$fileserver_tmpfile_path/${build_num_file}
+ fi
}